fix(gateway): isolate qualification evidence

gateway/gateway_test.go

@@ -28,6 +28,8 @@ import (
 	protocol "git.sechmachine.io.vn/sechmachine/VerseVDI-Protocol/gen/go/protocol"
 )
 
+const protocolTerminalReceiptVector = "VGF1\x00\x03\x00\x00"
+
 func TestFrameValidationAndFragmentation(t *testing.T) {
 	frames, err := FragmentPayload(ChannelVideo, 7, 11, make([]byte, 1180))
 	if err != nil || len(frames) != 2 || len(frames[0].Payload) != 1179 || len(frames[1].Payload) != 1 {
@@ -110,7 +112,7 @@ func TestClientFeedbackUsesFixedProtocolVGFVector(t *testing.T) {
 		t.Fatalf("decoded disconnected event = %#v, %v", event, err)
 	}
 	receipt, err := EncodeClientFeedback(Feedback{Kind: FeedbackTerminalReceipt})
-	if err != nil || hex.EncodeToString(receipt) != "5647463100030000" {
+	if err != nil || string(receipt) != protocolTerminalReceiptVector {
 		t.Fatalf("terminal receipt vector = %x, %v", receipt, err)
 	}
 }
@@ -893,11 +895,7 @@ func (c *independentGatewayClient) receiveProviderEvent(ctx context.Context, ack
 		}
 		event, err := DecodeProviderEvent(payload)
 		if acknowledge && err == nil && (event.Kind == ProviderEventTerminated || event.Kind == ProviderEventDisconnected) {
-			receipt, encodeErr := EncodeClientFeedback(Feedback{Kind: FeedbackTerminalReceipt})
-			if encodeErr != nil {
-				return ProviderEvent{}, encodeErr
-			}
-			ack, encodeErr := protocol.EncodeChannelFrame(testChannelFrame("control.ack.v1", 0, receipt))
+			ack, encodeErr := protocol.EncodeChannelFrame(testChannelFrame("control.ack.v1", 0, []byte(protocolTerminalReceiptVector)))
 			if encodeErr != nil {
 				return ProviderEvent{}, encodeErr
 			}

gateway/qualification_contract_test.go

@@ -1,11 +1,13 @@
 package gateway
 
 import (
+	"bytes"
 	"compress/gzip"
+	"context"
 	"encoding/binary"
+	"errors"
 	"io"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"reflect"
 	"strconv"
@@ -107,7 +109,7 @@ func TestQualificationShortProcessingWritesRawArtifact(t *testing.T) {
 	}
 	if summary.Count < 1 || summary.RawSamplesSHA256 == "" || summary.RawSamplesBytes < 1 ||
 		summary.ResourceSamples < 2 || summary.RawResourcesSHA256 == "" || summary.RawResourcesBytes < 1 ||
-		summary.CPUScope != "isolated gateway qualification process (gateway plus bounded fixture/client driver)" {
+		summary.CPUScope != qualificationGatewayCPUScope || summary.ClockOverhead <= 0 || summary.ClockMethod == "" {
 		t.Fatalf("processing summary = %#v", summary)
 	}
 	file, err := os.Open(rawPath)
@@ -126,11 +128,66 @@ func TestQualificationShortProcessingWritesRawArtifact(t *testing.T) {
 	if err := reader.Close(); err != nil {
 		t.Fatal(err)
 	}
-	if !strings.HasPrefix(string(raw), "elapsed_ns,processing_ns\n") || strings.Count(string(raw), "\n") != int(summary.Count)+1 {
+	if !strings.HasPrefix(string(raw), "elapsed_ns,queue_ns,processing_ns,pacing_ns\n") ||
+		strings.Count(string(raw), "\n") != int(summary.Count)+1 {
 		t.Fatalf("raw sample rows do not match summary count: %q", raw)
 	}
 }
 
+func TestQualificationShortProcessingSubprocessCoversFixedProfiles(t *testing.T) {
+	for _, profile := range qualificationMediaProfiles() {
+		profile.Duration = time.Second
+		profile.Warmup = 10 * time.Millisecond
+		summary, err := runQualificationProcessing(t, profile, filepath.Join(t.TempDir(), profile.Name+".csv.gz"))
+		if err != nil {
+			t.Fatalf("%s: %v", profile.Name, err)
+		}
+		if summary.Count < 1 || summary.CPUScope != qualificationGatewayCPUScope ||
+			summary.ClockOverhead <= 0 || summary.ClockMethod != qualificationClockOverheadMethod ||
+			summary.ObservedBitrateKbps < float64(profile.BitrateKbps)*0.95 ||
+			summary.ObservedBitrateKbps > float64(profile.BitrateKbps)*1.05 {
+			t.Fatalf("%s subprocess summary = %#v", profile.Name, summary)
+		}
+	}
+}
+
+func TestQualificationProcessingResourcesExcludeParentDriverCPU(t *testing.T) {
+	profile := qualificationMediaProfile{
+		Name: "resource-isolation", Codec: "h264", BitrateKbps: 1000,
+		Duration: 300 * time.Millisecond, Warmup: time.Millisecond, PacketBytes: 1000,
+	}
+	baseline, err := runQualificationProcessing(t, profile, filepath.Join(t.TempDir(), "baseline.csv.gz"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	stop := make(chan struct{})
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		var value uint64 = 1
+		for {
+			select {
+			case <-stop:
+				if value == 0 {
+					panic("bounded parent work was optimized away")
+				}
+				return
+			default:
+				value = value*2862933555777941757 + 3037000493
+			}
+		}
+	}()
+	busy, err := runQualificationProcessing(t, profile, filepath.Join(t.TempDir(), "busy.csv.gz"))
+	close(stop)
+	<-done
+	if err != nil {
+		t.Fatal(err)
+	}
+	if busy.CPUSeconds > baseline.CPUSeconds+50*time.Millisecond.Seconds() {
+		t.Fatalf("parent CPU leaked into gateway sample: baseline=%.6fs busy=%.6fs", baseline.CPUSeconds, busy.CPUSeconds)
+	}
+}
+
 func TestQualificationProcessingPreservesPayload(t *testing.T) {
 	profile := qualificationMediaProfiles()[0]
 	payload := qualificationPayload(profile)
@@ -169,6 +226,31 @@ func TestQualificationRepeatedTraversalTracksEveryProductionStage(t *testing.T)
 	}
 }
 
+func TestQualificationZeroLossBaselinesAreAttributedAtNormativeScale(t *testing.T) {
+	for _, media := range qualificationMediaProfiles() {
+		observation, err := runQualificationImpairment(t, qualificationImpairmentProfiles()[0], media,
+			qualificationImpairmentPacketCount, filepath.Join(t.TempDir(), media.Name+".csv.gz"))
+		if err != nil {
+			t.Fatalf("%s: %v", media.Name, err)
+		}
+		if observation.InjectedDropped != 0 || observation.Dropped != 0 ||
+			observation.ProviderFECDropped != 0 || observation.ProviderEnqueueDropped != 0 ||
+			observation.ProviderQueueReplaced != 0 || observation.GatewayDropped != 0 ||
+			observation.QUICSendDropped != 0 || observation.ClientDeliveryDropped != 0 ||
+			observation.UnexplainedDropped != 0 {
+			t.Fatalf("%s clean-path loss attribution = %#v", media.Name, observation)
+		}
+		if observation.SourceEmitted != qualificationImpairmentPacketCount ||
+			observation.ProviderRecovered != qualificationImpairmentPacketCount ||
+			observation.ProviderEnqueued != qualificationImpairmentPacketCount ||
+			observation.GatewayForwarded != qualificationImpairmentPacketCount ||
+			observation.QUICSent != qualificationImpairmentPacketCount ||
+			observation.Delivered != qualificationImpairmentPacketCount {
+			t.Fatalf("%s stage counts = %#v", media.Name, observation)
+		}
+	}
+}
+
 func TestQualificationImpairmentIsDeterministicAndBounded(t *testing.T) {
 	profile := qualificationImpairmentProfiles()[3]
 	first, err := runQualificationImpairment(t, profile, qualificationMediaProfiles()[0], 1000, filepath.Join(t.TempDir(), "first.csv.gz"))
@@ -237,81 +319,108 @@ func TestQualificationFixedSeedJitterIsObservableOnTraversedTraffic(t *testing.T
 	if observation.ObservedLatency < 5*time.Millisecond || observation.ObservedLatency > 100*time.Millisecond {
 		t.Fatalf("observed one-way latency %s does not reflect configured traversal", observation.ObservedLatency)
 	}
-	if observation.ObservedJitter < 5*time.Millisecond || observation.ObservedJitter > 80*time.Millisecond {
-		t.Fatalf("observed jitter %s is outside reviewed fixed-seed tolerance", observation.ObservedJitter)
+	if observation.AppliedJitter < 10*time.Millisecond || observation.AppliedJitter > 25*time.Millisecond {
+		t.Fatalf("applied fixed-seed jitter %s is outside the uniform-delay tolerance", observation.AppliedJitter)
 	}
-	if observation.ObservedOutOfOrder == 0 {
-		t.Fatal("fixed-seed jitter was serialized away before production traversal")
+	if observation.ObservedJitter <= 0 || observation.ObservedJitter > observation.AppliedJitter {
+		t.Fatalf("ordered traversal jitter %s is not bounded by applied jitter %s", observation.ObservedJitter, observation.AppliedJitter)
+	}
+	if observation.InjectedReordered != 0 || observation.ObservedOutOfOrder != 0 {
+		t.Fatalf("reorder-off jitter changed source order: %#v", observation)
 	}
 }
 
-func TestQualificationCPUTracksConsumedWorkNotIdleCapacity(t *testing.T) {
-	started := time.Now()
-	before := qualificationRuntimeSample(started)
-	time.Sleep(100 * time.Millisecond)
-	idle := qualificationRuntimeSample(started).CPUSeconds - before.CPUSeconds
-	if idle > 50*time.Millisecond.Seconds() {
-		t.Fatalf("idle CPU consumption = %.6fs, want at most 0.05s", idle)
-	}
-
-	workBefore := qualificationRuntimeSample(started).CPUSeconds
-	deadline := time.Now().Add(75 * time.Millisecond)
-	var value uint64 = 1
-	for time.Now().Before(deadline) {
-		value = value*6364136223846793005 + 1
-	}
-	if value == 0 {
-		t.Fatal("bounded CPU work was optimized away")
-	}
-	work := qualificationRuntimeSample(started).CPUSeconds - workBefore
-	if work <= idle || work < 20*time.Millisecond.Seconds() {
-		t.Fatalf("bounded work CPU = %.6fs, idle = %.6fs", work, idle)
-	}
-}
-
-func TestQualificationCPUIsolationExcludesParentTestWork(t *testing.T) {
-	output := filepath.Join(t.TempDir(), "cpu.txt")
-	command := exec.Command(os.Args[0], "-test.run=^TestQualificationCPUChild$", "-test.count=1")
-	command.Env = append(os.Environ(), "VERSEVDI_QUALIFICATION_CPU_CHILD="+output)
-	if err := command.Start(); err != nil {
-		t.Fatal(err)
-	}
-	deadline := time.Now().Add(150 * time.Millisecond)
-	var value uint64 = 1
-	for time.Now().Before(deadline) {
-		value = value*2862933555777941757 + 3037000493
-	}
-	if value == 0 {
-		t.Fatal("parent CPU work was optimized away")
-	}
-	if err := command.Wait(); err != nil {
-		t.Fatalf("CPU child: %v", err)
-	}
-	raw, err := os.ReadFile(output)
+func TestQualificationLossOnlyDoesNotImplicitlyReorder(t *testing.T) {
+	observation, err := runQualificationImpairment(
+		t,
+		qualificationImpairmentProfiles()[3],
+		qualificationMediaProfiles()[0],
+		400,
+		filepath.Join(t.TempDir(), "loss.csv.gz"),
+	)
 	if err != nil {
 		t.Fatal(err)
 	}
-	consumed, err := strconv.ParseFloat(string(raw), 64)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if consumed > 50*time.Millisecond.Seconds() {
-		t.Fatalf("isolated idle qualification process consumed %.6fs while parent test was busy", consumed)
+	if observation.InjectedReordered != 0 || observation.ObservedOutOfOrder != 0 {
+		t.Fatalf("loss-only profile changed source order: %#v", observation)
 	}
 }
 
-func TestQualificationCPUChild(t *testing.T) {
-	output := os.Getenv("VERSEVDI_QUALIFICATION_CPU_CHILD")
-	if output == "" {
-		return
-	}
-	started := time.Now()
-	before := qualificationRuntimeSample(started)
-	time.Sleep(200 * time.Millisecond)
-	consumed := qualificationRuntimeSample(started).CPUSeconds - before.CPUSeconds
-	if err := os.WriteFile(output, []byte(strconv.FormatFloat(consumed, 'f', 9, 64)), 0o600); err != nil {
+func TestQualificationExplicitReorderIsBoundedAndAttributed(t *testing.T) {
+	observation, err := runQualificationImpairment(
+		t,
+		qualificationImpairmentProfiles()[4],
+		qualificationMediaProfiles()[0],
+		400,
+		filepath.Join(t.TempDir(), "reorder.csv.gz"),
+	)
+	if err != nil {
 		t.Fatal(err)
 	}
+	if observation.InjectedReordered == 0 || observation.ObservedOutOfOrder != observation.InjectedReordered {
+		t.Fatalf("explicit reorder attribution = %#v", observation)
+	}
+}
+
+func TestQualificationGatewaySubprocessResourcesResetAndTrackWork(t *testing.T) {
+	profile := qualificationMediaProfile{
+		Name: "resource-process", Codec: "h264", BitrateKbps: 100000,
+		Duration: time.Second, Warmup: time.Millisecond, PacketBytes: 1000,
+	}
+	path := newQualificationProcessingPath(t, profile, qualificationMediaPacerKbps(profile, 0))
+	defer path.Close()
+	output := t.TempDir()
+	record := func(name string, work func() error) qualificationProcessRecordResult {
+		t.Helper()
+		if err := path.process.startRecording(
+			filepath.Join(output, name+".csv.gz"),
+			filepath.Join(output, name+"-resources.csv.gz"),
+		); err != nil {
+			t.Fatal(err)
+		}
+		if err := work(); err != nil {
+			t.Fatal(err)
+		}
+		result, err := path.process.stopRecording()
+		if err != nil {
+			t.Fatal(err)
+		}
+		return result
+	}
+	idle := record("idle", func() error {
+		time.Sleep(150 * time.Millisecond)
+		return nil
+	})
+	payload := qualificationPayload(profile)
+	work := record("work", func() error {
+		for index := 0; index < 500; index++ {
+			if _, err := path.emit(t, payload); err != nil {
+				return err
+			}
+			recovered, err := path.receivePayload(context.Background())
+			if err != nil || !bytes.Equal(recovered, payload) {
+				return errors.New("gateway subprocess work payload mismatch")
+			}
+		}
+		return nil
+	})
+	secondIdle := record("idle-again", func() error {
+		time.Sleep(150 * time.Millisecond)
+		return nil
+	})
+	if idle.CPUSeconds > 50*time.Millisecond.Seconds() || secondIdle.CPUSeconds > 50*time.Millisecond.Seconds() {
+		t.Fatalf("idle gateway CPU was reported as consumed work: first=%.6fs second=%.6fs", idle.CPUSeconds, secondIdle.CPUSeconds)
+	}
+	if work.CPUSeconds <= idle.CPUSeconds || work.Count != 500 || work.Mallocs == 0 ||
+		work.AllocatedBytes == 0 || work.PeakHeapBytes == 0 || work.PeakGoroutines == 0 {
+		t.Fatalf("gateway work resource sample = %#v idle=%#v", work, idle)
+	}
+	if secondIdle.Mallocs >= work.Mallocs || secondIdle.AllocatedBytes >= work.AllocatedBytes {
+		t.Fatalf("successive recording inherited counters: work=%#v second=%#v", work, secondIdle)
+	}
+	if work.ClockOverhead <= 0 || work.ClockMethod != qualificationClockOverheadMethod {
+		t.Fatalf("clock overhead evidence = %#v", work)
+	}
 }
 
 func qualificationRawMeanLatency(t *testing.T, path string) time.Duration {

gateway/qualification_harness_test.go

@@ -38,12 +38,14 @@ import (
 )
 
 const (
-	qualificationToolVersion                   = "versevdi-gateway-qualification/v4"
+	qualificationToolVersion                   = "versevdi-gateway-qualification/v5"
 	qualificationImpairmentQueuePackets        = 64
 	qualificationImpairmentMaxPackets          = 100_000
 	qualificationImpairmentPacketCount         = 10_000
 	qualificationProcessingLimit               = 5 * time.Millisecond
 	qualificationImpairmentSeed         uint64 = 0x3c6a11ce
+	qualificationClockOverheadMethod           = "median of 1000 batches of 100 monotonic time reads"
+	qualificationGatewayCPUScope               = "isolated gateway subprocess; bounded recorder/control included, fixture and client driver excluded"
 )
 
 type qualificationMediaProfile struct {
@@ -74,10 +76,12 @@ type qualificationPath struct {
 	session   *nativeApolloSession
 	fixture   *qualificationApolloFixture
 	backend   *qualificationTracingBackend
+	process   *qualificationGatewayProcess
 	key       []byte
 	flow      string
 	frame     uint32
 	bootTrace atomic.Bool
+	sourceUDP atomic.Uint64
 	closeOnce sync.Once
 	shutdown  func()
 }
@@ -109,6 +113,7 @@ type qualificationProcessingSummary struct {
 	Mean                  time.Duration  `json:"mean_ns"`
 	StandardDeviation     time.Duration  `json:"standard_deviation_ns"`
 	ClockOverhead         time.Duration  `json:"clock_overhead_ns"`
+	ClockMethod           string         `json:"clock_overhead_method"`
 	Histogram             map[string]int `json:"histogram"`
 	PayloadSHA256         string         `json:"payload_sha256"`
 	RawSamples            string         `json:"raw_samples"`
@@ -135,6 +140,20 @@ type qualificationImpairmentObservation struct {
 	Dropped                  int                         `json:"dropped"`
 	InjectedDropped          int                         `json:"injected_dropped"`
 	InjectedReordered        int                         `json:"injected_reordered"`
+	SourceEmitted            int                         `json:"source_emitted"`
+	SourceDatagrams          uint64                      `json:"source_datagrams"`
+	ProviderIngressDatagrams uint64                      `json:"provider_ingress_datagrams"`
+	ProviderRecovered        int                         `json:"provider_recovered"`
+	ProviderFECDropped       int                         `json:"provider_fec_dropped"`
+	ProviderEnqueued         int                         `json:"provider_enqueued"`
+	ProviderEnqueueDropped   int                         `json:"provider_enqueue_dropped"`
+	ProviderQueueReplaced    int                         `json:"provider_queue_replaced"`
+	GatewayForwarded         int                         `json:"gateway_forwarded"`
+	GatewayDropped           int                         `json:"gateway_dropped"`
+	QUICSent                 int                         `json:"quic_sent"`
+	QUICSendDropped          int                         `json:"quic_send_dropped"`
+	ClientDeliveryDropped    int                         `json:"client_delivery_dropped"`
+	UnexplainedDropped       int                         `json:"unexplained_dropped"`
 	ObservedOutOfOrder       int                         `json:"observed_out_of_order"`
 	ObservedLatency          time.Duration               `json:"observed_one_way_latency_ns"`
 	ObservedRTT              time.Duration               `json:"observed_rtt_ns"`
@@ -146,6 +165,7 @@ type qualificationImpairmentObservation struct {
 	MaxQueuePackets          int                         `json:"max_queue_packets"`
 	ConfiguredRTT            time.Duration               `json:"configured_rtt_ns"`
 	ConfiguredJitter         time.Duration               `json:"configured_jitter_ns"`
+	AppliedJitter            time.Duration               `json:"applied_jitter_ns"`
 	ConfiguredLossPercent    float64                     `json:"configured_loss_percent"`
 	ConfiguredReorder        bool                        `json:"configured_reorder"`
 	ConfiguredCapacitySteps  []int                       `json:"configured_capacity_steps_percent"`
@@ -255,6 +275,11 @@ func qualificationPayload(profile qualificationMediaProfile) []byte {
 	return payload
 }
 
+func qualificationMediaPacerKbps(profile qualificationMediaProfile, reduction int) int64 {
+	payloadKbps := profile.BitrateKbps * int64(100-reduction) / 100
+	return (payloadKbps*int64(profile.PacketBytes+frameHeaderSize) + int64(profile.PacketBytes) - 1) / int64(profile.PacketBytes)
+}
+
 type qualificationTracingBackend struct {
 	native   *NativeApolloBackend
 	setups   atomic.Uint64
@@ -867,6 +892,46 @@ func newQualificationPathWithImpairment(t *testing.T, profile qualificationMedia
 	return path
 }
 
+func newQualificationProcessingPath(t *testing.T, profile qualificationMediaProfile, pacerKbps int64) *qualificationPath {
+	t.Helper()
+	serverTLS, clientTLS := testTLS(t)
+	fixture := newQualificationApolloFixture(t, serverTLS, clientTLS, "qualification-session", profile)
+	authority := protocol.SessionAuthority{
+		Version: "1", SessionID: "qualification-session", GatewayID: "gateway-1",
+		Audience: "versevdi-gateway", ReconnectSequence: 0,
+		ExpiresAt:    time.Now().Add(45 * time.Minute).UTC().Format(time.RFC3339Nano),
+		Capabilities: DefaultCapabilities(), ProviderProfile: ProviderProfileApollo,
+		ProviderIdentity: fixture.work.ProviderIdentity,
+	}
+	work := fixture.work
+	work.ExpiresAt = authority.ExpiresAt
+	process := startQualificationGatewayProcess(t, serverTLS, authority, work, pacerKbps)
+	request := protocol.TunnelAdmissionRequest{
+		Version: "1", SessionID: authority.SessionID, GatewayID: authority.GatewayID,
+		Audience: authority.Audience, Grant: strings.Repeat("g", 64),
+		ClientNonce: "nonce-qualification", DeviceSignature: strings.Repeat("s", 86),
+		Capabilities: DefaultCapabilities(),
+	}
+	client, err := Dial(context.Background(), process.ready.GatewayAddress, clientTLS, request)
+	if err != nil {
+		process.Close()
+		t.Fatal(err)
+	}
+	key, err := fixture.streamKey()
+	if err != nil {
+		_ = client.Close()
+		process.Close()
+		t.Fatal(err)
+	}
+	path := &qualificationPath{client: client, fixture: fixture, process: process, key: key}
+	path.shutdown = func() {
+		_ = client.Close()
+		process.Close()
+	}
+	t.Cleanup(path.Close)
+	return path
+}
+
 func (p *qualificationPath) Close() {
 	if p != nil {
 		p.closeOnce.Do(p.shutdown)
@@ -913,7 +978,7 @@ func (p *qualificationPath) traverse(t *testing.T, payload []byte) (qualificatio
 
 func (p *qualificationPath) emit(t *testing.T, payload []byte) (qualificationPathTrace, error) {
 	t.Helper()
-	if p == nil || p.session == nil || len(payload) == 0 || len(payload) > 2*apolloVideoShardPayloadSize-8 {
+	if p == nil || p.fixture == nil || len(payload) == 0 || len(payload) > 2*apolloVideoShardPayloadSize-8 {
 		return qualificationPathTrace{}, ErrProviderMalformed
 	}
 	p.frame++
@@ -923,6 +988,7 @@ func (p *qualificationPath) emit(t *testing.T, payload []byte) (qualificationPat
 	if err := p.fixture.sendVideo(ctx, packets); err != nil {
 		return qualificationPathTrace{}, err
 	}
+	p.sourceUDP.Add(uint64(len(packets)))
 	return qualificationPathTrace{}, nil
 }
 
@@ -1140,7 +1206,7 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 		ConfiguredCapacitySteps: append([]int(nil), profile.CapacitySteps...),
 		RTTSource:               "apollo_enet_acknowledge",
 	}
-	path := newQualificationImpairedPath(t, media, media.BitrateKbps, profile)
+	path := newQualificationImpairedPath(t, media, qualificationMediaPacerKbps(media, 0), profile)
 	defer path.Close()
 	payload := qualificationPayload(media)
 	rawSamples := make([]rawSample, packetCount)
@@ -1166,31 +1232,36 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 		jobs = append(jobs, scheduledPacket{index: index, target: time.Duration(index)*spacing + delay})
 		rawSamples[index].outcome = "traversal_dropped"
 	}
+	var jitterMean, jitterM2 float64
+	var jitterSamples int
+	for _, packet := range jobs {
+		applied := float64(packet.target - time.Duration(packet.index)*spacing - profile.RTT/2)
+		jitterSamples++
+		delta := applied - jitterMean
+		jitterMean += delta / float64(jitterSamples)
+		jitterM2 += delta * (applied - jitterMean)
+	}
+	if jitterSamples > 0 {
+		observation.AppliedJitter = time.Duration(math.Sqrt(jitterM2 / float64(jitterSamples)))
+	}
 	if profile.Reorder {
 		for index := 18; index+1 < packetCount; index += 20 {
 			first, second := jobByIndex[index], jobByIndex[index+1]
 			if first < 0 || second < 0 {
 				continue
 			}
-			earlier := min(jobs[first].target, jobs[second].target)
-			later := max(jobs[first].target, jobs[second].target)
-			jobs[second].target = earlier
-			jobs[first].target = later + time.Nanosecond
+			jobs[first], jobs[second] = jobs[second], jobs[first]
 			observation.InjectedReordered++
 		}
 	}
-	sort.SliceStable(jobs, func(first, second int) bool {
-		if jobs[first].target == jobs[second].target {
-			return jobs[first].index < jobs[second].index
-		}
-		return jobs[first].target < jobs[second].target
-	})
 
 	started := time.Now()
 	beforeMetrics := path.server.Metrics()
 	beforeIngress := path.session.mediaIngress.Load()
 	beforeRecovered := path.session.mediaRecovered.Load()
 	beforeEnqueued := path.session.mediaEnqueued.Load()
+	beforeProviderDrops := path.session.mediaDrops.Load()
+	beforeSourceUDP := path.sourceUDP.Load()
 	beforePacer := path.server.pacer.reservations.Load()
 	grace := max(2*profile.RTT+2*profile.Jitter, 2*time.Second)
 	lastTarget := time.Duration(packetCount) * spacing
@@ -1254,17 +1325,28 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 	}()
 
 	stepAt := make(map[int]time.Time, len(profile.CapacitySteps))
+	emitted := 0
+	var previousRelease time.Time
+	maxCatchup := spacing
 	for _, packet := range jobs {
-		if delay := time.Until(started.Add(packet.target)); delay > 0 {
+		release := started.Add(packet.target)
+		if minimum := previousRelease.Add(spacing); !previousRelease.IsZero() && release.Before(minimum) {
+			release = minimum
+		}
+		if lag := time.Since(release); lag > maxCatchup {
+			release = release.Add(lag - maxCatchup)
+		}
+		if delay := time.Until(release); delay > 0 {
 			time.Sleep(delay)
 		}
+		previousRelease = release
 		if len(profile.CapacitySteps) == 2 {
 			switch {
 			case packet.index >= packetCount*2/3 && stepAt[profile.CapacitySteps[1]].IsZero():
-				path.server.pacer.setKbps(media.BitrateKbps * int64(100-profile.CapacitySteps[1]) / 100)
+				path.server.pacer.setKbps(qualificationMediaPacerKbps(media, profile.CapacitySteps[1]))
 				stepAt[profile.CapacitySteps[1]] = time.Now()
 			case packet.index >= packetCount/3 && stepAt[profile.CapacitySteps[0]].IsZero():
-				path.server.pacer.setKbps(media.BitrateKbps * int64(100-profile.CapacitySteps[0]) / 100)
+				path.server.pacer.setKbps(qualificationMediaPacerKbps(media, profile.CapacitySteps[0]))
 				stepAt[profile.CapacitySteps[0]] = time.Now()
 			}
 		}
@@ -1274,6 +1356,7 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 			receiveCancel()
 			return qualificationImpairmentObservation{}, err
 		}
+		emitted++
 	}
 	received := <-receivedDone
 	receiveCancel()
@@ -1316,6 +1399,23 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 		completedAt = received.packets[len(received.packets)-1].deliveredAt
 	}
 	afterMetrics := path.server.Metrics()
+	observation.SourceEmitted = emitted
+	observation.SourceDatagrams = path.sourceUDP.Load() - beforeSourceUDP
+	observation.ProviderIngressDatagrams = path.session.mediaIngress.Load() - beforeIngress
+	observation.ProviderRecovered = int(path.session.mediaRecovered.Load() - beforeRecovered)
+	observation.ProviderEnqueued = int(path.session.mediaEnqueued.Load() - beforeEnqueued)
+	observation.ProviderQueueReplaced = int(path.session.mediaDrops.Load() - beforeProviderDrops)
+	observation.GatewayForwarded = int(afterMetrics.ProcessingSamples - beforeMetrics.ProcessingSamples)
+	observation.QUICSent = int(afterMetrics.MediaPackets - beforeMetrics.MediaPackets)
+	observation.ProviderFECDropped = max(observation.SourceEmitted-observation.ProviderRecovered, 0)
+	observation.ProviderEnqueueDropped = max(observation.ProviderRecovered-observation.ProviderEnqueued, 0)
+	observation.GatewayDropped = max(observation.ProviderEnqueued-observation.ProviderQueueReplaced-observation.GatewayForwarded, 0)
+	observation.QUICSendDropped = max(observation.GatewayForwarded-observation.QUICSent, 0)
+	observation.ClientDeliveryDropped = max(observation.QUICSent-observation.Delivered, 0)
+	observation.UnexplainedDropped = observation.Dropped - observation.InjectedDropped -
+		observation.ProviderFECDropped - observation.ProviderEnqueueDropped -
+		observation.ProviderQueueReplaced - observation.GatewayDropped -
+		observation.QUICSendDropped - observation.ClientDeliveryDropped
 	if observation.Delivered > 0 && (path.session.mediaIngress.Load() <= beforeIngress ||
 		path.session.mediaRecovered.Load() <= beforeRecovered ||
 		path.session.mediaEnqueued.Load() <= beforeEnqueued ||
@@ -1391,6 +1491,7 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 		}
 	}
 	if observation.Delivered+observation.Dropped != observation.Sent ||
+		observation.UnexplainedDropped != 0 ||
 		observation.MaxQueuePackets > qualificationImpairmentQueuePackets {
 		return qualificationImpairmentObservation{}, errors.New("qualification impairment accounting invalid")
 	}
@@ -1469,75 +1570,58 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
 	if len(payload) < 4 {
 		return qualificationProcessingSummary{}, errors.New("qualification payload too small")
 	}
-	pacerKbps := (profile.BitrateKbps*int64(profile.PacketBytes+frameHeaderSize) + int64(profile.PacketBytes) - 1) / int64(profile.PacketBytes)
-	path := newQualificationPath(t, profile, pacerKbps)
+	path := newQualificationProcessingPath(t, profile, qualificationMediaPacerKbps(profile, 0))
 	defer path.Close()
-	if err := runQualificationWarmup(t, path, profile, payload); err != nil {
+	if err := runQualificationProcessWarmup(t, path, profile, payload); err != nil {
 		return qualificationProcessingSummary{}, err
 	}
-	file, err := os.OpenFile(rawPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640)
+	resourcePath := strings.TrimSuffix(rawPath, ".csv.gz") + "-resources.csv.gz"
+	if err := path.process.startRecording(rawPath, resourcePath); err != nil {
+		return qualificationProcessingSummary{}, err
+	}
+	before, err := path.process.snapshot()
 	if err != nil {
 		return qualificationProcessingSummary{}, err
 	}
-	compressed, err := gzip.NewWriterLevel(file, gzip.BestSpeed)
-	if err != nil {
-		_ = file.Close()
-		return qualificationProcessingSummary{}, err
-	}
-	buffered := bufio.NewWriterSize(compressed, 1<<20)
-	closed := false
-	defer func() {
-		if !closed {
-			_ = buffered.Flush()
-			_ = compressed.Close()
-			_ = file.Close()
-		}
-	}()
-	if _, err := buffered.WriteString("elapsed_ns,processing_ns\n"); err != nil {
-		return qualificationProcessingSummary{}, err
-	}
 	bytesPerSecond := profile.BitrateKbps * 1000 / 8
 	targetBytes := bytesPerSecond * profile.Duration.Nanoseconds() / int64(time.Second)
 	targetPackets := (targetBytes + int64(profile.PacketBytes) - 1) / int64(profile.PacketBytes)
-	samples := make([]time.Duration, 0, int(targetPackets))
 	started := time.Now()
-	resources := []qualificationResourceSample{qualificationRuntimeSample(started)}
-	lastResourceSample := started
 	var processed int64
 	for processed < targetPackets || time.Since(started) < profile.Duration {
 		current := append([]byte(nil), payload...)
 		binary.BigEndian.PutUint32(current[len(current)-4:], uint32(processed))
-		trace, sample, traverseErr := path.traverse(t, current)
-		if traverseErr != nil {
-			return qualificationProcessingSummary{}, traverseErr
+		if _, err := path.emit(t, current); err != nil {
+			return qualificationProcessingSummary{}, err
 		}
-		if !trace.NativeUDPIngress || !trace.ApolloRecovered || !trace.ProductionQueue ||
-			!trace.ProductionMediaLoop || !trace.ProductionPacer || !trace.VerseQUIC ||
-			!trace.PublicClientDecode || !trace.PayloadPreserved {
-			return qualificationProcessingSummary{}, fmt.Errorf("qualification bypassed the production provider-to-client path: %#v", trace)
-		}
-		samples = append(samples, sample)
-		if _, writeErr := fmt.Fprintf(buffered, "%d,%d\n", time.Since(started).Nanoseconds(), sample.Nanoseconds()); writeErr != nil {
-			return qualificationProcessingSummary{}, writeErr
+		recovered, err := path.receivePayload(context.Background())
+		if err != nil || !bytes.Equal(recovered, current) {
+			return qualificationProcessingSummary{}, errors.New("qualification processing payload integrity failure")
 		}
 		processed++
-		if time.Since(lastResourceSample) >= time.Second {
-			resources = append(resources, qualificationRuntimeSample(started))
-			lastResourceSample = time.Now()
-		}
 	}
 	actualDuration := time.Since(started)
-	resources = append(resources, qualificationRuntimeSample(started))
-	if err := buffered.Flush(); err != nil {
+	record, err := path.process.stopRecording()
+	if err != nil {
 		return qualificationProcessingSummary{}, err
 	}
-	if err := compressed.Close(); err != nil {
+	after, err := path.process.snapshot()
+	if err != nil {
 		return qualificationProcessingSummary{}, err
 	}
-	if err := file.Close(); err != nil {
+	if after.NativeSetups != 1 || after.NativeOpens != 1 ||
+		after.MediaRecovered-before.MediaRecovered != uint64(processed) ||
+		after.MediaEnqueued-before.MediaEnqueued != uint64(processed) ||
+		after.MediaDrops != before.MediaDrops ||
+		after.Metrics.ProcessingSamples-before.Metrics.ProcessingSamples != uint64(processed) ||
+		after.PacerReservations <= before.PacerReservations ||
+		after.Metrics.MediaPackets-before.Metrics.MediaPackets != uint64(processed) {
+		return qualificationProcessingSummary{}, fmt.Errorf("qualification subprocess bypassed a production stage: before=%#v after=%#v processed=%d", before, after, processed)
+	}
+	samples, err := readQualificationProcessingSamples(rawPath, record.Count)
+	if err != nil {
 		return qualificationProcessingSummary{}, err
 	}
-	closed = true
 	summary, err := summarizeQualificationSamples(samples)
 	if err != nil {
 		return qualificationProcessingSummary{}, err
@@ -1553,15 +1637,12 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
 	summary.Warmup = profile.Warmup
 	summary.ConfiguredDuration = profile.Duration
 	summary.ActualDuration = actualDuration
-	summary.ClockOverhead = qualificationClockOverhead()
+	summary.ClockOverhead = record.ClockOverhead
+	summary.ClockMethod = record.ClockMethod
 	summary.PayloadSHA256 = fmt.Sprintf("%x", sha256.Sum256(payload))
 	summary.RawSamples = filepath.Base(rawPath)
 	summary.RawSamplesSHA256 = sum
 	summary.RawSamplesBytes = size
-	resourcePath := strings.TrimSuffix(rawPath, ".csv.gz") + "-resources.csv.gz"
-	if err := writeQualificationResourceSamples(resourcePath, resources); err != nil {
-		return qualificationProcessingSummary{}, err
-	}
 	resourceSum, resourceSize, err := qualificationFileSHA256(resourcePath)
 	if err != nil {
 		return qualificationProcessingSummary{}, err
@@ -1569,23 +1650,16 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
 	summary.RawResources = filepath.Base(resourcePath)
 	summary.RawResourcesSHA256 = resourceSum
 	summary.RawResourcesBytes = resourceSize
-	summary.ResourceSamples = len(resources)
-	summary.CPUScope = "isolated gateway qualification process (gateway plus bounded fixture/client driver)"
-	firstResource, lastResource := resources[0], resources[len(resources)-1]
-	if firstResource.CPUSeconds < 0 || lastResource.CPUSeconds < 0 {
+	summary.ResourceSamples = record.ResourceSamples
+	summary.CPUScope = qualificationGatewayCPUScope
+	summary.CPUSeconds = record.CPUSeconds
+	summary.PeakHeapBytes = record.PeakHeapBytes
+	summary.PeakGoroutines = record.PeakGoroutines
+	summary.Mallocs = record.Mallocs
+	summary.AllocatedBytes = record.AllocatedBytes
+	if summary.CPUSeconds < 0 || summary.ClockOverhead <= 0 || summary.ClockMethod == "" {
 		return qualificationProcessingSummary{}, errors.New("process CPU usage unavailable")
 	}
-	summary.CPUSeconds = math.Max(0, lastResource.CPUSeconds-firstResource.CPUSeconds)
-	summary.Mallocs = lastResource.Mallocs - firstResource.Mallocs
-	summary.AllocatedBytes = lastResource.Allocated - firstResource.Allocated
-	for _, resource := range resources {
-		if resource.HeapBytes > summary.PeakHeapBytes {
-			summary.PeakHeapBytes = resource.HeapBytes
-		}
-		if resource.Goroutines > summary.PeakGoroutines {
-			summary.PeakGoroutines = resource.Goroutines
-		}
-	}
 	if actualDuration < profile.Duration || actualDuration > profile.Duration*105/100+250*time.Millisecond {
 		return qualificationProcessingSummary{}, fmt.Errorf("wall-clock duration %s outside [%s,%s]", actualDuration, profile.Duration, profile.Duration*105/100+250*time.Millisecond)
 	}
@@ -1598,6 +1672,57 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
 	return summary, nil
 }
 
+func readQualificationProcessingSamples(path string, expected int) ([]time.Duration, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+	compressed, err := gzip.NewReader(file)
+	if err != nil {
+		return nil, err
+	}
+	defer compressed.Close()
+	scanner := bufio.NewScanner(compressed)
+	if !scanner.Scan() || scanner.Text() != "elapsed_ns,queue_ns,processing_ns,pacing_ns" {
+		return nil, errors.New("qualification processing header invalid")
+	}
+	samples := make([]time.Duration, 0, expected)
+	for scanner.Scan() {
+		fields := strings.Split(scanner.Text(), ",")
+		if len(fields) != 4 {
+			return nil, errors.New("qualification processing row invalid")
+		}
+		value, err := strconv.ParseInt(fields[2], 10, 64)
+		if err != nil || value < 0 {
+			return nil, errors.New("qualification processing sample invalid")
+		}
+		samples = append(samples, time.Duration(value))
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+	if len(samples) != expected {
+		return nil, fmt.Errorf("qualification processing rows=%d want=%d", len(samples), expected)
+	}
+	return samples, nil
+}
+
+func runQualificationProcessWarmup(t *testing.T, path *qualificationPath, profile qualificationMediaProfile, payload []byte) error {
+	t.Helper()
+	started := time.Now()
+	for time.Since(started) < profile.Warmup {
+		if _, err := path.emit(t, payload); err != nil {
+			return err
+		}
+		recovered, err := path.receivePayload(context.Background())
+		if err != nil || !bytes.Equal(recovered, payload) {
+			return errors.New("qualification warmup payload integrity failure")
+		}
+	}
+	return nil
+}
+
 func runQualificationWarmup(t *testing.T, path *qualificationPath, profile qualificationMediaProfile, payload []byte) error {
 	t.Helper()
 	started := time.Now()
@@ -1657,11 +1782,17 @@ func writeQualificationResourceSamples(path string, samples []qualificationResou
 }
 
 func qualificationClockOverhead() time.Duration {
-	samples := make([]time.Duration, 10_000)
+	const readsPerBatch = 100
+	samples := make([]time.Duration, 1000)
+	var observed time.Time
 	for index := range samples {
 		started := time.Now()
-		samples[index] = time.Since(started)
+		for range readsPerBatch {
+			observed = time.Now()
 		}
+		samples[index] = time.Since(started) / readsPerBatch
+	}
+	runtime.KeepAlive(observed)
 	summary, _ := summarizeQualificationSamples(samples)
 	return summary.Median
 }
@@ -2000,10 +2131,10 @@ func TestSection7Qualification(t *testing.T) {
 		StartedAt: started.Format(time.RFC3339Nano), GoVersion: runtime.Version(),
 		ToolVersions: toolVersions,
 		OS:           runtime.GOOS, Architecture: runtime.GOARCH,
-		Topology:        "source-shaped encrypted Apollo fixture -> native recovery/FEC -> bounded provider queue -> production fair pacer -> Verse framing over mTLS/QUIC -> public Verse client decoder",
+		Topology:        "parent source-shaped encrypted Apollo fixture -> isolated gateway subprocess for processing/resource evidence -> public Verse client decoder; impairment uses the same native recovery/FEC, bounded queue, production pacer, framing, and QUIC path",
 		Direction:       "provider_to_client",
-		QueueDiscipline: "bounded 16-packet native provider queue, production equal-tier fair pacer, deterministic fixed-seed source impairment",
-		Evidence:        []string{"deterministic source-shaped Apollo recovery", "local real-time production path", "mTLS/QUIC fixture transport", "path impairment", "production fair pacer"},
+		QueueDiscipline: "ordered fixed-seed source delay queue with one-serialization-interval catch-up, bounded 16-packet native provider queue, production equal-tier fair pacer",
+		Evidence:        []string{"deterministic source-shaped Apollo recovery", "isolated gateway-process resources", "local real-time production path", "mTLS/QUIC fixture transport", "attributed path impairment", "production fair pacer"},
 		Deferred:        []string{"live Apollo", "macOS client", "physical firewall and packet route", "real encoder fidelity", "multi-host scale"},
 	}
 	for _, profile := range media {

gateway/qualification_process_test.go

@@ -0,0 +1,485 @@
+package gateway
+
+import (
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	protocol "git.sechmachine.io.vn/sechmachine/VerseVDI-Protocol/gen/go/protocol"
+)
+
+const qualificationProcessTokenHeader = "X-VerseVDI-Qualification-Token"
+
+type qualificationGatewayProcessConfig struct {
+	ServerCertificatePEM string
+	ServerPrivateKeyPEM  string
+	ClientCAPEM          string
+	Authority            protocol.SessionAuthority
+	Work                 protocol.ProviderSessionWork
+	PacerKbps            int64
+	ReadyPath            string
+	Token                string
+}
+
+type qualificationGatewayProcessReady struct {
+	GatewayAddress string
+	ControlAddress string
+}
+
+type qualificationGatewayProcessSnapshot struct {
+	Metrics           MetricsSnapshot
+	NativeSetups      uint64
+	NativeOpens       uint64
+	MediaIngress      uint64
+	MediaRecovered    uint64
+	MediaEnqueued     uint64
+	MediaDrops        uint64
+	MediaQueueMaximum uint64
+	PacerReservations uint64
+	ProviderTelemetry ProviderTelemetry
+}
+
+type qualificationProcessRecordRequest struct {
+	RawPath      string
+	ResourcePath string
+}
+
+type qualificationProcessRecordResult struct {
+	Count            int
+	ClockOverhead    time.Duration
+	ClockMethod      string
+	ResourceSamples  int
+	CPUSeconds       float64
+	PeakHeapBytes    uint64
+	PeakGoroutines   int
+	Mallocs          uint64
+	AllocatedBytes   uint64
+	RecordingElapsed time.Duration
+}
+
+type qualificationProcessRecorder struct {
+	mu            sync.Mutex
+	active        bool
+	started       time.Time
+	rawFile       *os.File
+	rawCompressed *gzip.Writer
+	rawBuffered   *bufio.Writer
+	resourcePath  string
+	resources     []qualificationResourceSample
+	samples       int
+	recordErr     error
+	tickerStop    chan struct{}
+	tickerDone    chan struct{}
+	clock         time.Duration
+}
+
+func (r *qualificationProcessRecorder) start(request qualificationProcessRecordRequest) error {
+	if err := validateQualificationOutputDir(filepath.Dir(request.RawPath)); err != nil {
+		return err
+	}
+	if filepath.Dir(request.RawPath) != filepath.Dir(request.ResourcePath) || request.RawPath == request.ResourcePath {
+		return errors.New("qualification process output paths invalid")
+	}
+	file, err := os.OpenFile(request.RawPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640)
+	if err != nil {
+		return err
+	}
+	compressed, err := gzip.NewWriterLevel(file, gzip.BestSpeed)
+	if err != nil {
+		_ = file.Close()
+		return err
+	}
+	buffered := bufio.NewWriterSize(compressed, 1<<20)
+	if _, err = buffered.WriteString("elapsed_ns,queue_ns,processing_ns,pacing_ns\n"); err != nil {
+		_ = compressed.Close()
+		_ = file.Close()
+		return err
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if r.active {
+		_ = buffered.Flush()
+		_ = compressed.Close()
+		_ = file.Close()
+		return errors.New("qualification process recording already active")
+	}
+	clock := qualificationClockOverhead()
+	r.active = true
+	r.started = time.Now()
+	r.rawFile = file
+	r.rawCompressed = compressed
+	r.rawBuffered = buffered
+	r.resourcePath = request.ResourcePath
+	r.resources = []qualificationResourceSample{qualificationRuntimeSample(r.started)}
+	r.samples = 0
+	r.recordErr = nil
+	r.clock = clock
+	r.tickerStop = make(chan struct{})
+	r.tickerDone = make(chan struct{})
+	go r.sampleResources()
+	return nil
+}
+
+func (r *qualificationProcessRecorder) observe(observation mediaTimingObservation) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if !r.active || r.recordErr != nil {
+		return
+	}
+	_, r.recordErr = fmt.Fprintf(r.rawBuffered, "%d,%d,%d,%d\n",
+		time.Since(r.started).Nanoseconds(), observation.QueueDelay.Nanoseconds(),
+		observation.ProcessingDelay.Nanoseconds(), observation.PacingDelay.Nanoseconds())
+	r.samples++
+}
+
+func (r *qualificationProcessRecorder) sampleResources() {
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+	defer close(r.tickerDone)
+	for {
+		select {
+		case <-r.tickerStop:
+			return
+		case <-ticker.C:
+			r.mu.Lock()
+			if r.active {
+				r.resources = append(r.resources, qualificationRuntimeSample(r.started))
+			}
+			r.mu.Unlock()
+		}
+	}
+}
+
+func (r *qualificationProcessRecorder) stop() (qualificationProcessRecordResult, error) {
+	r.mu.Lock()
+	if !r.active {
+		r.mu.Unlock()
+		return qualificationProcessRecordResult{}, errors.New("qualification process recording is not active")
+	}
+	r.active = false
+	stop, done := r.tickerStop, r.tickerDone
+	r.mu.Unlock()
+	close(stop)
+	<-done
+
+	r.mu.Lock()
+	r.resources = append(r.resources, qualificationRuntimeSample(r.started))
+	elapsed := time.Since(r.started)
+	err := r.recordErr
+	if flushErr := r.rawBuffered.Flush(); err == nil {
+		err = flushErr
+	}
+	if closeErr := r.rawCompressed.Close(); err == nil {
+		err = closeErr
+	}
+	if closeErr := r.rawFile.Close(); err == nil {
+		err = closeErr
+	}
+	resources := append([]qualificationResourceSample(nil), r.resources...)
+	result := qualificationProcessRecordResult{
+		Count: r.samples, ClockOverhead: r.clock, ClockMethod: qualificationClockOverheadMethod,
+		ResourceSamples: len(resources), RecordingElapsed: elapsed,
+	}
+	resourcePath := r.resourcePath
+	r.mu.Unlock()
+	if err != nil {
+		return qualificationProcessRecordResult{}, err
+	}
+	if err := writeQualificationResourceSamples(resourcePath, resources); err != nil {
+		return qualificationProcessRecordResult{}, err
+	}
+	first, last := resources[0], resources[len(resources)-1]
+	result.CPUSeconds = max(last.CPUSeconds-first.CPUSeconds, 0)
+	result.Mallocs = last.Mallocs - first.Mallocs
+	result.AllocatedBytes = last.Allocated - first.Allocated
+	for _, sample := range resources {
+		result.PeakHeapBytes = max(result.PeakHeapBytes, sample.HeapBytes)
+		result.PeakGoroutines = max(result.PeakGoroutines, sample.Goroutines)
+	}
+	return result, nil
+}
+
+type qualificationGatewayProcess struct {
+	command *exec.Cmd
+	cancel  context.CancelFunc
+	done    chan error
+	output  *bytes.Buffer
+	ready   qualificationGatewayProcessReady
+	token   string
+	client  *http.Client
+	once    sync.Once
+}
+
+func startQualificationGatewayProcess(t *testing.T, serverTLS *tls.Config, authority protocol.SessionAuthority, work protocol.ProviderSessionWork, pacerKbps int64) *qualificationGatewayProcess {
+	t.Helper()
+	temp := t.TempDir()
+	configPath := filepath.Join(temp, "gateway-config.json")
+	readyPath := filepath.Join(temp, "gateway-ready.json")
+	tokenBytes := make([]byte, 32)
+	if _, err := rand.Read(tokenBytes); err != nil {
+		t.Fatal(err)
+	}
+	config := qualificationGatewayProcessConfig{
+		ServerCertificatePEM: qualificationCertificateChainPEM(t, serverTLS.Certificates[0]),
+		ServerPrivateKeyPEM:  privateKeyPEM(t, serverTLS.Certificates[0]),
+		ClientCAPEM:          string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverTLS.Certificates[0].Certificate[1]})),
+		Authority:            authority, Work: work, PacerKbps: pacerKbps, ReadyPath: readyPath,
+		Token: hex.EncodeToString(tokenBytes),
+	}
+	encoded, err := json.Marshal(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(configPath, encoded, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestQualificationGatewayProcessChild$", "-test.count=1")
+	command.Env = append(os.Environ(), "VERSEVDI_QUALIFICATION_GATEWAY_CONFIG="+configPath)
+	output := &bytes.Buffer{}
+	command.Stdout, command.Stderr = output, output
+	if err := command.Start(); err != nil {
+		cancel()
+		t.Fatal(err)
+	}
+	process := &qualificationGatewayProcess{
+		command: command, cancel: cancel, done: make(chan error, 1), output: output,
+		token: config.Token, client: &http.Client{Timeout: 5 * time.Second},
+	}
+	go func() { process.done <- command.Wait() }()
+	deadline := time.Now().Add(10 * time.Second)
+	for time.Now().Before(deadline) {
+		raw, readErr := os.ReadFile(readyPath)
+		if readErr == nil && json.Unmarshal(raw, &process.ready) == nil &&
+			process.ready.GatewayAddress != "" && process.ready.ControlAddress != "" {
+			return process
+		}
+		select {
+		case waitErr := <-process.done:
+			cancel()
+			t.Fatalf("qualification gateway child exited before ready: %v\n%s", waitErr, output)
+		case <-time.After(10 * time.Millisecond):
+		}
+	}
+	process.Close()
+	t.Fatalf("qualification gateway child did not become ready\n%s", output)
+	return nil
+}
+
+func qualificationCertificateChainPEM(t *testing.T, certificate tls.Certificate) string {
+	t.Helper()
+	var encoded strings.Builder
+	for _, der := range certificate.Certificate {
+		if err := pem.Encode(&encoded, &pem.Block{Type: "CERTIFICATE", Bytes: der}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return encoded.String()
+}
+
+func (p *qualificationGatewayProcess) request(method, path string, body any, response any) error {
+	var reader io.Reader
+	if body != nil {
+		encoded, err := json.Marshal(body)
+		if err != nil {
+			return err
+		}
+		reader = bytes.NewReader(encoded)
+	}
+	request, err := http.NewRequest(method, "http://"+p.ready.ControlAddress+path, reader)
+	if err != nil {
+		return err
+	}
+	request.Header.Set(qualificationProcessTokenHeader, p.token)
+	result, err := p.client.Do(request)
+	if err != nil {
+		return err
+	}
+	defer result.Body.Close()
+	if result.StatusCode != http.StatusOK {
+		raw, _ := io.ReadAll(io.LimitReader(result.Body, 4096))
+		return fmt.Errorf("qualification gateway control %s: %s", result.Status, raw)
+	}
+	if response != nil {
+		return json.NewDecoder(io.LimitReader(result.Body, 1<<20)).Decode(response)
+	}
+	return nil
+}
+
+func (p *qualificationGatewayProcess) startRecording(rawPath, resourcePath string) error {
+	return p.request(http.MethodPost, "/record/start", qualificationProcessRecordRequest{RawPath: rawPath, ResourcePath: resourcePath}, nil)
+}
+
+func (p *qualificationGatewayProcess) stopRecording() (qualificationProcessRecordResult, error) {
+	var result qualificationProcessRecordResult
+	err := p.request(http.MethodPost, "/record/stop", nil, &result)
+	return result, err
+}
+
+func (p *qualificationGatewayProcess) snapshot() (qualificationGatewayProcessSnapshot, error) {
+	var result qualificationGatewayProcessSnapshot
+	err := p.request(http.MethodGet, "/snapshot", nil, &result)
+	return result, err
+}
+
+func (p *qualificationGatewayProcess) Close() {
+	if p == nil {
+		return
+	}
+	p.once.Do(func() {
+		_ = p.request(http.MethodPost, "/shutdown", nil, nil)
+		select {
+		case <-p.done:
+		case <-time.After(5 * time.Second):
+			p.cancel()
+			<-p.done
+		}
+		p.cancel()
+	})
+}
+
+func TestQualificationGatewayProcessChild(t *testing.T) {
+	configPath := os.Getenv("VERSEVDI_QUALIFICATION_GATEWAY_CONFIG")
+	if configPath == "" {
+		return
+	}
+	raw, err := os.ReadFile(configPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var config qualificationGatewayProcessConfig
+	if err := json.Unmarshal(raw, &config); err != nil {
+		t.Fatal(err)
+	}
+	certificate, err := tls.X509KeyPair([]byte(config.ServerCertificatePEM), []byte(config.ServerPrivateKeyPEM))
+	if err != nil {
+		t.Fatal(err)
+	}
+	clientCAs := x509.NewCertPool()
+	if !clientCAs.AppendCertsFromPEM([]byte(config.ClientCAPEM)) {
+		t.Fatal("qualification gateway client CA invalid")
+	}
+	serverTLS := &tls.Config{
+		MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{certificate},
+		ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: clientCAs,
+	}
+	admission := &oneTimeAdmission{
+		authority: config.Authority, released: make(chan struct{}), disableClipboard: true,
+		providerWork: &config.Work,
+	}
+	backend := &qualificationTracingBackend{native: NewNativeApolloBackend()}
+	recorder := &qualificationProcessRecorder{}
+	server, err := NewServer(ServerConfig{
+		ListenAddress: "127.0.0.1:0", TLSConfig: serverTLS, GatewayID: config.Authority.GatewayID,
+		Capabilities: DefaultCapabilities(), ProviderCapabilities: DefaultCapabilities(),
+		Admission: admission, ProviderStateReporter: &recordingProviderStateReporter{},
+		Provider: NewApolloAdapter(backend, ProviderIdentity{}), PacerKbps: config.PacerKbps,
+		mediaObserver: recorder.observe,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	serveDone := make(chan error, 1)
+	go func() { serveDone <- server.Serve(ctx) }()
+	shutdown := make(chan struct{})
+	var shutdownOnce sync.Once
+	handler := http.NewServeMux()
+	authorized := func(response http.ResponseWriter, request *http.Request) bool {
+		if request.Header.Get(qualificationProcessTokenHeader) != config.Token {
+			http.Error(response, "unauthorized", http.StatusUnauthorized)
+			return false
+		}
+		return true
+	}
+	handler.HandleFunc("/snapshot", func(response http.ResponseWriter, request *http.Request) {
+		if request.Method != http.MethodGet || !authorized(response, request) {
+			return
+		}
+		snapshot := qualificationGatewayProcessSnapshot{
+			Metrics: server.Metrics(), NativeSetups: backend.setups.Load(), NativeOpens: backend.opens.Load(),
+			PacerReservations: server.pacer.reservations.Load(),
+		}
+		if session := backend.session(config.Authority.SessionID); session != nil {
+			snapshot.MediaIngress = session.mediaIngress.Load()
+			snapshot.MediaRecovered = session.mediaRecovered.Load()
+			snapshot.MediaEnqueued = session.mediaEnqueued.Load()
+			snapshot.MediaDrops = session.mediaDrops.Load()
+			snapshot.MediaQueueMaximum = session.mediaQueueMaximum.Load()
+			snapshot.ProviderTelemetry = session.Telemetry()
+		}
+		_ = json.NewEncoder(response).Encode(snapshot)
+	})
+	handler.HandleFunc("/record/start", func(response http.ResponseWriter, request *http.Request) {
+		if request.Method != http.MethodPost || !authorized(response, request) {
+			return
+		}
+		var recordRequest qualificationProcessRecordRequest
+		if err := json.NewDecoder(io.LimitReader(request.Body, 4096)).Decode(&recordRequest); err != nil {
+			http.Error(response, "invalid record request", http.StatusBadRequest)
+			return
+		}
+		if err := recorder.start(recordRequest); err != nil {
+			http.Error(response, err.Error(), http.StatusConflict)
+		}
+	})
+	handler.HandleFunc("/record/stop", func(response http.ResponseWriter, request *http.Request) {
+		if request.Method != http.MethodPost || !authorized(response, request) {
+			return
+		}
+		result, err := recorder.stop()
+		if err != nil {
+			http.Error(response, err.Error(), http.StatusConflict)
+			return
+		}
+		_ = json.NewEncoder(response).Encode(result)
+	})
+	handler.HandleFunc("/shutdown", func(response http.ResponseWriter, request *http.Request) {
+		if request.Method != http.MethodPost || !authorized(response, request) {
+			return
+		}
+		shutdownOnce.Do(func() { close(shutdown) })
+	})
+	control, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	controlServer := &http.Server{Handler: handler, ReadHeaderTimeout: time.Second}
+	go func() { _ = controlServer.Serve(control) }()
+	ready, err := json.Marshal(qualificationGatewayProcessReady{
+		GatewayAddress: server.Addr().String(), ControlAddress: control.Addr().String(),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(config.ReadyPath, ready, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	<-shutdown
+	_, _ = recorder.stop()
+	cancel()
+	_ = server.Close()
+	_ = controlServer.Shutdown(context.Background())
+	if err := <-serveDone; err != nil {
+		t.Fatal(err)
+	}
+}

gateway/transport.go

@@ -77,6 +77,13 @@ type ServerConfig struct {
 	ProviderProfile        string
 	ProviderIdentity       string
 	PacerKbps              int64
+	mediaObserver          func(mediaTimingObservation)
+}
+
+type mediaTimingObservation struct {
+	QueueDelay      time.Duration
+	ProcessingDelay time.Duration
+	PacingDelay     time.Duration
 }
 
 type Server struct {
@@ -725,11 +732,17 @@ func (s *gatewaySession) sendMedia(channel byte, media ProviderMedia) error {
 		s.server.metrics.MediaPackets.Add(1)
 		s.server.metrics.MediaBytes.Add(uint64(len(encoded)))
 	}
-	processingDelay := media.EnqueuedAt.Sub(media.ReceivedAt) + time.Since(processingStarted) - pacingDelay
-	s.server.metrics.QueueDelayNanos.Add(uint64(dequeuedAt.Sub(media.EnqueuedAt)))
-	s.server.metrics.ProcessingDelayNanos.Add(uint64(max(processingDelay, 0)))
+	queueDelay := dequeuedAt.Sub(media.EnqueuedAt)
+	processingDelay := max(media.EnqueuedAt.Sub(media.ReceivedAt)+time.Since(processingStarted)-pacingDelay, 0)
+	s.server.metrics.QueueDelayNanos.Add(uint64(queueDelay))
+	s.server.metrics.ProcessingDelayNanos.Add(uint64(processingDelay))
 	s.server.metrics.PacingDelayNanos.Add(uint64(pacingDelay))
 	s.server.metrics.ProcessingSamples.Add(1)
+	if s.server.config.mediaObserver != nil {
+		s.server.config.mediaObserver(mediaTimingObservation{
+			QueueDelay: queueDelay, ProcessingDelay: processingDelay, PacingDelay: pacingDelay,
+		})
+	}
 	return nil
 }
 

openspec/changes/correct-gateway-qualification-evidence/.openspec.yaml

@@ -0,0 +1,2 @@
+schema: spec-driven
+created: 2026-07-30

openspec/changes/correct-gateway-qualification-evidence/design.md

@@ -0,0 +1,31 @@
+## Context
+
+The qualification driver already reaches the production Apollo-to-QUIC path, but its source shaper reorders jitter even when reorder is disabled, its packet accounting cannot identify unexplained loss, and in-process resource counters include the provider/client driver.
+
+## Goals / Non-Goals
+
+**Goals:**
+
+- Attribute every source unit to one bounded production-path outcome.
+- Keep impairment axes independently configured and observed.
+- Sample CPU, heap, allocation, and goroutine use from the gateway process only.
+- Record measured monotonic-clock overhead.
+
+**Non-Goals:**
+
+- No second simulator, profiling service, production dependency, or expanded impairment matrix.
+- No larger queues or relaxed acceptance limits without measured need.
+
+## Decisions
+
+- Reuse the existing source-boundary shaper, preserve source order unless explicit reorder is enabled, and limit catch-up to one media serialization interval. Record the fixed-seed applied-delay standard deviation separately from the jitter observed after ordered traversal.
+- Assign stable source sequence identifiers and retain per-stage counts so injected loss, provider/FEC drop, queue replacement, QUIC failure, and client miss are disjoint.
+- Reuse the established gateway child-test pattern for the actual gateway server; the Apollo fixture and QUIC client remain in the parent driver. A token-protected loopback test control endpoint starts and stops bounded child-owned recording and returns aggregate stage state.
+- Stream queue, processing, and pacing samples from the production `sendMedia` boundary to child-owned raw evidence. Sample child `RUSAGE_SELF`, Go heap, allocations, and goroutines once per second with independent per-run baselines.
+- Measure clock overhead as the median elapsed time per read across 1,000 batches of 100 monotonic reads and record that method.
+
+## Risks / Trade-offs
+
+- [Ordered release suppresses some delivered jitter] → Retain both the applied fixed-seed delay distribution and the separately observed ordered-traversal jitter.
+- [Stage attribution double-counts a unit] → Record one terminal outcome per source sequence and validate accounting equality.
+- [Process sampling perturbs qualification] → Use bounded low-rate samples and include the sampling method in evidence.

openspec/changes/correct-gateway-qualification-evidence/proposal.md

@@ -0,0 +1,25 @@
+## Why
+
+RC8 qualification evidence cannot support candidate readiness because clean traffic loses packets without stage attribution, reorder-off jitter reorders traffic, and resource counters include the provider/client driver rather than the gateway process alone.
+
+## What Changes
+
+- Attribute every production-path packet outcome at the source fixture, native provider queue, gateway forwarding, QUIC, and public-client stages.
+- Preserve source order for reorder-off profiles while retaining configured latency and jitter; inject bounded reorder only when enabled.
+- Measure gateway CPU, heap, allocations, and goroutines from the gateway process only, with isolated per-profile counters.
+- Measure and record bounded nonzero monotonic-clock overhead using a batched method.
+- Retain the existing six-profile matrix and real provider-to-public-client traversal.
+
+## Capabilities
+
+### New Capabilities
+
+None.
+
+### Modified Capabilities
+
+- `gateway-qualification`: Require attributable clean-path delivery, independent impairment axes, gateway-process-only resource evidence, and truthful timing-overhead evidence.
+
+## Impact
+
+P3C-002, P3C-026, P3C-028, P3C-029, and P3C-033; the GPL Data Plane qualification driver, production gateway subprocess boundary, raw evidence, and append-only Phase 3C-G evidence. No Protocol wire contract, provider route, transcode path, or closed Server dependency is introduced.

openspec/changes/correct-gateway-qualification-evidence/specs/gateway-qualification/spec.md

@@ -0,0 +1,27 @@
+## MODIFIED Requirements
+
+### Requirement: Fixed media processing qualification
+The qualification harness SHALL drive pinned-mTLS Apollo management, encrypted RTSP, ENet, and provider UDP through native source validation, `readUDPMedia`, recovery/FEC, bounded production queues, the production fair pacer, Verse framing/QUIC, and a public or independent client decoder for 1080p60 H.264 at 20 Mbps, 1440p120 HEVC at 50 Mbps, and 4K60 HEVC at 80 Mbps. After a recorded warm-up, the frozen candidate SHALL run each profile for ten wall-clock minutes, preserve encoded payload bytes, retain every monotonic processing sample plus bounded provider-queue observations, and report count, min, median, p90, p95, p99, max, mean, standard deviation, measured batched monotonic-clock overhead and method, and observed bitrate. Processing begins at complete provider-unit receipt and ends at QUIC handoff, excluding client transit and pacing. Queue delay SHALL measure provider-queue residence, processing SHALL measure gateway work before pacing, and pacing delay SHALL measure scheduler waiting. CPU, heap, allocations, and goroutines SHALL be measured from the isolated gateway process only; CPU SHALL be actual OS user plus system consumption and MUST NOT include idle wall capacity or unrelated parent fixture/client work. Successive profiles SHALL use independent resource-counter baselines. Any bypass, payload mutation, wall-duration violation, bitrate outside both lower and upper bounds, unexplained clean-path loss, zero or unbounded clock overhead, or p95 above 5 ms SHALL fail.
+
+#### Scenario: Healthy fixed profile
+- **WHEN** a frozen candidate runs one fixed profile for the normative duration in the isolated qualification command
+- **THEN** the harness emits compressed raw path and gateway-process resource samples plus a summary tied to the exact command, CPU scope, timing-overhead method, topology, source commit, immutable Protocol version, environment, and payload hash
+
+#### Scenario: Processing gate failure
+- **WHEN** any production path stage lacks a per-traversal observation, stage accounting does not balance, payload integrity fails, duration or bitrate bounds fail, measured p95 exceeds 5 ms, parent work changes gateway CPU, idle capacity is reported as consumed CPU, or timing overhead is absent
+- **THEN** the qualification command exits unsuccessfully without recording a passing candidate
+
+### Requirement: Bounded impairment qualification
+The harness SHALL run exactly the baseline, latency, jitter, loss, reorder, and constrained Section 7.2 profiles once by applying fixed-seed impairment at the source-shaped provider network boundary while traffic concurrently traverses the production gateway path. Baseline SHALL cover all three media profiles and the other profiles SHALL cover 1080p60. The harness MUST NOT serialize a complete provider-to-client traversal per source unit. Reorder-off profiles SHALL preserve source order through an ordered delay queue whose catch-up is limited to one media serialization interval; the fixed-seed applied-delay distribution and jitter observed after ordered traversal SHALL be reported separately. Loss-only traffic SHALL NOT gain implicit reorder. Reorder-on profiles SHALL inject and record only the fixed bounded reorder pattern. Each source unit SHALL have one attributable outcome across source emission, injected drop, native provider/FEC handling, bounded queue replacement, gateway forwarding, QUIC send/receive, and public-client delivery. Each artifact SHALL retain raw impairment and queue observations and record tool version, exact command/configuration, environment, candidate commit, immutable Protocol version, direction, queue discipline, topology, fixed seed, observed one-way latency, acknowledged Apollo ENet RTT, applied and observed jitter, injected and unexplained loss, reorder, throughput, drops, and capacity-step statistics.
+
+#### Scenario: Complete six-profile run
+- **WHEN** the frozen candidate runs impairment qualification
+- **THEN** one result exists for each named profile, configured and observed impairment axes remain separately attributable, reorder-off profiles preserve source order, RTT comes from real request/response acknowledgement timing, and raw statistics come from actual traversal
+
+#### Scenario: Clean production traversal
+- **WHEN** 10,000 source packets traverse a zero-loss baseline profile
+- **THEN** stage accounting identifies every packet and fails on any unexplained loss while each fixed media bitrate remains within its reviewed healthy-path contract
+
+#### Scenario: Unsupported or unbounded configuration
+- **WHEN** a profile name, packet count, queue bound, loss, reorder, or bandwidth step falls outside the fixed catalog
+- **THEN** the harness rejects it before allocating or running traffic

openspec/changes/correct-gateway-qualification-evidence/tasks.md

@@ -0,0 +1,23 @@
+## 1. Loss Attribution
+
+- [x] 1.1 Add a 10,000-packet production-path regression that records source, injected-drop, provider/FEC, queue, gateway, QUIC, and public-client outcomes
+- [x] 1.2 Reproduce and repair unexplained zero-loss baseline loss without relaxing bounds or hiding drops
+- [x] 1.3 Prove all three fixed baseline bitrates meet the healthy-path contract
+
+## 2. Impairment Semantics
+
+- [x] 2.1 Add fixed-seed regressions for reorder-off jitter, loss-only order, and bounded reorder-on behavior
+- [x] 2.2 Repair the existing source-boundary shaper and retain separately attributable configured and observed axes
+
+## 3. Resource and Timing Attribution
+
+- [x] 3.1 Add child-process regressions proving gateway-only CPU, heap, allocation, and goroutine samples
+- [x] 3.2 Prove parent CPU isolation, idle/work behavior, and independent per-profile counter baselines
+- [x] 3.3 Measure and record bounded nonzero batched monotonic-clock overhead and method
+
+## 4. Verification and Evidence
+
+- [x] 4.1 Run focused production-path, race, fuzz, cancellation, slow-reader, amplification, parser-resource, and bounded soak checks
+- [ ] 4.2 Run strict OpenSpec validation, normal-module verification, and reproducible Linux artifact inspection
+- [ ] 4.3 Freeze all executable inputs and run the corrected normative Section 7 qualification once for the candidate
+- [ ] 4.4 Preserve failed attempts and append superseding evidence and ledger rows without rewriting RC8