fix(gateway): isolate qualification evidence

gateway/gateway_test.go

@@ -28,6 +28,8 @@ import (
 	protocol "git.sechmachine.io.vn/sechmachine/VerseVDI-Protocol/gen/go/protocol"
 )
 
+const protocolTerminalReceiptVector = "VGF1\x00\x03\x00\x00"
+
 func TestFrameValidationAndFragmentation(t *testing.T) {
 	frames, err := FragmentPayload(ChannelVideo, 7, 11, make([]byte, 1180))
 	if err != nil || len(frames) != 2 || len(frames[0].Payload) != 1179 || len(frames[1].Payload) != 1 {
@@ -110,7 +112,7 @@ func TestClientFeedbackUsesFixedProtocolVGFVector(t *testing.T) {
 		t.Fatalf("decoded disconnected event = %#v, %v", event, err)
 	}
 	receipt, err := EncodeClientFeedback(Feedback{Kind: FeedbackTerminalReceipt})
-	if err != nil || hex.EncodeToString(receipt) != "5647463100030000" {
+	if err != nil || string(receipt) != protocolTerminalReceiptVector {
 		t.Fatalf("terminal receipt vector = %x, %v", receipt, err)
 	}
 }
@@ -893,11 +895,7 @@ func (c *independentGatewayClient) receiveProviderEvent(ctx context.Context, ack
 		}
 		event, err := DecodeProviderEvent(payload)
 		if acknowledge && err == nil && (event.Kind == ProviderEventTerminated || event.Kind == ProviderEventDisconnected) {
-			receipt, encodeErr := EncodeClientFeedback(Feedback{Kind: FeedbackTerminalReceipt})
-			if encodeErr != nil {
-				return ProviderEvent{}, encodeErr
-			}
-			ack, encodeErr := protocol.EncodeChannelFrame(testChannelFrame("control.ack.v1", 0, receipt))
+			ack, encodeErr := protocol.EncodeChannelFrame(testChannelFrame("control.ack.v1", 0, []byte(protocolTerminalReceiptVector)))
 			if encodeErr != nil {
 				return ProviderEvent{}, encodeErr
 			}

gateway/qualification_contract_test.go

@@ -1,11 +1,13 @@
 package gateway
 
 import (
+	"bytes"
 	"compress/gzip"
+	"context"
 	"encoding/binary"
+	"errors"
 	"io"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"reflect"
 	"strconv"
@@ -107,7 +109,7 @@ func TestQualificationShortProcessingWritesRawArtifact(t *testing.T) {
 	}
 	if summary.Count < 1 || summary.RawSamplesSHA256 == "" || summary.RawSamplesBytes < 1 ||
 		summary.ResourceSamples < 2 || summary.RawResourcesSHA256 == "" || summary.RawResourcesBytes < 1 ||
-		summary.CPUScope != "isolated gateway qualification process (gateway plus bounded fixture/client driver)" {
+		summary.CPUScope != qualificationGatewayCPUScope || summary.ClockOverhead <= 0 || summary.ClockMethod == "" {
 		t.Fatalf("processing summary = %#v", summary)
 	}
 	file, err := os.Open(rawPath)
@@ -126,11 +128,66 @@ func TestQualificationShortProcessingWritesRawArtifact(t *testing.T) {
 	if err := reader.Close(); err != nil {
 		t.Fatal(err)
 	}
-	if !strings.HasPrefix(string(raw), "elapsed_ns,processing_ns\n") || strings.Count(string(raw), "\n") != int(summary.Count)+1 {
+	if !strings.HasPrefix(string(raw), "elapsed_ns,queue_ns,processing_ns,pacing_ns\n") ||
+		strings.Count(string(raw), "\n") != int(summary.Count)+1 {
 		t.Fatalf("raw sample rows do not match summary count: %q", raw)
 	}
 }
 
+func TestQualificationShortProcessingSubprocessCoversFixedProfiles(t *testing.T) {
+	for _, profile := range qualificationMediaProfiles() {
+		profile.Duration = time.Second
+		profile.Warmup = 10 * time.Millisecond
+		summary, err := runQualificationProcessing(t, profile, filepath.Join(t.TempDir(), profile.Name+".csv.gz"))
+		if err != nil {
+			t.Fatalf("%s: %v", profile.Name, err)
+		}
+		if summary.Count < 1 || summary.CPUScope != qualificationGatewayCPUScope ||
+			summary.ClockOverhead <= 0 || summary.ClockMethod != qualificationClockOverheadMethod ||
+			summary.ObservedBitrateKbps < float64(profile.BitrateKbps)*0.95 ||
+			summary.ObservedBitrateKbps > float64(profile.BitrateKbps)*1.05 {
+			t.Fatalf("%s subprocess summary = %#v", profile.Name, summary)
+		}
+	}
+}
+
+func TestQualificationProcessingResourcesExcludeParentDriverCPU(t *testing.T) {
+	profile := qualificationMediaProfile{
+		Name: "resource-isolation", Codec: "h264", BitrateKbps: 1000,
+		Duration: 300 * time.Millisecond, Warmup: time.Millisecond, PacketBytes: 1000,
+	}
+	baseline, err := runQualificationProcessing(t, profile, filepath.Join(t.TempDir(), "baseline.csv.gz"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	stop := make(chan struct{})
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		var value uint64 = 1
+		for {
+			select {
+			case <-stop:
+				if value == 0 {
+					panic("bounded parent work was optimized away")
+				}
+				return
+			default:
+				value = value*2862933555777941757 + 3037000493
+			}
+		}
+	}()
+	busy, err := runQualificationProcessing(t, profile, filepath.Join(t.TempDir(), "busy.csv.gz"))
+	close(stop)
+	<-done
+	if err != nil {
+		t.Fatal(err)
+	}
+	if busy.CPUSeconds > baseline.CPUSeconds+50*time.Millisecond.Seconds() {
+		t.Fatalf("parent CPU leaked into gateway sample: baseline=%.6fs busy=%.6fs", baseline.CPUSeconds, busy.CPUSeconds)
+	}
+}
+
 func TestQualificationProcessingPreservesPayload(t *testing.T) {
 	profile := qualificationMediaProfiles()[0]
 	payload := qualificationPayload(profile)
@@ -169,6 +226,31 @@ func TestQualificationRepeatedTraversalTracksEveryProductionStage(t *testing.T)
 	}
 }
 
+func TestQualificationZeroLossBaselinesAreAttributedAtNormativeScale(t *testing.T) {
+	for _, media := range qualificationMediaProfiles() {
+		observation, err := runQualificationImpairment(t, qualificationImpairmentProfiles()[0], media,
+			qualificationImpairmentPacketCount, filepath.Join(t.TempDir(), media.Name+".csv.gz"))
+		if err != nil {
+			t.Fatalf("%s: %v", media.Name, err)
+		}
+		if observation.InjectedDropped != 0 || observation.Dropped != 0 ||
+			observation.ProviderFECDropped != 0 || observation.ProviderEnqueueDropped != 0 ||
+			observation.ProviderQueueReplaced != 0 || observation.GatewayDropped != 0 ||
+			observation.QUICSendDropped != 0 || observation.ClientDeliveryDropped != 0 ||
+			observation.UnexplainedDropped != 0 {
+			t.Fatalf("%s clean-path loss attribution = %#v", media.Name, observation)
+		}
+		if observation.SourceEmitted != qualificationImpairmentPacketCount ||
+			observation.ProviderRecovered != qualificationImpairmentPacketCount ||
+			observation.ProviderEnqueued != qualificationImpairmentPacketCount ||
+			observation.GatewayForwarded != qualificationImpairmentPacketCount ||
+			observation.QUICSent != qualificationImpairmentPacketCount ||
+			observation.Delivered != qualificationImpairmentPacketCount {
+			t.Fatalf("%s stage counts = %#v", media.Name, observation)
+		}
+	}
+}
+
 func TestQualificationImpairmentIsDeterministicAndBounded(t *testing.T) {
 	profile := qualificationImpairmentProfiles()[3]
 	first, err := runQualificationImpairment(t, profile, qualificationMediaProfiles()[0], 1000, filepath.Join(t.TempDir(), "first.csv.gz"))
@@ -237,81 +319,108 @@ func TestQualificationFixedSeedJitterIsObservableOnTraversedTraffic(t *testing.T
 	if observation.ObservedLatency < 5*time.Millisecond || observation.ObservedLatency > 100*time.Millisecond {
 		t.Fatalf("observed one-way latency %s does not reflect configured traversal", observation.ObservedLatency)
 	}
-	if observation.ObservedJitter < 5*time.Millisecond || observation.ObservedJitter > 80*time.Millisecond {
-		t.Fatalf("observed jitter %s is outside reviewed fixed-seed tolerance", observation.ObservedJitter)
+	if observation.AppliedJitter < 10*time.Millisecond || observation.AppliedJitter > 25*time.Millisecond {
+		t.Fatalf("applied fixed-seed jitter %s is outside the uniform-delay tolerance", observation.AppliedJitter)
 	}
-	if observation.ObservedOutOfOrder == 0 {
-		t.Fatal("fixed-seed jitter was serialized away before production traversal")
+	if observation.ObservedJitter <= 0 || observation.ObservedJitter > observation.AppliedJitter {
+		t.Fatalf("ordered traversal jitter %s is not bounded by applied jitter %s", observation.ObservedJitter, observation.AppliedJitter)
+	}
+	if observation.InjectedReordered != 0 || observation.ObservedOutOfOrder != 0 {
+		t.Fatalf("reorder-off jitter changed source order: %#v", observation)
 	}
 }
 
-func TestQualificationCPUTracksConsumedWorkNotIdleCapacity(t *testing.T) {
-	started := time.Now()
-	before := qualificationRuntimeSample(started)
-	time.Sleep(100 * time.Millisecond)
-	idle := qualificationRuntimeSample(started).CPUSeconds - before.CPUSeconds
-	if idle > 50*time.Millisecond.Seconds() {
-		t.Fatalf("idle CPU consumption = %.6fs, want at most 0.05s", idle)
-	}
-
-	workBefore := qualificationRuntimeSample(started).CPUSeconds
-	deadline := time.Now().Add(75 * time.Millisecond)
-	var value uint64 = 1
-	for time.Now().Before(deadline) {
-		value = value*6364136223846793005 + 1
-	}
-	if value == 0 {
-		t.Fatal("bounded CPU work was optimized away")
-	}
-	work := qualificationRuntimeSample(started).CPUSeconds - workBefore
-	if work <= idle || work < 20*time.Millisecond.Seconds() {
-		t.Fatalf("bounded work CPU = %.6fs, idle = %.6fs", work, idle)
-	}
-}
-
-func TestQualificationCPUIsolationExcludesParentTestWork(t *testing.T) {
-	output := filepath.Join(t.TempDir(), "cpu.txt")
-	command := exec.Command(os.Args[0], "-test.run=^TestQualificationCPUChild$", "-test.count=1")
-	command.Env = append(os.Environ(), "VERSEVDI_QUALIFICATION_CPU_CHILD="+output)
-	if err := command.Start(); err != nil {
-		t.Fatal(err)
-	}
-	deadline := time.Now().Add(150 * time.Millisecond)
-	var value uint64 = 1
-	for time.Now().Before(deadline) {
-		value = value*2862933555777941757 + 3037000493
-	}
-	if value == 0 {
-		t.Fatal("parent CPU work was optimized away")
-	}
-	if err := command.Wait(); err != nil {
-		t.Fatalf("CPU child: %v", err)
-	}
-	raw, err := os.ReadFile(output)
+func TestQualificationLossOnlyDoesNotImplicitlyReorder(t *testing.T) {
+	observation, err := runQualificationImpairment(
+		t,
+		qualificationImpairmentProfiles()[3],
+		qualificationMediaProfiles()[0],
+		400,
+		filepath.Join(t.TempDir(), "loss.csv.gz"),
+	)
 	if err != nil {
 		t.Fatal(err)
 	}
-	consumed, err := strconv.ParseFloat(string(raw), 64)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if consumed > 50*time.Millisecond.Seconds() {
-		t.Fatalf("isolated idle qualification process consumed %.6fs while parent test was busy", consumed)
+	if observation.InjectedReordered != 0 || observation.ObservedOutOfOrder != 0 {
+		t.Fatalf("loss-only profile changed source order: %#v", observation)
 	}
 }
 
-func TestQualificationCPUChild(t *testing.T) {
-	output := os.Getenv("VERSEVDI_QUALIFICATION_CPU_CHILD")
-	if output == "" {
-		return
-	}
-	started := time.Now()
-	before := qualificationRuntimeSample(started)
-	time.Sleep(200 * time.Millisecond)
-	consumed := qualificationRuntimeSample(started).CPUSeconds - before.CPUSeconds
-	if err := os.WriteFile(output, []byte(strconv.FormatFloat(consumed, 'f', 9, 64)), 0o600); err != nil {
+func TestQualificationExplicitReorderIsBoundedAndAttributed(t *testing.T) {
+	observation, err := runQualificationImpairment(
+		t,
+		qualificationImpairmentProfiles()[4],
+		qualificationMediaProfiles()[0],
+		400,
+		filepath.Join(t.TempDir(), "reorder.csv.gz"),
+	)
+	if err != nil {
 		t.Fatal(err)
 	}
+	if observation.InjectedReordered == 0 || observation.ObservedOutOfOrder != observation.InjectedReordered {
+		t.Fatalf("explicit reorder attribution = %#v", observation)
+	}
+}
+
+func TestQualificationGatewaySubprocessResourcesResetAndTrackWork(t *testing.T) {
+	profile := qualificationMediaProfile{
+		Name: "resource-process", Codec: "h264", BitrateKbps: 100000,
+		Duration: time.Second, Warmup: time.Millisecond, PacketBytes: 1000,
+	}
+	path := newQualificationProcessingPath(t, profile, qualificationMediaPacerKbps(profile, 0))
+	defer path.Close()
+	output := t.TempDir()
+	record := func(name string, work func() error) qualificationProcessRecordResult {
+		t.Helper()
+		if err := path.process.startRecording(
+			filepath.Join(output, name+".csv.gz"),
+			filepath.Join(output, name+"-resources.csv.gz"),
+		); err != nil {
+			t.Fatal(err)
+		}
+		if err := work(); err != nil {
+			t.Fatal(err)
+		}
+		result, err := path.process.stopRecording()
+		if err != nil {
+			t.Fatal(err)
+		}
+		return result
+	}
+	idle := record("idle", func() error {
+		time.Sleep(150 * time.Millisecond)
+		return nil
+	})
+	payload := qualificationPayload(profile)
+	work := record("work", func() error {
+		for index := 0; index < 500; index++ {
+			if _, err := path.emit(t, payload); err != nil {
+				return err
+			}
+			recovered, err := path.receivePayload(context.Background())
+			if err != nil || !bytes.Equal(recovered, payload) {
+				return errors.New("gateway subprocess work payload mismatch")
+			}
+		}
+		return nil
+	})
+	secondIdle := record("idle-again", func() error {
+		time.Sleep(150 * time.Millisecond)
+		return nil
+	})
+	if idle.CPUSeconds > 50*time.Millisecond.Seconds() || secondIdle.CPUSeconds > 50*time.Millisecond.Seconds() {
+		t.Fatalf("idle gateway CPU was reported as consumed work: first=%.6fs second=%.6fs", idle.CPUSeconds, secondIdle.CPUSeconds)
+	}
+	if work.CPUSeconds <= idle.CPUSeconds || work.Count != 500 || work.Mallocs == 0 ||
+		work.AllocatedBytes == 0 || work.PeakHeapBytes == 0 || work.PeakGoroutines == 0 {
+		t.Fatalf("gateway work resource sample = %#v idle=%#v", work, idle)
+	}
+	if secondIdle.Mallocs >= work.Mallocs || secondIdle.AllocatedBytes >= work.AllocatedBytes {
+		t.Fatalf("successive recording inherited counters: work=%#v second=%#v", work, secondIdle)
+	}
+	if work.ClockOverhead <= 0 || work.ClockMethod != qualificationClockOverheadMethod {
+		t.Fatalf("clock overhead evidence = %#v", work)
+	}
 }
 
 func qualificationRawMeanLatency(t *testing.T, path string) time.Duration {

gateway/qualification_harness_test.go

@@ -38,12 +38,14 @@ import (
 )
 
 const (
-	qualificationToolVersion                   = "versevdi-gateway-qualification/v4"
+	qualificationToolVersion                   = "versevdi-gateway-qualification/v5"
 	qualificationImpairmentQueuePackets        = 64
 	qualificationImpairmentMaxPackets          = 100_000
 	qualificationImpairmentPacketCount         = 10_000
 	qualificationProcessingLimit               = 5 * time.Millisecond
 	qualificationImpairmentSeed         uint64 = 0x3c6a11ce
+	qualificationClockOverheadMethod           = "median of 1000 batches of 100 monotonic time reads"
+	qualificationGatewayCPUScope               = "isolated gateway subprocess; bounded recorder/control included, fixture and client driver excluded"
 )
 
 type qualificationMediaProfile struct {
@@ -74,10 +76,12 @@ type qualificationPath struct {
 	session   *nativeApolloSession
 	fixture   *qualificationApolloFixture
 	backend   *qualificationTracingBackend
+	process   *qualificationGatewayProcess
 	key       []byte
 	flow      string
 	frame     uint32
 	bootTrace atomic.Bool
+	sourceUDP atomic.Uint64
 	closeOnce sync.Once
 	shutdown  func()
 }
@@ -109,6 +113,7 @@ type qualificationProcessingSummary struct {
 	Mean                  time.Duration  `json:"mean_ns"`
 	StandardDeviation     time.Duration  `json:"standard_deviation_ns"`
 	ClockOverhead         time.Duration  `json:"clock_overhead_ns"`
+	ClockMethod           string         `json:"clock_overhead_method"`
 	Histogram             map[string]int `json:"histogram"`
 	PayloadSHA256         string         `json:"payload_sha256"`
 	RawSamples            string         `json:"raw_samples"`
@@ -135,6 +140,20 @@ type qualificationImpairmentObservation struct {
 	Dropped                  int                         `json:"dropped"`
 	InjectedDropped          int                         `json:"injected_dropped"`
 	InjectedReordered        int                         `json:"injected_reordered"`
+	SourceEmitted            int                         `json:"source_emitted"`
+	SourceDatagrams          uint64                      `json:"source_datagrams"`
+	ProviderIngressDatagrams uint64                      `json:"provider_ingress_datagrams"`
+	ProviderRecovered        int                         `json:"provider_recovered"`
+	ProviderFECDropped       int                         `json:"provider_fec_dropped"`
+	ProviderEnqueued         int                         `json:"provider_enqueued"`
+	ProviderEnqueueDropped   int                         `json:"provider_enqueue_dropped"`
+	ProviderQueueReplaced    int                         `json:"provider_queue_replaced"`
+	GatewayForwarded         int                         `json:"gateway_forwarded"`
+	GatewayDropped           int                         `json:"gateway_dropped"`
+	QUICSent                 int                         `json:"quic_sent"`
+	QUICSendDropped          int                         `json:"quic_send_dropped"`
+	ClientDeliveryDropped    int                         `json:"client_delivery_dropped"`
+	UnexplainedDropped       int                         `json:"unexplained_dropped"`
 	ObservedOutOfOrder       int                         `json:"observed_out_of_order"`
 	ObservedLatency          time.Duration               `json:"observed_one_way_latency_ns"`
 	ObservedRTT              time.Duration               `json:"observed_rtt_ns"`
@@ -146,6 +165,7 @@ type qualificationImpairmentObservation struct {
 	MaxQueuePackets          int                         `json:"max_queue_packets"`
 	ConfiguredRTT            time.Duration               `json:"configured_rtt_ns"`
 	ConfiguredJitter         time.Duration               `json:"configured_jitter_ns"`
+	AppliedJitter            time.Duration               `json:"applied_jitter_ns"`
 	ConfiguredLossPercent    float64                     `json:"configured_loss_percent"`
 	ConfiguredReorder        bool                        `json:"configured_reorder"`
 	ConfiguredCapacitySteps  []int                       `json:"configured_capacity_steps_percent"`
@@ -255,6 +275,11 @@ func qualificationPayload(profile qualificationMediaProfile) []byte {
 	return payload
 }
 
+func qualificationMediaPacerKbps(profile qualificationMediaProfile, reduction int) int64 {
+	payloadKbps := profile.BitrateKbps * int64(100-reduction) / 100
+	return (payloadKbps*int64(profile.PacketBytes+frameHeaderSize) + int64(profile.PacketBytes) - 1) / int64(profile.PacketBytes)
+}
+
 type qualificationTracingBackend struct {
 	native   *NativeApolloBackend
 	setups   atomic.Uint64
@@ -867,6 +892,46 @@ func newQualificationPathWithImpairment(t *testing.T, profile qualificationMedia
 	return path
 }
 
+func newQualificationProcessingPath(t *testing.T, profile qualificationMediaProfile, pacerKbps int64) *qualificationPath {
+	t.Helper()
+	serverTLS, clientTLS := testTLS(t)
+	fixture := newQualificationApolloFixture(t, serverTLS, clientTLS, "qualification-session", profile)
+	authority := protocol.SessionAuthority{
+		Version: "1", SessionID: "qualification-session", GatewayID: "gateway-1",
+		Audience: "versevdi-gateway", ReconnectSequence: 0,
+		ExpiresAt:    time.Now().Add(45 * time.Minute).UTC().Format(time.RFC3339Nano),
+		Capabilities: DefaultCapabilities(), ProviderProfile: ProviderProfileApollo,
+		ProviderIdentity: fixture.work.ProviderIdentity,
+	}
+	work := fixture.work
+	work.ExpiresAt = authority.ExpiresAt
+	process := startQualificationGatewayProcess(t, serverTLS, authority, work, pacerKbps)
+	request := protocol.TunnelAdmissionRequest{
+		Version: "1", SessionID: authority.SessionID, GatewayID: authority.GatewayID,
+		Audience: authority.Audience, Grant: strings.Repeat("g", 64),
+		ClientNonce: "nonce-qualification", DeviceSignature: strings.Repeat("s", 86),
+		Capabilities: DefaultCapabilities(),
+	}
+	client, err := Dial(context.Background(), process.ready.GatewayAddress, clientTLS, request)
+	if err != nil {
+		process.Close()
+		t.Fatal(err)
+	}
+	key, err := fixture.streamKey()
+	if err != nil {
+		_ = client.Close()
+		process.Close()
+		t.Fatal(err)
+	}
+	path := &qualificationPath{client: client, fixture: fixture, process: process, key: key}
+	path.shutdown = func() {
+		_ = client.Close()
+		process.Close()
+	}
+	t.Cleanup(path.Close)
+	return path
+}
+
 func (p *qualificationPath) Close() {
 	if p != nil {
 		p.closeOnce.Do(p.shutdown)
@@ -913,7 +978,7 @@ func (p *qualificationPath) traverse(t *testing.T, payload []byte) (qualificatio
 
 func (p *qualificationPath) emit(t *testing.T, payload []byte) (qualificationPathTrace, error) {
 	t.Helper()
-	if p == nil || p.session == nil || len(payload) == 0 || len(payload) > 2*apolloVideoShardPayloadSize-8 {
+	if p == nil || p.fixture == nil || len(payload) == 0 || len(payload) > 2*apolloVideoShardPayloadSize-8 {
 		return qualificationPathTrace{}, ErrProviderMalformed
 	}
 	p.frame++
@@ -923,6 +988,7 @@ func (p *qualificationPath) emit(t *testing.T, payload []byte) (qualificationPat
 	if err := p.fixture.sendVideo(ctx, packets); err != nil {
 		return qualificationPathTrace{}, err
 	}
+	p.sourceUDP.Add(uint64(len(packets)))
 	return qualificationPathTrace{}, nil
 }
 
@@ -1140,7 +1206,7 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 		ConfiguredCapacitySteps: append([]int(nil), profile.CapacitySteps...),
 		RTTSource:               "apollo_enet_acknowledge",
 	}
-	path := newQualificationImpairedPath(t, media, media.BitrateKbps, profile)
+	path := newQualificationImpairedPath(t, media, qualificationMediaPacerKbps(media, 0), profile)
 	defer path.Close()
 	payload := qualificationPayload(media)
 	rawSamples := make([]rawSample, packetCount)
@@ -1166,31 +1232,36 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 		jobs = append(jobs, scheduledPacket{index: index, target: time.Duration(index)*spacing + delay})
 		rawSamples[index].outcome = "traversal_dropped"
 	}
+	var jitterMean, jitterM2 float64
+	var jitterSamples int
+	for _, packet := range jobs {
+		applied := float64(packet.target - time.Duration(packet.index)*spacing - profile.RTT/2)
+		jitterSamples++
+		delta := applied - jitterMean
+		jitterMean += delta / float64(jitterSamples)
+		jitterM2 += delta * (applied - jitterMean)
+	}
+	if jitterSamples > 0 {
+		observation.AppliedJitter = time.Duration(math.Sqrt(jitterM2 / float64(jitterSamples)))
+	}
 	if profile.Reorder {
 		for index := 18; index+1 < packetCount; index += 20 {
 			first, second := jobByIndex[index], jobByIndex[index+1]
 			if first < 0 || second < 0 {
 				continue
 			}
-			earlier := min(jobs[first].target, jobs[second].target)
-			later := max(jobs[first].target, jobs[second].target)
-			jobs[second].target = earlier
-			jobs[first].target = later + time.Nanosecond
+			jobs[first], jobs[second] = jobs[second], jobs[first]
 			observation.InjectedReordered++
 		}
 	}
-	sort.SliceStable(jobs, func(first, second int) bool {
-		if jobs[first].target == jobs[second].target {
-			return jobs[first].index < jobs[second].index
-		}
-		return jobs[first].target < jobs[second].target
-	})
 
 	started := time.Now()
 	beforeMetrics := path.server.Metrics()
 	beforeIngress := path.session.mediaIngress.Load()
 	beforeRecovered := path.session.mediaRecovered.Load()
 	beforeEnqueued := path.session.mediaEnqueued.Load()
+	beforeProviderDrops := path.session.mediaDrops.Load()
+	beforeSourceUDP := path.sourceUDP.Load()
 	beforePacer := path.server.pacer.reservations.Load()
 	grace := max(2*profile.RTT+2*profile.Jitter, 2*time.Second)
 	lastTarget := time.Duration(packetCount) * spacing
@@ -1254,17 +1325,28 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 	}()
 
 	stepAt := make(map[int]time.Time, len(profile.CapacitySteps))
+	emitted := 0
+	var previousRelease time.Time
+	maxCatchup := spacing
 	for _, packet := range jobs {
-		if delay := time.Until(started.Add(packet.target)); delay > 0 {
+		release := started.Add(packet.target)
+		if minimum := previousRelease.Add(spacing); !previousRelease.IsZero() && release.Before(minimum) {
+			release = minimum
+		}
+		if lag := time.Since(release); lag > maxCatchup {
+			release = release.Add(lag - maxCatchup)
+		}
+		if delay := time.Until(release); delay > 0 {
 			time.Sleep(delay)
 		}
+		previousRelease = release
 		if len(profile.CapacitySteps) == 2 {
 			switch {
 			case packet.index >= packetCount*2/3 && stepAt[profile.CapacitySteps[1]].IsZero():
-				path.server.pacer.setKbps(media.BitrateKbps * int64(100-profile.CapacitySteps[1]) / 100)
+				path.server.pacer.setKbps(qualificationMediaPacerKbps(media, profile.CapacitySteps[1]))
 				stepAt[profile.CapacitySteps[1]] = time.Now()
 			case packet.index >= packetCount/3 && stepAt[profile.CapacitySteps[0]].IsZero():
-				path.server.pacer.setKbps(media.BitrateKbps * int64(100-profile.CapacitySteps[0]) / 100)
+				path.server.pacer.setKbps(qualificationMediaPacerKbps(media, profile.CapacitySteps[0]))
 				stepAt[profile.CapacitySteps[0]] = time.Now()
 			}
 		}
@@ -1274,6 +1356,7 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 			receiveCancel()
 			return qualificationImpairmentObservation{}, err
 		}
+		emitted++
 	}
 	received := <-receivedDone
 	receiveCancel()
@@ -1316,6 +1399,23 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 		completedAt = received.packets[len(received.packets)-1].deliveredAt
 	}
 	afterMetrics := path.server.Metrics()
+	observation.SourceEmitted = emitted
+	observation.SourceDatagrams = path.sourceUDP.Load() - beforeSourceUDP
+	observation.ProviderIngressDatagrams = path.session.mediaIngress.Load() - beforeIngress
+	observation.ProviderRecovered = int(path.session.mediaRecovered.Load() - beforeRecovered)
+	observation.ProviderEnqueued = int(path.session.mediaEnqueued.Load() - beforeEnqueued)
+	observation.ProviderQueueReplaced = int(path.session.mediaDrops.Load() - beforeProviderDrops)
+	observation.GatewayForwarded = int(afterMetrics.ProcessingSamples - beforeMetrics.ProcessingSamples)
+	observation.QUICSent = int(afterMetrics.MediaPackets - beforeMetrics.MediaPackets)
+	observation.ProviderFECDropped = max(observation.SourceEmitted-observation.ProviderRecovered, 0)
+	observation.ProviderEnqueueDropped = max(observation.ProviderRecovered-observation.ProviderEnqueued, 0)
+	observation.GatewayDropped = max(observation.ProviderEnqueued-observation.ProviderQueueReplaced-observation.GatewayForwarded, 0)
+	observation.QUICSendDropped = max(observation.GatewayForwarded-observation.QUICSent, 0)
+	observation.ClientDeliveryDropped = max(observation.QUICSent-observation.Delivered, 0)
+	observation.UnexplainedDropped = observation.Dropped - observation.InjectedDropped -
+		observation.ProviderFECDropped - observation.ProviderEnqueueDropped -
+		observation.ProviderQueueReplaced - observation.GatewayDropped -
+		observation.QUICSendDropped - observation.ClientDeliveryDropped
 	if observation.Delivered > 0 && (path.session.mediaIngress.Load() <= beforeIngress ||
 		path.session.mediaRecovered.Load() <= beforeRecovered ||
 		path.session.mediaEnqueued.Load() <= beforeEnqueued ||
@@ -1391,6 +1491,7 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 		}
 	}
 	if observation.Delivered+observation.Dropped != observation.Sent ||
+		observation.UnexplainedDropped != 0 ||
 		observation.MaxQueuePackets > qualificationImpairmentQueuePackets {
 		return qualificationImpairmentObservation{}, errors.New("qualification impairment accounting invalid")
 	}
@@ -1469,75 +1570,58 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
 	if len(payload) < 4 {
 		return qualificationProcessingSummary{}, errors.New("qualification payload too small")
 	}
-	pacerKbps := (profile.BitrateKbps*int64(profile.PacketBytes+frameHeaderSize) + int64(profile.PacketBytes) - 1) / int64(profile.PacketBytes)
-	path := newQualificationPath(t, profile, pacerKbps)
+	path := newQualificationProcessingPath(t, profile, qualificationMediaPacerKbps(profile, 0))
 	defer path.Close()
-	if err := runQualificationWarmup(t, path, profile, payload); err != nil {
+	if err := runQualificationProcessWarmup(t, path, profile, payload); err != nil {
 		return qualificationProcessingSummary{}, err
 	}
-	file, err := os.OpenFile(rawPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640)
+	resourcePath := strings.TrimSuffix(rawPath, ".csv.gz") + "-resources.csv.gz"
+	if err := path.process.startRecording(rawPath, resourcePath); err != nil {
+		return qualificationProcessingSummary{}, err
+	}
+	before, err := path.process.snapshot()
 	if err != nil {
 		return qualificationProcessingSummary{}, err
 	}
-	compressed, err := gzip.NewWriterLevel(file, gzip.BestSpeed)
-	if err != nil {
-		_ = file.Close()
-		return qualificationProcessingSummary{}, err
-	}
-	buffered := bufio.NewWriterSize(compressed, 1<<20)
-	closed := false
-	defer func() {
-		if !closed {
-			_ = buffered.Flush()
-			_ = compressed.Close()
-			_ = file.Close()
-		}
-	}()
-	if _, err := buffered.WriteString("elapsed_ns,processing_ns\n"); err != nil {
-		return qualificationProcessingSummary{}, err
-	}
 	bytesPerSecond := profile.BitrateKbps * 1000 / 8
 	targetBytes := bytesPerSecond * profile.Duration.Nanoseconds() / int64(time.Second)
 	targetPackets := (targetBytes + int64(profile.PacketBytes) - 1) / int64(profile.PacketBytes)
-	samples := make([]time.Duration, 0, int(targetPackets))
 	started := time.Now()
-	resources := []qualificationResourceSample{qualificationRuntimeSample(started)}
-	lastResourceSample := started
 	var processed int64
 	for processed < targetPackets || time.Since(started) < profile.Duration {
 		current := append([]byte(nil), payload...)
 		binary.BigEndian.PutUint32(current[len(current)-4:], uint32(processed))
-		trace, sample, traverseErr := path.traverse(t, current)
-		if traverseErr != nil {
-			return qualificationProcessingSummary{}, traverseErr
+		if _, err := path.emit(t, current); err != nil {
+			return qualificationProcessingSummary{}, err
 		}
-		if !trace.NativeUDPIngress || !trace.ApolloRecovered || !trace.ProductionQueue ||
-			!trace.ProductionMediaLoop || !trace.ProductionPacer || !trace.VerseQUIC ||
-			!trace.PublicClientDecode || !trace.PayloadPreserved {
-			return qualificationProcessingSummary{}, fmt.Errorf("qualification bypassed the production provider-to-client path: %#v", trace)
-		}
-		samples = append(samples, sample)
-		if _, writeErr := fmt.Fprintf(buffered, "%d,%d\n", time.Since(started).Nanoseconds(), sample.Nanoseconds()); writeErr != nil {
-			return qualificationProcessingSummary{}, writeErr
+		recovered, err := path.receivePayload(context.Background())
+		if err != nil || !bytes.Equal(recovered, current) {
+			return qualificationProcessingSummary{}, errors.New("qualification processing payload integrity failure")
 		}
 		processed++
-		if time.Since(lastResourceSample) >= time.Second {
-			resources = append(resources, qualificationRuntimeSample(started))
-			lastResourceSample = time.Now()
-		}
 	}
 	actualDuration := time.Since(started)
-	resources = append(resources, qualificationRuntimeSample(started))
-	if err := buffered.Flush(); err != nil {
+	record, err := path.process.stopRecording()
+	if err != nil {
 		return qualificationProcessingSummary{}, err
 	}
-	if err := compressed.Close(); err != nil {
+	after, err := path.process.snapshot()
+	if err != nil {
 		return qualificationProcessingSummary{}, err
 	}
-	if err := file.Close(); err != nil {
+	if after.NativeSetups != 1 || after.NativeOpens != 1 ||
+		after.MediaRecovered-before.MediaRecovered != uint64(processed) ||
+		after.MediaEnqueued-before.MediaEnqueued != uint64(processed) ||
+		after.MediaDrops != before.MediaDrops ||
+		after.Metrics.ProcessingSamples-before.Metrics.ProcessingSamples != uint64(processed) ||
+		after.PacerReservations <= before.PacerReservations ||
+		after.Metrics.MediaPackets-before.Metrics.MediaPackets != uint64(processed) {
+		return qualificationProcessingSummary{}, fmt.Errorf("qualification subprocess bypassed a production stage: before=%#v after=%#v processed=%d", before, after, processed)
+	}
+	samples, err := readQualificationProcessingSamples(rawPath, record.Count)
+	if err != nil {
 		return qualificationProcessingSummary{}, err
 	}
-	closed = true
 	summary, err := summarizeQualificationSamples(samples)
 	if err != nil {
 		return qualificationProcessingSummary{}, err
@@ -1553,15 +1637,12 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
 	summary.Warmup = profile.Warmup
 	summary.ConfiguredDuration = profile.Duration
 	summary.ActualDuration = actualDuration
-	summary.ClockOverhead = qualificationClockOverhead()
+	summary.ClockOverhead = record.ClockOverhead
+	summary.ClockMethod = record.ClockMethod
 	summary.PayloadSHA256 = fmt.Sprintf("%x", sha256.Sum256(payload))
 	summary.RawSamples = filepath.Base(rawPath)
 	summary.RawSamplesSHA256 = sum
 	summary.RawSamplesBytes = size
-	resourcePath := strings.TrimSuffix(rawPath, ".csv.gz") + "-resources.csv.gz"
-	if err := writeQualificationResourceSamples(resourcePath, resources); err != nil {
-		return qualificationProcessingSummary{}, err
-	}
 	resourceSum, resourceSize, err := qualificationFileSHA256(resourcePath)
 	if err != nil {
 		return qualificationProcessingSummary{}, err
@@ -1569,23 +1650,16 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
 	summary.RawResources = filepath.Base(resourcePath)
 	summary.RawResourcesSHA256 = resourceSum
 	summary.RawResourcesBytes = resourceSize
-	summary.ResourceSamples = len(resources)
-	summary.CPUScope = "isolated gateway qualification process (gateway plus bounded fixture/client driver)"
-	firstResource, lastResource := resources[0], resources[len(resources)-1]
-	if firstResource.CPUSeconds < 0 || lastResource.CPUSeconds < 0 {
+	summary.ResourceSamples = record.ResourceSamples
+	summary.CPUScope = qualificationGatewayCPUScope
+	summary.CPUSeconds = record.CPUSeconds
+	summary.PeakHeapBytes = record.PeakHeapBytes
+	summary.PeakGoroutines = record.PeakGoroutines
+	summary.Mallocs = record.Mallocs
+	summary.AllocatedBytes = record.AllocatedBytes
+	if summary.CPUSeconds < 0 || summary.ClockOverhead <= 0 || summary.ClockMethod == "" {
 		return qualificationProcessingSummary{}, errors.New("process CPU usage unavailable")
 	}
-	summary.CPUSeconds = math.Max(0, lastResource.CPUSeconds-firstResource.CPUSeconds)
-	summary.Mallocs = lastResource.Mallocs - firstResource.Mallocs
-	summary.AllocatedBytes = lastResource.Allocated - firstResource.Allocated
-	for _, resource := range resources {
-		if resource.HeapBytes > summary.PeakHeapBytes {
-			summary.PeakHeapBytes = resource.HeapBytes
-		}
-		if resource.Goroutines > summary.PeakGoroutines {
-			summary.PeakGoroutines = resource.Goroutines
-		}
-	}
 	if actualDuration < profile.Duration || actualDuration > profile.Duration*105/100+250*time.Millisecond {
 		return qualificationProcessingSummary{}, fmt.Errorf("wall-clock duration %s outside [%s,%s]", actualDuration, profile.Duration, profile.Duration*105/100+250*time.Millisecond)
 	}
@@ -1598,6 +1672,57 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
 	return summary, nil
 }
 
+func readQualificationProcessingSamples(path string, expected int) ([]time.Duration, error) {
+	file, err := os.Open(path)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+	compressed, err := gzip.NewReader(file)
+	if err != nil {
+		return nil, err
+	}
+	defer compressed.Close()
+	scanner := bufio.NewScanner(compressed)
+	if !scanner.Scan() || scanner.Text() != "elapsed_ns,queue_ns,processing_ns,pacing_ns" {
+		return nil, errors.New("qualification processing header invalid")
+	}
+	samples := make([]time.Duration, 0, expected)
+	for scanner.Scan() {
+		fields := strings.Split(scanner.Text(), ",")
+		if len(fields) != 4 {
+			return nil, errors.New("qualification processing row invalid")
+		}
+		value, err := strconv.ParseInt(fields[2], 10, 64)
+		if err != nil || value < 0 {
+			return nil, errors.New("qualification processing sample invalid")
+		}
+		samples = append(samples, time.Duration(value))
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, err
+	}
+	if len(samples) != expected {
+		return nil, fmt.Errorf("qualification processing rows=%d want=%d", len(samples), expected)
+	}
+	return samples, nil
+}
+
+func runQualificationProcessWarmup(t *testing.T, path *qualificationPath, profile qualificationMediaProfile, payload []byte) error {
+	t.Helper()
+	started := time.Now()
+	for time.Since(started) < profile.Warmup {
+		if _, err := path.emit(t, payload); err != nil {
+			return err
+		}
+		recovered, err := path.receivePayload(context.Background())
+		if err != nil || !bytes.Equal(recovered, payload) {
+			return errors.New("qualification warmup payload integrity failure")
+		}
+	}
+	return nil
+}
+
 func runQualificationWarmup(t *testing.T, path *qualificationPath, profile qualificationMediaProfile, payload []byte) error {
 	t.Helper()
 	started := time.Now()
@@ -1657,11 +1782,17 @@ func writeQualificationResourceSamples(path string, samples []qualificationResou
 }
 
 func qualificationClockOverhead() time.Duration {
-	samples := make([]time.Duration, 10_000)
+	const readsPerBatch = 100
+	samples := make([]time.Duration, 1000)
+	var observed time.Time
 	for index := range samples {
 		started := time.Now()
-		samples[index] = time.Since(started)
+		for range readsPerBatch {
+			observed = time.Now()
+		}
+		samples[index] = time.Since(started) / readsPerBatch
 	}
+	runtime.KeepAlive(observed)
 	summary, _ := summarizeQualificationSamples(samples)
 	return summary.Median
 }
@@ -2000,10 +2131,10 @@ func TestSection7Qualification(t *testing.T) {
 		StartedAt: started.Format(time.RFC3339Nano), GoVersion: runtime.Version(),
 		ToolVersions: toolVersions,
 		OS:           runtime.GOOS, Architecture: runtime.GOARCH,
-		Topology:        "source-shaped encrypted Apollo fixture -> native recovery/FEC -> bounded provider queue -> production fair pacer -> Verse framing over mTLS/QUIC -> public Verse client decoder",
+		Topology:        "parent source-shaped encrypted Apollo fixture -> isolated gateway subprocess for processing/resource evidence -> public Verse client decoder; impairment uses the same native recovery/FEC, bounded queue, production pacer, framing, and QUIC path",
 		Direction:       "provider_to_client",
-		QueueDiscipline: "bounded 16-packet native provider queue, production equal-tier fair pacer, deterministic fixed-seed source impairment",
-		Evidence:        []string{"deterministic source-shaped Apollo recovery", "local real-time production path", "mTLS/QUIC fixture transport", "path impairment", "production fair pacer"},
+		QueueDiscipline: "ordered fixed-seed source delay queue with one-serialization-interval catch-up, bounded 16-packet native provider queue, production equal-tier fair pacer",
+		Evidence:        []string{"deterministic source-shaped Apollo recovery", "isolated gateway-process resources", "local real-time production path", "mTLS/QUIC fixture transport", "attributed path impairment", "production fair pacer"},
 		Deferred:        []string{"live Apollo", "macOS client", "physical firewall and packet route", "real encoder fidelity", "multi-host scale"},
 	}
 	for _, profile := range media {

gateway/qualification_process_test.go

@@ -0,0 +1,485 @@
+package gateway
+
+import (
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"context"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/hex"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+
+	protocol "git.sechmachine.io.vn/sechmachine/VerseVDI-Protocol/gen/go/protocol"
+)
+
+const qualificationProcessTokenHeader = "X-VerseVDI-Qualification-Token"
+
+type qualificationGatewayProcessConfig struct {
+	ServerCertificatePEM string
+	ServerPrivateKeyPEM  string
+	ClientCAPEM          string
+	Authority            protocol.SessionAuthority
+	Work                 protocol.ProviderSessionWork
+	PacerKbps            int64
+	ReadyPath            string
+	Token                string
+}
+
+type qualificationGatewayProcessReady struct {
+	GatewayAddress string
+	ControlAddress string
+}
+
+type qualificationGatewayProcessSnapshot struct {
+	Metrics           MetricsSnapshot
+	NativeSetups      uint64
+	NativeOpens       uint64
+	MediaIngress      uint64
+	MediaRecovered    uint64
+	MediaEnqueued     uint64
+	MediaDrops        uint64
+	MediaQueueMaximum uint64
+	PacerReservations uint64
+	ProviderTelemetry ProviderTelemetry
+}
+
+type qualificationProcessRecordRequest struct {
+	RawPath      string
+	ResourcePath string
+}
+
+type qualificationProcessRecordResult struct {
+	Count            int
+	ClockOverhead    time.Duration
+	ClockMethod      string
+	ResourceSamples  int
+	CPUSeconds       float64
+	PeakHeapBytes    uint64
+	PeakGoroutines   int
+	Mallocs          uint64
+	AllocatedBytes   uint64
+	RecordingElapsed time.Duration
+}
+
+type qualificationProcessRecorder struct {
+	mu            sync.Mutex
+	active        bool
+	started       time.Time
+	rawFile       *os.File
+	rawCompressed *gzip.Writer
+	rawBuffered   *bufio.Writer
+	resourcePath  string
+	resources     []qualificationResourceSample
+	samples       int
+	recordErr     error
+	tickerStop    chan struct{}
+	tickerDone    chan struct{}
+	clock         time.Duration
+}
+
+func (r *qualificationProcessRecorder) start(request qualificationProcessRecordRequest) error {
+	if err := validateQualificationOutputDir(filepath.Dir(request.RawPath)); err != nil {
+		return err
+	}
+	if filepath.Dir(request.RawPath) != filepath.Dir(request.ResourcePath) || request.RawPath == request.ResourcePath {
+		return errors.New("qualification process output paths invalid")
+	}
+	file, err := os.OpenFile(request.RawPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640)
+	if err != nil {
+		return err
+	}
+	compressed, err := gzip.NewWriterLevel(file, gzip.BestSpeed)
+	if err != nil {
+		_ = file.Close()
+		return err
+	}
+	buffered := bufio.NewWriterSize(compressed, 1<<20)
+	if _, err = buffered.WriteString("elapsed_ns,queue_ns,processing_ns,pacing_ns\n"); err != nil {
+		_ = compressed.Close()
+		_ = file.Close()
+		return err
+	}
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if r.active {
+		_ = buffered.Flush()
+		_ = compressed.Close()
+		_ = file.Close()
+		return errors.New("qualification process recording already active")
+	}
+	clock := qualificationClockOverhead()
+	r.active = true
+	r.started = time.Now()
+	r.rawFile = file
+	r.rawCompressed = compressed
+	r.rawBuffered = buffered
+	r.resourcePath = request.ResourcePath
+	r.resources = []qualificationResourceSample{qualificationRuntimeSample(r.started)}
+	r.samples = 0
+	r.recordErr = nil
+	r.clock = clock
+	r.tickerStop = make(chan struct{})
+	r.tickerDone = make(chan struct{})
+	go r.sampleResources()
+	return nil
+}
+
+func (r *qualificationProcessRecorder) observe(observation mediaTimingObservation) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	if !r.active || r.recordErr != nil {
+		return
+	}
+	_, r.recordErr = fmt.Fprintf(r.rawBuffered, "%d,%d,%d,%d\n",
+		time.Since(r.started).Nanoseconds(), observation.QueueDelay.Nanoseconds(),
+		observation.ProcessingDelay.Nanoseconds(), observation.PacingDelay.Nanoseconds())
+	r.samples++
+}
+
+func (r *qualificationProcessRecorder) sampleResources() {
+	ticker := time.NewTicker(time.Second)
+	defer ticker.Stop()
+	defer close(r.tickerDone)
+	for {
+		select {
+		case <-r.tickerStop:
+			return
+		case <-ticker.C:
+			r.mu.Lock()
+			if r.active {
+				r.resources = append(r.resources, qualificationRuntimeSample(r.started))
+			}
+			r.mu.Unlock()
+		}
+	}
+}
+
+func (r *qualificationProcessRecorder) stop() (qualificationProcessRecordResult, error) {
+	r.mu.Lock()
+	if !r.active {
+		r.mu.Unlock()
+		return qualificationProcessRecordResult{}, errors.New("qualification process recording is not active")
+	}
+	r.active = false
+	stop, done := r.tickerStop, r.tickerDone
+	r.mu.Unlock()
+	close(stop)
+	<-done
+
+	r.mu.Lock()
+	r.resources = append(r.resources, qualificationRuntimeSample(r.started))
+	elapsed := time.Since(r.started)
+	err := r.recordErr
+	if flushErr := r.rawBuffered.Flush(); err == nil {
+		err = flushErr
+	}
+	if closeErr := r.rawCompressed.Close(); err == nil {
+		err = closeErr
+	}
+	if closeErr := r.rawFile.Close(); err == nil {
+		err = closeErr
+	}
+	resources := append([]qualificationResourceSample(nil), r.resources...)
+	result := qualificationProcessRecordResult{
+		Count: r.samples, ClockOverhead: r.clock, ClockMethod: qualificationClockOverheadMethod,
+		ResourceSamples: len(resources), RecordingElapsed: elapsed,
+	}
+	resourcePath := r.resourcePath
+	r.mu.Unlock()
+	if err != nil {
+		return qualificationProcessRecordResult{}, err
+	}
+	if err := writeQualificationResourceSamples(resourcePath, resources); err != nil {
+		return qualificationProcessRecordResult{}, err
+	}
+	first, last := resources[0], resources[len(resources)-1]
+	result.CPUSeconds = max(last.CPUSeconds-first.CPUSeconds, 0)
+	result.Mallocs = last.Mallocs - first.Mallocs
+	result.AllocatedBytes = last.Allocated - first.Allocated
+	for _, sample := range resources {
+		result.PeakHeapBytes = max(result.PeakHeapBytes, sample.HeapBytes)
+		result.PeakGoroutines = max(result.PeakGoroutines, sample.Goroutines)
+	}
+	return result, nil
+}
+
+type qualificationGatewayProcess struct {
+	command *exec.Cmd
+	cancel  context.CancelFunc
+	done    chan error
+	output  *bytes.Buffer
+	ready   qualificationGatewayProcessReady
+	token   string
+	client  *http.Client
+	once    sync.Once
+}
+
+func startQualificationGatewayProcess(t *testing.T, serverTLS *tls.Config, authority protocol.SessionAuthority, work protocol.ProviderSessionWork, pacerKbps int64) *qualificationGatewayProcess {
+	t.Helper()
+	temp := t.TempDir()
+	configPath := filepath.Join(temp, "gateway-config.json")
+	readyPath := filepath.Join(temp, "gateway-ready.json")
+	tokenBytes := make([]byte, 32)
+	if _, err := rand.Read(tokenBytes); err != nil {
+		t.Fatal(err)
+	}
+	config := qualificationGatewayProcessConfig{
+		ServerCertificatePEM: qualificationCertificateChainPEM(t, serverTLS.Certificates[0]),
+		ServerPrivateKeyPEM:  privateKeyPEM(t, serverTLS.Certificates[0]),
+		ClientCAPEM:          string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverTLS.Certificates[0].Certificate[1]})),
+		Authority:            authority, Work: work, PacerKbps: pacerKbps, ReadyPath: readyPath,
+		Token: hex.EncodeToString(tokenBytes),
+	}
+	encoded, err := json.Marshal(config)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(configPath, encoded, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	command := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestQualificationGatewayProcessChild$", "-test.count=1")
+	command.Env = append(os.Environ(), "VERSEVDI_QUALIFICATION_GATEWAY_CONFIG="+configPath)
+	output := &bytes.Buffer{}
+	command.Stdout, command.Stderr = output, output
+	if err := command.Start(); err != nil {
+		cancel()
+		t.Fatal(err)
+	}
+	process := &qualificationGatewayProcess{
+		command: command, cancel: cancel, done: make(chan error, 1), output: output,
+		token: config.Token, client: &http.Client{Timeout: 5 * time.Second},
+	}
+	go func() { process.done <- command.Wait() }()
+	deadline := time.Now().Add(10 * time.Second)
+	for time.Now().Before(deadline) {
+		raw, readErr := os.ReadFile(readyPath)
+		if readErr == nil && json.Unmarshal(raw, &process.ready) == nil &&
+			process.ready.GatewayAddress != "" && process.ready.ControlAddress != "" {
+			return process
+		}
+		select {
+		case waitErr := <-process.done:
+			cancel()
+			t.Fatalf("qualification gateway child exited before ready: %v\n%s", waitErr, output)
+		case <-time.After(10 * time.Millisecond):
+		}
+	}
+	process.Close()
+	t.Fatalf("qualification gateway child did not become ready\n%s", output)
+	return nil
+}
+
+func qualificationCertificateChainPEM(t *testing.T, certificate tls.Certificate) string {
+	t.Helper()
+	var encoded strings.Builder
+	for _, der := range certificate.Certificate {
+		if err := pem.Encode(&encoded, &pem.Block{Type: "CERTIFICATE", Bytes: der}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	return encoded.String()
+}
+
+func (p *qualificationGatewayProcess) request(method, path string, body any, response any) error {
+	var reader io.Reader
+	if body != nil {
+		encoded, err := json.Marshal(body)
+		if err != nil {
+			return err
+		}
+		reader = bytes.NewReader(encoded)
+	}
+	request, err := http.NewRequest(method, "http://"+p.ready.ControlAddress+path, reader)
+	if err != nil {
+		return err
+	}
+	request.Header.Set(qualificationProcessTokenHeader, p.token)
+	result, err := p.client.Do(request)
+	if err != nil {
+		return err
+	}
+	defer result.Body.Close()
+	if result.StatusCode != http.StatusOK {
+		raw, _ := io.ReadAll(io.LimitReader(result.Body, 4096))
+		return fmt.Errorf("qualification gateway control %s: %s", result.Status, raw)
+	}
+	if response != nil {
+		return json.NewDecoder(io.LimitReader(result.Body, 1<<20)).Decode(response)
+	}
+	return nil
+}
+
+func (p *qualificationGatewayProcess) startRecording(rawPath, resourcePath string) error {
+	return p.request(http.MethodPost, "/record/start", qualificationProcessRecordRequest{RawPath: rawPath, ResourcePath: resourcePath}, nil)
+}
+
+func (p *qualificationGatewayProcess) stopRecording() (qualificationProcessRecordResult, error) {
+	var result qualificationProcessRecordResult
+	err := p.request(http.MethodPost, "/record/stop", nil, &result)
+	return result, err
+}
+
+func (p *qualificationGatewayProcess) snapshot() (qualificationGatewayProcessSnapshot, error) {
+	var result qualificationGatewayProcessSnapshot
+	err := p.request(http.MethodGet, "/snapshot", nil, &result)
+	return result, err
+}
+
+func (p *qualificationGatewayProcess) Close() {
+	if p == nil {
+		return
+	}
+	p.once.Do(func() {
+		_ = p.request(http.MethodPost, "/shutdown", nil, nil)
+		select {
+		case <-p.done:
+		case <-time.After(5 * time.Second):
+			p.cancel()
+			<-p.done
+		}
+		p.cancel()
+	})
+}
+
+func TestQualificationGatewayProcessChild(t *testing.T) {
+	configPath := os.Getenv("VERSEVDI_QUALIFICATION_GATEWAY_CONFIG")
+	if configPath == "" {
+		return
+	}
+	raw, err := os.ReadFile(configPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	var config qualificationGatewayProcessConfig
+	if err := json.Unmarshal(raw, &config); err != nil {
+		t.Fatal(err)
+	}
+	certificate, err := tls.X509KeyPair([]byte(config.ServerCertificatePEM), []byte(config.ServerPrivateKeyPEM))
+	if err != nil {
+		t.Fatal(err)
+	}
+	clientCAs := x509.NewCertPool()
+	if !clientCAs.AppendCertsFromPEM([]byte(config.ClientCAPEM)) {
+		t.Fatal("qualification gateway client CA invalid")
+	}
+	serverTLS := &tls.Config{
+		MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{certificate},
+		ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: clientCAs,
+	}
+	admission := &oneTimeAdmission{
+		authority: config.Authority, released: make(chan struct{}), disableClipboard: true,
+		providerWork: &config.Work,
+	}
+	backend := &qualificationTracingBackend{native: NewNativeApolloBackend()}
+	recorder := &qualificationProcessRecorder{}
+	server, err := NewServer(ServerConfig{
+		ListenAddress: "127.0.0.1:0", TLSConfig: serverTLS, GatewayID: config.Authority.GatewayID,
+		Capabilities: DefaultCapabilities(), ProviderCapabilities: DefaultCapabilities(),
+		Admission: admission, ProviderStateReporter: &recordingProviderStateReporter{},
+		Provider: NewApolloAdapter(backend, ProviderIdentity{}), PacerKbps: config.PacerKbps,
+		mediaObserver: recorder.observe,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	ctx, cancel := context.WithCancel(context.Background())
+	serveDone := make(chan error, 1)
+	go func() { serveDone <- server.Serve(ctx) }()
+	shutdown := make(chan struct{})
+	var shutdownOnce sync.Once
+	handler := http.NewServeMux()
+	authorized := func(response http.ResponseWriter, request *http.Request) bool {
+		if request.Header.Get(qualificationProcessTokenHeader) != config.Token {
+			http.Error(response, "unauthorized", http.StatusUnauthorized)
+			return false
+		}
+		return true
+	}
+	handler.HandleFunc("/snapshot", func(response http.ResponseWriter, request *http.Request) {
+		if request.Method != http.MethodGet || !authorized(response, request) {
+			return
+		}
+		snapshot := qualificationGatewayProcessSnapshot{
+			Metrics: server.Metrics(), NativeSetups: backend.setups.Load(), NativeOpens: backend.opens.Load(),
+			PacerReservations: server.pacer.reservations.Load(),
+		}
+		if session := backend.session(config.Authority.SessionID); session != nil {
+			snapshot.MediaIngress = session.mediaIngress.Load()
+			snapshot.MediaRecovered = session.mediaRecovered.Load()
+			snapshot.MediaEnqueued = session.mediaEnqueued.Load()
+			snapshot.MediaDrops = session.mediaDrops.Load()
+			snapshot.MediaQueueMaximum = session.mediaQueueMaximum.Load()
+			snapshot.ProviderTelemetry = session.Telemetry()
+		}
+		_ = json.NewEncoder(response).Encode(snapshot)
+	})
+	handler.HandleFunc("/record/start", func(response http.ResponseWriter, request *http.Request) {
+		if request.Method != http.MethodPost || !authorized(response, request) {
+			return
+		}
+		var recordRequest qualificationProcessRecordRequest
+		if err := json.NewDecoder(io.LimitReader(request.Body, 4096)).Decode(&recordRequest); err != nil {
+			http.Error(response, "invalid record request", http.StatusBadRequest)
+			return
+		}
+		if err := recorder.start(recordRequest); err != nil {
+			http.Error(response, err.Error(), http.StatusConflict)
+		}
+	})
+	handler.HandleFunc("/record/stop", func(response http.ResponseWriter, request *http.Request) {
+		if request.Method != http.MethodPost || !authorized(response, request) {
+			return
+		}
+		result, err := recorder.stop()
+		if err != nil {
+			http.Error(response, err.Error(), http.StatusConflict)
+			return
+		}
+		_ = json.NewEncoder(response).Encode(result)
+	})
+	handler.HandleFunc("/shutdown", func(response http.ResponseWriter, request *http.Request) {
+		if request.Method != http.MethodPost || !authorized(response, request) {
+			return
+		}
+		shutdownOnce.Do(func() { close(shutdown) })
+	})
+	control, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	controlServer := &http.Server{Handler: handler, ReadHeaderTimeout: time.Second}
+	go func() { _ = controlServer.Serve(control) }()
+	ready, err := json.Marshal(qualificationGatewayProcessReady{
+		GatewayAddress: server.Addr().String(), ControlAddress: control.Addr().String(),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(config.ReadyPath, ready, 0o600); err != nil {
+		t.Fatal(err)
+	}
+	<-shutdown
+	_, _ = recorder.stop()
+	cancel()
+	_ = server.Close()
+	_ = controlServer.Shutdown(context.Background())
+	if err := <-serveDone; err != nil {
+		t.Fatal(err)
+	}
+}

gateway/transport.go

@@ -77,6 +77,13 @@ type ServerConfig struct {
 	ProviderProfile        string
 	ProviderIdentity       string
 	PacerKbps              int64
+	mediaObserver          func(mediaTimingObservation)
+}
+
+type mediaTimingObservation struct {
+	QueueDelay      time.Duration
+	ProcessingDelay time.Duration
+	PacingDelay     time.Duration
 }
 
 type Server struct {
@@ -725,11 +732,17 @@ func (s *gatewaySession) sendMedia(channel byte, media ProviderMedia) error {
 		s.server.metrics.MediaPackets.Add(1)
 		s.server.metrics.MediaBytes.Add(uint64(len(encoded)))
 	}
-	processingDelay := media.EnqueuedAt.Sub(media.ReceivedAt) + time.Since(processingStarted) - pacingDelay
-	s.server.metrics.QueueDelayNanos.Add(uint64(dequeuedAt.Sub(media.EnqueuedAt)))
-	s.server.metrics.ProcessingDelayNanos.Add(uint64(max(processingDelay, 0)))
+	queueDelay := dequeuedAt.Sub(media.EnqueuedAt)
+	processingDelay := max(media.EnqueuedAt.Sub(media.ReceivedAt)+time.Since(processingStarted)-pacingDelay, 0)
+	s.server.metrics.QueueDelayNanos.Add(uint64(queueDelay))
+	s.server.metrics.ProcessingDelayNanos.Add(uint64(processingDelay))
 	s.server.metrics.PacingDelayNanos.Add(uint64(pacingDelay))
 	s.server.metrics.ProcessingSamples.Add(1)
+	if s.server.config.mediaObserver != nil {
+		s.server.config.mediaObserver(mediaTimingObservation{
+			QueueDelay: queueDelay, ProcessingDelay: processingDelay, PacingDelay: pacingDelay,
+		})
+	}
 	return nil
 }