fix(gateway): isolate qualification evidence

2026-07-30 15:06:23 +07:00
parent c0aabf2a72
commit b5aaff9e39
10 changed files with 1004 additions and 160 deletions
@@ -1,11 +1,13 @@
 package gateway

 import (
+	"bytes"
 	"compress/gzip"
+	"context"
 	"encoding/binary"
+	"errors"
 	"io"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"reflect"
 	"strconv"
@@ -107,7 +109,7 @@ func TestQualificationShortProcessingWritesRawArtifact(t *testing.T) {
 	}
 	if summary.Count < 1 || summary.RawSamplesSHA256 == "" || summary.RawSamplesBytes < 1 ||
 		summary.ResourceSamples < 2 || summary.RawResourcesSHA256 == "" || summary.RawResourcesBytes < 1 ||
-		summary.CPUScope != "isolated gateway qualification process (gateway plus bounded fixture/client driver)" {
+		summary.CPUScope != qualificationGatewayCPUScope || summary.ClockOverhead <= 0 || summary.ClockMethod == "" {
 		t.Fatalf("processing summary = %#v", summary)
 	}
 	file, err := os.Open(rawPath)
@@ -126,11 +128,66 @@ func TestQualificationShortProcessingWritesRawArtifact(t *testing.T) {
 	if err := reader.Close(); err != nil {
 		t.Fatal(err)
 	}
-	if !strings.HasPrefix(string(raw), "elapsed_ns,processing_ns\n") || strings.Count(string(raw), "\n") != int(summary.Count)+1 {
+	if !strings.HasPrefix(string(raw), "elapsed_ns,queue_ns,processing_ns,pacing_ns\n") ||
+		strings.Count(string(raw), "\n") != int(summary.Count)+1 {
 		t.Fatalf("raw sample rows do not match summary count: %q", raw)
 	}
 }

+func TestQualificationShortProcessingSubprocessCoversFixedProfiles(t *testing.T) {
+	for _, profile := range qualificationMediaProfiles() {
+		profile.Duration = time.Second
+		profile.Warmup = 10 * time.Millisecond
+		summary, err := runQualificationProcessing(t, profile, filepath.Join(t.TempDir(), profile.Name+".csv.gz"))
+		if err != nil {
+			t.Fatalf("%s: %v", profile.Name, err)
+		}
+		if summary.Count < 1 || summary.CPUScope != qualificationGatewayCPUScope ||
+			summary.ClockOverhead <= 0 || summary.ClockMethod != qualificationClockOverheadMethod ||
+			summary.ObservedBitrateKbps < float64(profile.BitrateKbps)*0.95 ||
+			summary.ObservedBitrateKbps > float64(profile.BitrateKbps)*1.05 {
+			t.Fatalf("%s subprocess summary = %#v", profile.Name, summary)
+		}
+	}
+}
+
+func TestQualificationProcessingResourcesExcludeParentDriverCPU(t *testing.T) {
+	profile := qualificationMediaProfile{
+		Name: "resource-isolation", Codec: "h264", BitrateKbps: 1000,
+		Duration: 300 * time.Millisecond, Warmup: time.Millisecond, PacketBytes: 1000,
+	}
+	baseline, err := runQualificationProcessing(t, profile, filepath.Join(t.TempDir(), "baseline.csv.gz"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	stop := make(chan struct{})
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		var value uint64 = 1
+		for {
+			select {
+			case <-stop:
+				if value == 0 {
+					panic("bounded parent work was optimized away")
+				}
+				return
+			default:
+				value = value*2862933555777941757 + 3037000493
+			}
+		}
+	}()
+	busy, err := runQualificationProcessing(t, profile, filepath.Join(t.TempDir(), "busy.csv.gz"))
+	close(stop)
+	<-done
+	if err != nil {
+		t.Fatal(err)
+	}
+	if busy.CPUSeconds > baseline.CPUSeconds+50*time.Millisecond.Seconds() {
+		t.Fatalf("parent CPU leaked into gateway sample: baseline=%.6fs busy=%.6fs", baseline.CPUSeconds, busy.CPUSeconds)
+	}
+}
+
 func TestQualificationProcessingPreservesPayload(t *testing.T) {
 	profile := qualificationMediaProfiles()[0]
 	payload := qualificationPayload(profile)
@@ -169,6 +226,31 @@ func TestQualificationRepeatedTraversalTracksEveryProductionStage(t *testing.T)
 	}
 }

+func TestQualificationZeroLossBaselinesAreAttributedAtNormativeScale(t *testing.T) {
+	for _, media := range qualificationMediaProfiles() {
+		observation, err := runQualificationImpairment(t, qualificationImpairmentProfiles()[0], media,
+			qualificationImpairmentPacketCount, filepath.Join(t.TempDir(), media.Name+".csv.gz"))
+		if err != nil {
+			t.Fatalf("%s: %v", media.Name, err)
+		}
+		if observation.InjectedDropped != 0 || observation.Dropped != 0 ||
+			observation.ProviderFECDropped != 0 || observation.ProviderEnqueueDropped != 0 ||
+			observation.ProviderQueueReplaced != 0 || observation.GatewayDropped != 0 ||
+			observation.QUICSendDropped != 0 || observation.ClientDeliveryDropped != 0 ||
+			observation.UnexplainedDropped != 0 {
+			t.Fatalf("%s clean-path loss attribution = %#v", media.Name, observation)
+		}
+		if observation.SourceEmitted != qualificationImpairmentPacketCount ||
+			observation.ProviderRecovered != qualificationImpairmentPacketCount ||
+			observation.ProviderEnqueued != qualificationImpairmentPacketCount ||
+			observation.GatewayForwarded != qualificationImpairmentPacketCount ||
+			observation.QUICSent != qualificationImpairmentPacketCount ||
+			observation.Delivered != qualificationImpairmentPacketCount {
+			t.Fatalf("%s stage counts = %#v", media.Name, observation)
+		}
+	}
+}
+
 func TestQualificationImpairmentIsDeterministicAndBounded(t *testing.T) {
 	profile := qualificationImpairmentProfiles()[3]
 	first, err := runQualificationImpairment(t, profile, qualificationMediaProfiles()[0], 1000, filepath.Join(t.TempDir(), "first.csv.gz"))
@@ -237,81 +319,108 @@ func TestQualificationFixedSeedJitterIsObservableOnTraversedTraffic(t *testing.T
 	if observation.ObservedLatency < 5*time.Millisecond || observation.ObservedLatency > 100*time.Millisecond {
 		t.Fatalf("observed one-way latency %s does not reflect configured traversal", observation.ObservedLatency)
 	}
-	if observation.ObservedJitter < 5*time.Millisecond || observation.ObservedJitter > 80*time.Millisecond {
-		t.Fatalf("observed jitter %s is outside reviewed fixed-seed tolerance", observation.ObservedJitter)
+	if observation.AppliedJitter < 10*time.Millisecond || observation.AppliedJitter > 25*time.Millisecond {
+		t.Fatalf("applied fixed-seed jitter %s is outside the uniform-delay tolerance", observation.AppliedJitter)
 	}
-	if observation.ObservedOutOfOrder == 0 {
-		t.Fatal("fixed-seed jitter was serialized away before production traversal")
+	if observation.ObservedJitter <= 0 || observation.ObservedJitter > observation.AppliedJitter {
+		t.Fatalf("ordered traversal jitter %s is not bounded by applied jitter %s", observation.ObservedJitter, observation.AppliedJitter)
+	}
+	if observation.InjectedReordered != 0 || observation.ObservedOutOfOrder != 0 {
+		t.Fatalf("reorder-off jitter changed source order: %#v", observation)
 	}
 }

-func TestQualificationCPUTracksConsumedWorkNotIdleCapacity(t *testing.T) {
-	started := time.Now()
-	before := qualificationRuntimeSample(started)
-	time.Sleep(100 * time.Millisecond)
-	idle := qualificationRuntimeSample(started).CPUSeconds - before.CPUSeconds
-	if idle > 50*time.Millisecond.Seconds() {
-		t.Fatalf("idle CPU consumption = %.6fs, want at most 0.05s", idle)
-	}
-
-	workBefore := qualificationRuntimeSample(started).CPUSeconds
-	deadline := time.Now().Add(75 * time.Millisecond)
-	var value uint64 = 1
-	for time.Now().Before(deadline) {
-		value = value*6364136223846793005 + 1
-	}
-	if value == 0 {
-		t.Fatal("bounded CPU work was optimized away")
-	}
-	work := qualificationRuntimeSample(started).CPUSeconds - workBefore
-	if work <= idle || work < 20*time.Millisecond.Seconds() {
-		t.Fatalf("bounded work CPU = %.6fs, idle = %.6fs", work, idle)
-	}
-}
-
-func TestQualificationCPUIsolationExcludesParentTestWork(t *testing.T) {
-	output := filepath.Join(t.TempDir(), "cpu.txt")
-	command := exec.Command(os.Args[0], "-test.run=^TestQualificationCPUChild$", "-test.count=1")
-	command.Env = append(os.Environ(), "VERSEVDI_QUALIFICATION_CPU_CHILD="+output)
-	if err := command.Start(); err != nil {
-		t.Fatal(err)
-	}
-	deadline := time.Now().Add(150 * time.Millisecond)
-	var value uint64 = 1
-	for time.Now().Before(deadline) {
-		value = value*2862933555777941757 + 3037000493
-	}
-	if value == 0 {
-		t.Fatal("parent CPU work was optimized away")
-	}
-	if err := command.Wait(); err != nil {
-		t.Fatalf("CPU child: %v", err)
-	}
-	raw, err := os.ReadFile(output)
+func TestQualificationLossOnlyDoesNotImplicitlyReorder(t *testing.T) {
+	observation, err := runQualificationImpairment(
+		t,
+		qualificationImpairmentProfiles()[3],
+		qualificationMediaProfiles()[0],
+		400,
+		filepath.Join(t.TempDir(), "loss.csv.gz"),
+	)
 	if err != nil {
 		t.Fatal(err)
 	}
-	consumed, err := strconv.ParseFloat(string(raw), 64)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if consumed > 50*time.Millisecond.Seconds() {
-		t.Fatalf("isolated idle qualification process consumed %.6fs while parent test was busy", consumed)
+	if observation.InjectedReordered != 0 || observation.ObservedOutOfOrder != 0 {
+		t.Fatalf("loss-only profile changed source order: %#v", observation)
 	}
 }

-func TestQualificationCPUChild(t *testing.T) {
-	output := os.Getenv("VERSEVDI_QUALIFICATION_CPU_CHILD")
-	if output == "" {
-		return
-	}
-	started := time.Now()
-	before := qualificationRuntimeSample(started)
-	time.Sleep(200 * time.Millisecond)
-	consumed := qualificationRuntimeSample(started).CPUSeconds - before.CPUSeconds
-	if err := os.WriteFile(output, []byte(strconv.FormatFloat(consumed, 'f', 9, 64)), 0o600); err != nil {
+func TestQualificationExplicitReorderIsBoundedAndAttributed(t *testing.T) {
+	observation, err := runQualificationImpairment(
+		t,
+		qualificationImpairmentProfiles()[4],
+		qualificationMediaProfiles()[0],
+		400,
+		filepath.Join(t.TempDir(), "reorder.csv.gz"),
+	)
+	if err != nil {
 		t.Fatal(err)
 	}
+	if observation.InjectedReordered == 0 || observation.ObservedOutOfOrder != observation.InjectedReordered {
+		t.Fatalf("explicit reorder attribution = %#v", observation)
+	}
+}
+
+func TestQualificationGatewaySubprocessResourcesResetAndTrackWork(t *testing.T) {
+	profile := qualificationMediaProfile{
+		Name: "resource-process", Codec: "h264", BitrateKbps: 100000,
+		Duration: time.Second, Warmup: time.Millisecond, PacketBytes: 1000,
+	}
+	path := newQualificationProcessingPath(t, profile, qualificationMediaPacerKbps(profile, 0))
+	defer path.Close()
+	output := t.TempDir()
+	record := func(name string, work func() error) qualificationProcessRecordResult {
+		t.Helper()
+		if err := path.process.startRecording(
+			filepath.Join(output, name+".csv.gz"),
+			filepath.Join(output, name+"-resources.csv.gz"),
+		); err != nil {
+			t.Fatal(err)
+		}
+		if err := work(); err != nil {
+			t.Fatal(err)
+		}
+		result, err := path.process.stopRecording()
+		if err != nil {
+			t.Fatal(err)
+		}
+		return result
+	}
+	idle := record("idle", func() error {
+		time.Sleep(150 * time.Millisecond)
+		return nil
+	})
+	payload := qualificationPayload(profile)
+	work := record("work", func() error {
+		for index := 0; index < 500; index++ {
+			if _, err := path.emit(t, payload); err != nil {
+				return err
+			}
+			recovered, err := path.receivePayload(context.Background())
+			if err != nil || !bytes.Equal(recovered, payload) {
+				return errors.New("gateway subprocess work payload mismatch")
+			}
+		}
+		return nil
+	})
+	secondIdle := record("idle-again", func() error {
+		time.Sleep(150 * time.Millisecond)
+		return nil
+	})
+	if idle.CPUSeconds > 50*time.Millisecond.Seconds() || secondIdle.CPUSeconds > 50*time.Millisecond.Seconds() {
+		t.Fatalf("idle gateway CPU was reported as consumed work: first=%.6fs second=%.6fs", idle.CPUSeconds, secondIdle.CPUSeconds)
+	}
+	if work.CPUSeconds <= idle.CPUSeconds || work.Count != 500 || work.Mallocs == 0 ||
+		work.AllocatedBytes == 0 || work.PeakHeapBytes == 0 || work.PeakGoroutines == 0 {
+		t.Fatalf("gateway work resource sample = %#v idle=%#v", work, idle)
+	}
+	if secondIdle.Mallocs >= work.Mallocs || secondIdle.AllocatedBytes >= work.AllocatedBytes {
+		t.Fatalf("successive recording inherited counters: work=%#v second=%#v", work, secondIdle)
+	}
+	if work.ClockOverhead <= 0 || work.ClockMethod != qualificationClockOverheadMethod {
+		t.Fatalf("clock overhead evidence = %#v", work)
+	}
 }

 func qualificationRawMeanLatency(t *testing.T, path string) time.Duration {