test(gateway): qualify production path artifacts

gateway/packaging_test.go

@@ -0,0 +1,66 @@
+package gateway
+
+import (
+	"bytes"
+	"debug/buildinfo"
+	"debug/elf"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+)
+
+func TestGatewayLinuxArtifactsAreReproduciblePureGoELF(t *testing.T) {
+	first, second := t.TempDir(), t.TempDir()
+	for _, output := range []string{first, second} {
+		command := exec.Command("make", "-C", "..", "gateway-linux", "DIST_DIR="+output)
+		command.Env = append(os.Environ(), "GOCACHE="+filepath.Join(t.TempDir(), "go-cache"))
+		if result, err := command.CombinedOutput(); err != nil {
+			t.Fatalf("gateway-linux: %v\n%s", err, result)
+		}
+	}
+
+	for _, architecture := range []struct {
+		name    string
+		machine elf.Machine
+	}{
+		{name: "amd64", machine: elf.EM_X86_64},
+		{name: "arm64", machine: elf.EM_AARCH64},
+	} {
+		firstPath := filepath.Join(first, "verse-gateway-linux-"+architecture.name)
+		secondPath := filepath.Join(second, "verse-gateway-linux-"+architecture.name)
+		firstBytes, err := os.ReadFile(firstPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+		secondBytes, err := os.ReadFile(secondPath)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if !bytes.Equal(firstBytes, secondBytes) {
+			t.Fatalf("linux/%s gateway build is not byte reproducible", architecture.name)
+		}
+		executable, err := elf.Open(firstPath)
+		if err != nil {
+			t.Fatalf("linux/%s ELF: %v", architecture.name, err)
+		}
+		if executable.FileHeader.Machine != architecture.machine {
+			_ = executable.Close()
+			t.Fatalf("linux/%s machine = %s", architecture.name, executable.FileHeader.Machine)
+		}
+		if err := executable.Close(); err != nil {
+			t.Fatal(err)
+		}
+		info, err := buildinfo.ReadFile(firstPath)
+		if err != nil {
+			t.Fatalf("linux/%s Go build info: %v", architecture.name, err)
+		}
+		settings := make(map[string]string, len(info.Settings))
+		for _, setting := range info.Settings {
+			settings[setting.Key] = setting.Value
+		}
+		if settings["GOOS"] != "linux" || settings["GOARCH"] != architecture.name || settings["CGO_ENABLED"] != "0" {
+			t.Fatalf("linux/%s build settings = %#v", architecture.name, settings)
+		}
+	}
+}

gateway/qualification_contract_test.go

@@ -5,8 +5,10 @@ import (
 	"encoding/binary"
 	"io"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -41,7 +43,7 @@ func TestQualificationCatalogMatchesSection7(t *testing.T) {
 }
 
 func TestQualificationProtocolVersionIsExplicitAndImmutable(t *testing.T) {
-	const version = "v1.0.0-phase3c-gateway-rc.6"
+	const version = "v1.0.0-phase3c-gateway-rc.8"
 	t.Setenv("VERSEVDI_QUALIFICATION_PROTOCOL_VERSION", version)
 	got, err := qualificationProtocolVersion()
 	if err != nil || got != version {
@@ -104,7 +106,8 @@ func TestQualificationShortProcessingWritesRawArtifact(t *testing.T) {
 		t.Fatal(err)
 	}
 	if summary.Count < 1 || summary.RawSamplesSHA256 == "" || summary.RawSamplesBytes < 1 ||
-		summary.ResourceSamples < 2 || summary.RawResourcesSHA256 == "" || summary.RawResourcesBytes < 1 {
+		summary.ResourceSamples < 2 || summary.RawResourcesSHA256 == "" || summary.RawResourcesBytes < 1 ||
+		summary.CPUScope != "isolated gateway qualification process (gateway plus bounded fixture/client driver)" {
 		t.Fatalf("processing summary = %#v", summary)
 	}
 	file, err := os.Open(rawPath)
@@ -194,6 +197,162 @@ func TestQualificationImpairmentIsDeterministicAndBounded(t *testing.T) {
 	}
 }
 
+func TestQualificationRTTIsNotSyntheticDoubleOneWayCompletion(t *testing.T) {
+	rawPath := filepath.Join(t.TempDir(), "latency.csv.gz")
+	observation, err := runQualificationImpairment(
+		t,
+		qualificationImpairmentProfiles()[1],
+		qualificationMediaProfiles()[0],
+		40,
+		rawPath,
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	meanLatency := qualificationRawMeanLatency(t, rawPath)
+	delta := observation.ObservedRTT - 2*meanLatency
+	if delta < 0 {
+		delta = -delta
+	}
+	if delta < 5*time.Millisecond {
+		t.Fatalf("RTT %s was synthesized as twice one-way completion %s", observation.ObservedRTT, meanLatency)
+	}
+}
+
+func TestQualificationFixedSeedJitterIsObservableOnTraversedTraffic(t *testing.T) {
+	profile := qualificationImpairmentProfiles()[2]
+	observation, err := runQualificationImpairment(
+		t,
+		profile,
+		qualificationMediaProfiles()[0],
+		200,
+		filepath.Join(t.TempDir(), "jitter.csv.gz"),
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if observation.RTTSource != "apollo_enet_acknowledge" || observation.ObservedRTT <= 0 {
+		t.Fatalf("RTT observation is not transport-acknowledged: %#v", observation)
+	}
+	if observation.ObservedLatency < 5*time.Millisecond || observation.ObservedLatency > 100*time.Millisecond {
+		t.Fatalf("observed one-way latency %s does not reflect configured traversal", observation.ObservedLatency)
+	}
+	if observation.ObservedJitter < 5*time.Millisecond || observation.ObservedJitter > 80*time.Millisecond {
+		t.Fatalf("observed jitter %s is outside reviewed fixed-seed tolerance", observation.ObservedJitter)
+	}
+	if observation.ObservedOutOfOrder == 0 {
+		t.Fatal("fixed-seed jitter was serialized away before production traversal")
+	}
+}
+
+func TestQualificationCPUTracksConsumedWorkNotIdleCapacity(t *testing.T) {
+	started := time.Now()
+	before := qualificationRuntimeSample(started)
+	time.Sleep(100 * time.Millisecond)
+	idle := qualificationRuntimeSample(started).CPUSeconds - before.CPUSeconds
+	if idle > 50*time.Millisecond.Seconds() {
+		t.Fatalf("idle CPU consumption = %.6fs, want at most 0.05s", idle)
+	}
+
+	workBefore := qualificationRuntimeSample(started).CPUSeconds
+	deadline := time.Now().Add(75 * time.Millisecond)
+	var value uint64 = 1
+	for time.Now().Before(deadline) {
+		value = value*6364136223846793005 + 1
+	}
+	if value == 0 {
+		t.Fatal("bounded CPU work was optimized away")
+	}
+	work := qualificationRuntimeSample(started).CPUSeconds - workBefore
+	if work <= idle || work < 20*time.Millisecond.Seconds() {
+		t.Fatalf("bounded work CPU = %.6fs, idle = %.6fs", work, idle)
+	}
+}
+
+func TestQualificationCPUIsolationExcludesParentTestWork(t *testing.T) {
+	output := filepath.Join(t.TempDir(), "cpu.txt")
+	command := exec.Command(os.Args[0], "-test.run=^TestQualificationCPUChild$", "-test.count=1")
+	command.Env = append(os.Environ(), "VERSEVDI_QUALIFICATION_CPU_CHILD="+output)
+	if err := command.Start(); err != nil {
+		t.Fatal(err)
+	}
+	deadline := time.Now().Add(150 * time.Millisecond)
+	var value uint64 = 1
+	for time.Now().Before(deadline) {
+		value = value*2862933555777941757 + 3037000493
+	}
+	if value == 0 {
+		t.Fatal("parent CPU work was optimized away")
+	}
+	if err := command.Wait(); err != nil {
+		t.Fatalf("CPU child: %v", err)
+	}
+	raw, err := os.ReadFile(output)
+	if err != nil {
+		t.Fatal(err)
+	}
+	consumed, err := strconv.ParseFloat(string(raw), 64)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if consumed > 50*time.Millisecond.Seconds() {
+		t.Fatalf("isolated idle qualification process consumed %.6fs while parent test was busy", consumed)
+	}
+}
+
+func TestQualificationCPUChild(t *testing.T) {
+	output := os.Getenv("VERSEVDI_QUALIFICATION_CPU_CHILD")
+	if output == "" {
+		return
+	}
+	started := time.Now()
+	before := qualificationRuntimeSample(started)
+	time.Sleep(200 * time.Millisecond)
+	consumed := qualificationRuntimeSample(started).CPUSeconds - before.CPUSeconds
+	if err := os.WriteFile(output, []byte(strconv.FormatFloat(consumed, 'f', 9, 64)), 0o600); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func qualificationRawMeanLatency(t *testing.T, path string) time.Duration {
+	t.Helper()
+	file, err := os.Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+	compressed, err := gzip.NewReader(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+	raw, err := io.ReadAll(compressed)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := compressed.Close(); err != nil {
+		t.Fatal(err)
+	}
+	var total time.Duration
+	var count int
+	for _, line := range strings.Split(string(raw), "\n")[1:] {
+		fields := strings.Split(line, ",")
+		if len(fields) < 5 || fields[4] != "delivered" {
+			continue
+		}
+		sent, sentErr := strconv.ParseInt(fields[1], 10, 64)
+		delivered, deliveredErr := strconv.ParseInt(fields[2], 10, 64)
+		if sentErr != nil || deliveredErr != nil || delivered < sent {
+			t.Fatalf("invalid raw latency row %q", line)
+		}
+		total += time.Duration(delivered - sent)
+		count++
+	}
+	if count == 0 {
+		t.Fatal("no delivered raw latency rows")
+	}
+	return total / time.Duration(count)
+}
+
 func TestQualificationSixImpairmentProfilesTraverseProductionPath(t *testing.T) {
 	profiles := qualificationImpairmentProfiles()
 	if len(profiles) != 6 {

gateway/qualification_harness_test.go

@@ -25,12 +25,12 @@ import (
 	"regexp"
 	"runtime"
 	"runtime/debug"
-	runtimemetrics "runtime/metrics"
 	"sort"
 	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
+	"syscall"
 	"testing"
 	"time"
 
@@ -38,7 +38,7 @@ import (
 )
 
 const (
-	qualificationToolVersion                   = "versevdi-gateway-qualification/v3"
+	qualificationToolVersion                   = "versevdi-gateway-qualification/v4"
 	qualificationImpairmentQueuePackets        = 64
 	qualificationImpairmentMaxPackets          = 100_000
 	qualificationImpairmentPacketCount         = 10_000
@@ -116,6 +116,7 @@ type qualificationProcessingSummary struct {
 	RawSamplesBytes       int64          `json:"raw_samples_bytes"`
 	ResourceSamples       int            `json:"resource_samples"`
 	CPUSeconds            float64        `json:"cpu_seconds"`
+	CPUScope              string         `json:"cpu_scope"`
 	PeakHeapBytes         uint64         `json:"peak_heap_bytes"`
 	PeakGoroutines        int            `json:"peak_goroutines"`
 	Mallocs               uint64         `json:"mallocs"`
@@ -132,9 +133,12 @@ type qualificationImpairmentObservation struct {
 	Sent                     int                         `json:"sent"`
 	Delivered                int                         `json:"delivered"`
 	Dropped                  int                         `json:"dropped"`
+	InjectedDropped          int                         `json:"injected_dropped"`
 	InjectedReordered        int                         `json:"injected_reordered"`
 	ObservedOutOfOrder       int                         `json:"observed_out_of_order"`
+	ObservedLatency          time.Duration               `json:"observed_one_way_latency_ns"`
 	ObservedRTT              time.Duration               `json:"observed_rtt_ns"`
+	RTTSource                string                      `json:"rtt_source"`
 	ObservedJitter           time.Duration               `json:"observed_jitter_ns"`
 	ObservedLossPercent      float64                     `json:"observed_loss_percent"`
 	ObservedReorderPercent   float64                     `json:"observed_reorder_percent"`
@@ -304,6 +308,11 @@ type qualificationApolloFixture struct {
 	closeOnce   sync.Once
 	sentPackets atomic.Uint64
 	work        protocol.ProviderSessionWork
+
+	controlImpairmentMu sync.Mutex
+	controlRTT          time.Duration
+	controlJitter       time.Duration
+	controlRandom       uint64
 }
 
 func newQualificationApolloFixture(t *testing.T, serverTLS, clientTLS *tls.Config, sessionID string, profile qualificationMediaProfile) *qualificationApolloFixture {
@@ -551,10 +560,7 @@ func (f *qualificationApolloFixture) serveControl() {
 		switch command {
 		case 1:
 		case apolloENetSendReliable, apolloENetPing, apolloENetDisconnect:
-			if _, err = f.control.WriteToUDP(sourceShapedENetAcknowledgePacket(7, 2, channel, sequence), remote); err != nil {
-				f.fail(err)
-				return
-			}
+			f.sendControlAcknowledge(sourceShapedENetAcknowledgePacket(7, 2, channel, sequence), remote)
 			if command == apolloENetDisconnect {
 				return
 			}
@@ -566,6 +572,45 @@ func (f *qualificationApolloFixture) serveControl() {
 	}
 }
 
+func (f *qualificationApolloFixture) setControlImpairment(profile qualificationImpairmentProfile) {
+	f.controlImpairmentMu.Lock()
+	f.controlRTT = profile.RTT
+	f.controlJitter = profile.Jitter
+	f.controlRandom = qualificationImpairmentSeed
+	f.controlImpairmentMu.Unlock()
+}
+
+func (f *qualificationApolloFixture) controlResponseDelay() time.Duration {
+	f.controlImpairmentMu.Lock()
+	defer f.controlImpairmentMu.Unlock()
+	delay := f.controlRTT
+	if f.controlJitter > 0 {
+		f.controlRandom ^= f.controlRandom << 13
+		f.controlRandom ^= f.controlRandom >> 7
+		f.controlRandom ^= f.controlRandom << 17
+		width := uint64(f.controlJitter*2 + 1)
+		delay += time.Duration(f.controlRandom%width) - f.controlJitter
+	}
+	return max(delay, 0)
+}
+
+func (f *qualificationApolloFixture) sendControlAcknowledge(packet []byte, remote *net.UDPAddr) {
+	delay := f.controlResponseDelay()
+	copyPacket := append([]byte(nil), packet...)
+	copyRemote := *remote
+	go func() {
+		timer := time.NewTimer(delay)
+		defer timer.Stop()
+		<-timer.C
+		if f.closed.Load() {
+			return
+		}
+		if _, err := f.control.WriteToUDP(copyPacket, &copyRemote); err != nil {
+			f.fail(err)
+		}
+	}()
+}
+
 func (f *qualificationApolloFixture) serveMedia(socket *net.UDPConn, video bool) {
 	buffer := make([]byte, apolloMediaMaximumPacket)
 	for {
@@ -751,9 +796,20 @@ func (f *qualificationFleet) Close() {
 }
 
 func newQualificationPath(t *testing.T, profile qualificationMediaProfile, pacerKbps int64) *qualificationPath {
+	return newQualificationPathWithImpairment(t, profile, pacerKbps, nil)
+}
+
+func newQualificationImpairedPath(t *testing.T, profile qualificationMediaProfile, pacerKbps int64, impairment qualificationImpairmentProfile) *qualificationPath {
+	return newQualificationPathWithImpairment(t, profile, pacerKbps, &impairment)
+}
+
+func newQualificationPathWithImpairment(t *testing.T, profile qualificationMediaProfile, pacerKbps int64, impairment *qualificationImpairmentProfile) *qualificationPath {
 	t.Helper()
 	serverTLS, clientTLS := testTLS(t)
 	fixture := newQualificationApolloFixture(t, serverTLS, clientTLS, "qualification-session", profile)
+	if impairment != nil {
+		fixture.setControlImpairment(*impairment)
+	}
 	backend := &qualificationTracingBackend{native: NewNativeApolloBackend()}
 	provider := NewApolloAdapter(backend, ProviderIdentity{})
 	authority := protocol.SessionAuthority{
@@ -1049,6 +1105,23 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 	if _, err := buffered.WriteString("source_sequence,sent_ns,delivered_ns,processing_ns,outcome,delivery_order,bytes,queue_packets\n"); err != nil {
 		return qualificationImpairmentObservation{}, err
 	}
+	type scheduledPacket struct {
+		index  int
+		target time.Duration
+	}
+	type rawSample struct {
+		sent, delivered time.Duration
+		processing      time.Duration
+		outcome         string
+		deliveryOrder   int
+		bytes           int
+		queuePackets    int
+	}
+	type receivedPacket struct {
+		index, deliveryOrder, queuePackets int
+		deliveredAt                        time.Time
+		processing                         time.Duration
+	}
 	state := qualificationImpairmentSeed
 	random := func() uint64 {
 		state ^= state << 13
@@ -1065,22 +1138,126 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 		Sent: packetCount, ConfiguredRTT: profile.RTT, ConfiguredJitter: profile.Jitter,
 		ConfiguredLossPercent: profile.LossPercent, ConfiguredReorder: profile.Reorder,
 		ConfiguredCapacitySteps: append([]int(nil), profile.CapacitySteps...),
+		RTTSource:               "apollo_enet_acknowledge",
 	}
-	path := newQualificationPath(t, media, media.BitrateKbps)
+	path := newQualificationImpairedPath(t, media, media.BitrateKbps, profile)
 	defer path.Close()
-	started := time.Now()
 	payload := qualificationPayload(media)
-	var deliveries []qualificationDeliverySample
-	var totalRTT, totalJitter, previousRTT time.Duration
-	previousDelivered := -1
-	deliveryOrder := 0
-	type pendingPacket struct {
-		index  int
-		jitter time.Duration
+	rawSamples := make([]rawSample, packetCount)
+	jobs := make([]scheduledPacket, 0, packetCount)
+	jobByIndex := make([]int, packetCount)
+	for index := range jobByIndex {
+		jobByIndex[index] = -1
 	}
-	pending := pendingPacket{index: -1}
+	for index := 0; index < packetCount; index++ {
+		jitter := time.Duration(0)
+		if profile.Jitter > 0 {
+			width := uint64(profile.Jitter*2 + 1)
+			jitter = time.Duration(random()%width) - profile.Jitter
+		}
+		rawSamples[index].sent = time.Duration(index) * spacing
+		if float64(random()%10_000) < profile.LossPercent*100 {
+			rawSamples[index].outcome = "injected_dropped"
+			observation.InjectedDropped++
+			continue
+		}
+		delay := max(profile.RTT/2+jitter, 0)
+		jobByIndex[index] = len(jobs)
+		jobs = append(jobs, scheduledPacket{index: index, target: time.Duration(index)*spacing + delay})
+		rawSamples[index].outcome = "traversal_dropped"
+	}
+	if profile.Reorder {
+		for index := 18; index+1 < packetCount; index += 20 {
+			first, second := jobByIndex[index], jobByIndex[index+1]
+			if first < 0 || second < 0 {
+				continue
+			}
+			earlier := min(jobs[first].target, jobs[second].target)
+			later := max(jobs[first].target, jobs[second].target)
+			jobs[second].target = earlier
+			jobs[first].target = later + time.Nanosecond
+			observation.InjectedReordered++
+		}
+	}
+	sort.SliceStable(jobs, func(first, second int) bool {
+		if jobs[first].target == jobs[second].target {
+			return jobs[first].index < jobs[second].index
+		}
+		return jobs[first].target < jobs[second].target
+	})
+
+	started := time.Now()
+	beforeMetrics := path.server.Metrics()
+	beforeIngress := path.session.mediaIngress.Load()
+	beforeRecovered := path.session.mediaRecovered.Load()
+	beforeEnqueued := path.session.mediaEnqueued.Load()
+	beforePacer := path.server.pacer.reservations.Load()
+	grace := max(2*profile.RTT+2*profile.Jitter, 2*time.Second)
+	lastTarget := time.Duration(packetCount) * spacing
+	if len(jobs) > 0 {
+		lastTarget = jobs[len(jobs)-1].target
+	}
+	receiveCtx, receiveCancel := context.WithDeadline(context.Background(), started.Add(lastTarget+grace))
+	defer receiveCancel()
+	receivedDone := make(chan struct {
+		packets []receivedPacket
+		err     error
+	}, 1)
+	go func() {
+		result := struct {
+			packets []receivedPacket
+			err     error
+		}{packets: make([]receivedPacket, 0, len(jobs))}
+		metrics := beforeMetrics
+		seen := make([]bool, packetCount)
+		for len(result.packets) < len(jobs) {
+			recovered, receiveErr := path.receivePayload(receiveCtx)
+			if receiveErr != nil {
+				if errors.Is(receiveErr, context.DeadlineExceeded) || errors.Is(receiveErr, context.Canceled) {
+					break
+				}
+				var timeout net.Error
+				if errors.As(receiveErr, &timeout) && timeout.Timeout() {
+					break
+				}
+				result.err = receiveErr
+				break
+			}
+			if len(recovered) != len(payload) {
+				result.err = errors.New("impaired payload length changed")
+				break
+			}
+			index := int(binary.BigEndian.Uint32(recovered[len(recovered)-4:]))
+			if index < 0 || index >= packetCount || jobByIndex[index] < 0 || seen[index] {
+				result.err = errors.New("impaired payload sequence invalid")
+				break
+			}
+			expected := append([]byte(nil), payload...)
+			binary.BigEndian.PutUint32(expected[len(expected)-4:], uint32(index))
+			if !bytes.Equal(recovered, expected) {
+				result.err = errors.New("impaired payload bytes changed")
+				break
+			}
+			seen[index] = true
+			current := path.server.Metrics()
+			processing := time.Duration(0)
+			if samples := current.ProcessingSamples - metrics.ProcessingSamples; samples > 0 {
+				processing = time.Duration((current.ProcessingDelayNanos - metrics.ProcessingDelayNanos) / samples)
+			}
+			metrics = current
+			result.packets = append(result.packets, receivedPacket{
+				index: index, deliveryOrder: len(result.packets) + 1,
+				deliveredAt: time.Now(), processing: processing, queuePackets: len(path.session.video),
+			})
+		}
+		receivedDone <- result
+	}()
+
 	stepAt := make(map[int]time.Time, len(profile.CapacitySteps))
-	deliver := func(packet pendingPacket) error {
+	for _, packet := range jobs {
+		if delay := time.Until(started.Add(packet.target)); delay > 0 {
+			time.Sleep(delay)
+		}
 		if len(profile.CapacitySteps) == 2 {
 			switch {
 			case packet.index >= packetCount*2/3 && stepAt[profile.CapacitySteps[1]].IsZero():
@@ -1091,79 +1268,67 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 				stepAt[profile.CapacitySteps[0]] = time.Now()
 			}
 		}
-		sentAt := started.Add(time.Duration(packet.index) * spacing)
-		target := sentAt.Add(profile.RTT/2 + packet.jitter)
-		if delay := time.Until(target); delay > 0 {
-			time.Sleep(delay)
-		}
 		current := append([]byte(nil), payload...)
 		binary.BigEndian.PutUint32(current[len(current)-4:], uint32(packet.index))
-		trace, processing, err := path.traverse(t, current)
-		if err != nil || !trace.PayloadPreserved || !trace.ProductionPacer {
-			if err == nil {
-				err = errors.New("impaired packet bypassed production gateway path")
-			}
-			return err
+		if _, err := path.emit(t, current); err != nil {
+			receiveCancel()
+			return qualificationImpairmentObservation{}, err
 		}
-		deliveredAt := time.Now()
-		rtt := 2 * deliveredAt.Sub(sentAt)
-		totalRTT += rtt
-		if previousRTT != 0 {
-			delta := rtt - previousRTT
+	}
+	received := <-receivedDone
+	receiveCancel()
+	if received.err != nil {
+		return qualificationImpairmentObservation{}, received.err
+	}
+
+	var deliveries []qualificationDeliverySample
+	var totalLatency, totalJitter, previousLatency time.Duration
+	previousDelivered := -1
+	for _, packet := range received.packets {
+		sample := &rawSamples[packet.index]
+		sample.delivered = packet.deliveredAt.Sub(started)
+		sample.processing = packet.processing
+		sample.outcome = "delivered"
+		sample.deliveryOrder = packet.deliveryOrder
+		sample.bytes = media.PacketBytes
+		sample.queuePackets = packet.queuePackets
+		latency := sample.delivered - sample.sent
+		totalLatency += latency
+		if previousLatency != 0 {
+			delta := latency - previousLatency
 			if delta < 0 {
 				delta = -delta
 			}
 			totalJitter += delta
 		}
-		previousRTT = rtt
+		previousLatency = latency
 		if previousDelivered >= 0 && packet.index < previousDelivered {
 			observation.ObservedOutOfOrder++
 		}
 		previousDelivered = packet.index
-		observation.Delivered++
-		deliveryOrder++
-		wireBytes := int64(len(current) + frameHeaderSize)
-		deliveries = append(deliveries, qualificationDeliverySample{At: deliveredAt, Bytes: wireBytes})
-		queuePackets := int(path.session.mediaQueueMaximum.Load())
-		if _, err := fmt.Fprintf(buffered, "%d,%d,%d,%d,delivered,%d,%d,%d\n", packet.index,
-			sentAt.Sub(started).Nanoseconds(), deliveredAt.Sub(started).Nanoseconds(),
-			processing.Nanoseconds(), deliveryOrder, len(current), queuePackets); err != nil {
-			return err
-		}
-		observation.MaxQueuePackets = max(observation.MaxQueuePackets, queuePackets)
-		return nil
+		deliveries = append(deliveries, qualificationDeliverySample{At: packet.deliveredAt, Bytes: int64(media.PacketBytes + frameHeaderSize)})
+		observation.MaxQueuePackets = max(observation.MaxQueuePackets, packet.queuePackets)
 	}
-	for index := 0; index < packetCount; index++ {
-		jitter := time.Duration(0)
-		if profile.Jitter > 0 {
-			width := uint64(profile.Jitter*2 + 1)
-			jitter = time.Duration(random()%width) - profile.Jitter
-		}
-		if float64(random()%10_000) < profile.LossPercent*100 {
-			observation.Dropped++
-			if _, err := fmt.Fprintf(buffered, "%d,%d,0,0,dropped,0,0,%d\n", index, time.Duration(index)*spacing, path.session.mediaQueueMaximum.Load()); err != nil {
-				return qualificationImpairmentObservation{}, err
-			}
-			continue
-		}
-		packet := pendingPacket{index: index, jitter: jitter}
-		if profile.Reorder && index%20 == 18 {
-			pending = packet
-			continue
-		}
-		if err := deliver(packet); err != nil {
-			return qualificationImpairmentObservation{}, err
-		}
-		if pending.index >= 0 {
-			if err := deliver(pending); err != nil {
-				return qualificationImpairmentObservation{}, err
-			}
-			pending.index = -1
-			observation.InjectedReordered++
-		}
+	observation.Delivered = len(received.packets)
+	observation.Dropped = packetCount - observation.Delivered
+	completedAt := started
+	if observation.Delivered > 0 {
+		completedAt = received.packets[len(received.packets)-1].deliveredAt
 	}
-	if pending.index >= 0 {
-		if err := deliver(pending); err != nil {
+	afterMetrics := path.server.Metrics()
+	if observation.Delivered > 0 && (path.session.mediaIngress.Load() <= beforeIngress ||
+		path.session.mediaRecovered.Load() <= beforeRecovered ||
+		path.session.mediaEnqueued.Load() <= beforeEnqueued ||
+		afterMetrics.ProcessingSamples <= beforeMetrics.ProcessingSamples ||
+		path.server.pacer.reservations.Load() <= beforePacer ||
+		afterMetrics.MediaPackets <= beforeMetrics.MediaPackets) {
+		return qualificationImpairmentObservation{}, errors.New("impaired traffic bypassed a production gateway stage")
+	}
+	observation.ObservedRTT = path.session.Telemetry().ControlRTT
+	for index, sample := range rawSamples {
+		if _, err := fmt.Fprintf(buffered, "%d,%d,%d,%d,%s,%d,%d,%d\n", index,
+			sample.sent.Nanoseconds(), sample.delivered.Nanoseconds(), sample.processing.Nanoseconds(),
+			sample.outcome, sample.deliveryOrder, sample.bytes, sample.queuePackets); err != nil {
 			return qualificationImpairmentObservation{}, err
 		}
 	}
@@ -1178,28 +1343,32 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 	}
 	closed = true
 	if observation.Delivered > 0 {
-		observation.ObservedRTT = totalRTT / time.Duration(observation.Delivered)
+		observation.ObservedLatency = totalLatency / time.Duration(observation.Delivered)
 		if observation.Delivered > 1 {
 			observation.ObservedJitter = totalJitter / time.Duration(observation.Delivered-1)
 		}
-		observation.ObservedThroughputKbps = float64(observation.Delivered*media.PacketBytes*8) / time.Since(started).Seconds() / 1000
+		observation.ObservedThroughputKbps = float64(observation.Delivered*media.PacketBytes*8) / completedAt.Sub(started).Seconds() / 1000
 	}
 	observation.ObservedLossPercent = float64(observation.Dropped) * 100 / float64(packetCount)
 	observation.ObservedReorderPercent = float64(observation.ObservedOutOfOrder) * 100 / float64(packetCount)
 	if packetCount >= qualificationImpairmentPacketCount {
 		capacityFactor := 1.0
 		if len(profile.CapacitySteps) > 0 {
-			inverseRates := 1.0
+			rates := 1.0
 			for _, reduction := range profile.CapacitySteps {
-				inverseRates += 100 / float64(100-reduction)
+				rates += float64(100-reduction) / 100
 			}
-			capacityFactor = float64(len(profile.CapacitySteps)+1) / inverseRates
+			capacityFactor = rates / float64(len(profile.CapacitySteps)+1)
 		}
 		expectedThroughput := float64(media.BitrateKbps) * (1 - profile.LossPercent/100) * capacityFactor
 		lowerThroughput := expectedThroughput * 0.90
 		upperThroughput := expectedThroughput * 1.05
 		if observation.ObservedThroughputKbps < lowerThroughput || observation.ObservedThroughputKbps > upperThroughput {
-			return qualificationImpairmentObservation{}, fmt.Errorf("observed throughput %.2f outside [%.2f,%.2f]", observation.ObservedThroughputKbps, lowerThroughput, upperThroughput)
+			return qualificationImpairmentObservation{}, fmt.Errorf(
+				"observed throughput %.2f outside [%.2f,%.2f] over %s with %d delivered/%d injected drops and steps %v",
+				observation.ObservedThroughputKbps, lowerThroughput, upperThroughput,
+				completedAt.Sub(started), observation.Delivered, observation.InjectedDropped, stepAt,
+			)
 		}
 	}
 	observation.RawSamples = filepath.Base(rawPath)
@@ -1401,7 +1570,11 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
 	summary.RawResourcesSHA256 = resourceSum
 	summary.RawResourcesBytes = resourceSize
 	summary.ResourceSamples = len(resources)
+	summary.CPUScope = "isolated gateway qualification process (gateway plus bounded fixture/client driver)"
 	firstResource, lastResource := resources[0], resources[len(resources)-1]
+	if firstResource.CPUSeconds < 0 || lastResource.CPUSeconds < 0 {
+		return qualificationProcessingSummary{}, errors.New("process CPU usage unavailable")
+	}
 	summary.CPUSeconds = math.Max(0, lastResource.CPUSeconds-firstResource.CPUSeconds)
 	summary.Mallocs = lastResource.Mallocs - firstResource.Mallocs
 	summary.AllocatedBytes = lastResource.Allocated - firstResource.Allocated
@@ -1441,12 +1614,16 @@ func runQualificationWarmup(t *testing.T, path *qualificationPath, profile quali
 }
 
 func qualificationRuntimeSample(started time.Time) qualificationResourceSample {
-	cpu := []runtimemetrics.Sample{{Name: "/cpu/classes/total:cpu-seconds"}}
-	runtimemetrics.Read(cpu)
+	var usage syscall.Rusage
+	cpuSeconds := -1.0
+	if syscall.Getrusage(syscall.RUSAGE_SELF, &usage) == nil {
+		cpuSeconds = float64(usage.Utime.Sec+usage.Stime.Sec) +
+			float64(usage.Utime.Usec+usage.Stime.Usec)/1_000_000
+	}
 	var memory runtime.MemStats
 	runtime.ReadMemStats(&memory)
 	return qualificationResourceSample{
-		Elapsed: time.Since(started), CPUSeconds: cpu[0].Value.Float64(),
+		Elapsed: time.Since(started), CPUSeconds: cpuSeconds,
 		HeapBytes: memory.HeapAlloc, Goroutines: runtime.NumGoroutine(),
 		Mallocs: memory.Mallocs, Allocated: memory.TotalAlloc,
 	}

gateway/telemetry.go

@@ -113,11 +113,6 @@ func (m *Metrics) observeProviderState(state string) {
 	}
 }
 
-type Pacer struct {
-	bytesPerSecond int64
-	last           time.Time
-}
-
 // fairPacer is the gateway's one shared, equal-tier media scheduler. Each
 // session can hold only its existing bounded provider media channel while it
 // waits for the next reservation, so a slow client cannot grow a global queue.
@@ -207,32 +202,3 @@ func (p *fairPacer) wait(ctx context.Context, flow string, bytes int) error {
 	}
 	return nil
 }
-
-func NewPacer(kbps int64) *Pacer {
-	if kbps < 1 {
-		return &Pacer{}
-	}
-	return &Pacer{bytesPerSecond: kbps * 1000 / 8}
-}
-
-func (p *Pacer) Wait(ctx context.Context, bytes int) error {
-	if p.bytesPerSecond < 1 || bytes < 1 {
-		return nil
-	}
-	now := time.Now()
-	if p.last.IsZero() || now.After(p.last) {
-		p.last = now
-	}
-	delay := time.Duration(float64(bytes) / float64(p.bytesPerSecond) * float64(time.Second))
-	p.last = p.last.Add(delay)
-	if wait := time.Until(p.last); wait > 0 {
-		timer := time.NewTimer(wait)
-		defer timer.Stop()
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		case <-timer.C:
-		}
-	}
-	return nil
-}