fix(gateway): preserve paced qualification traffic

gateway/qualification_harness_test.go

@@ -291,6 +291,21 @@ func qualificationBoundedRelease(target, next, now time.Time, spacing time.Durat
 	return release
 }
 
+func qualificationWaitUntil(target time.Time) {
+	const preciseWindow = 500 * time.Microsecond
+	for {
+		delay := time.Until(target)
+		if delay <= 0 {
+			return
+		}
+		if delay > preciseWindow {
+			time.Sleep(delay - preciseWindow)
+		} else if delay > 50*time.Microsecond {
+			runtime.Gosched()
+		}
+	}
+}
+
 type qualificationTracingBackend struct {
 	native   *NativeApolloBackend
 	setups   atomic.Uint64
@@ -1340,9 +1355,7 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 	var nextRelease time.Time
 	for _, packet := range jobs {
 		release := qualificationBoundedRelease(started.Add(packet.target), nextRelease, time.Now(), spacing)
-		if delay := time.Until(release); delay > 0 {
-			time.Sleep(delay)
-		}
+		qualificationWaitUntil(release)
 		nextRelease = release.Add(spacing)
 		if len(profile.CapacitySteps) == 2 {
 			switch {
@@ -1590,20 +1603,71 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
 	bytesPerSecond := profile.BitrateKbps * 1000 / 8
 	targetBytes := bytesPerSecond * profile.Duration.Nanoseconds() / int64(time.Second)
 	targetPackets := (targetBytes + int64(profile.PacketBytes) - 1) / int64(profile.PacketBytes)
+	spacing := time.Duration(int64(time.Second) * int64(profile.PacketBytes) * 8 / (profile.BitrateKbps * 1000))
+	maximumDuration := profile.Duration*105/100 + 250*time.Millisecond
 	started := time.Now()
+	receiveCtx, receiveCancel := context.WithDeadline(context.Background(), started.Add(maximumDuration+2*time.Second))
+	defer receiveCancel()
+	receivedDone := make(chan error, 1)
+	go func() {
+		for index := int64(0); index < targetPackets; index++ {
+			recovered, receiveErr := path.receivePayload(receiveCtx)
+			if receiveErr != nil {
+				receivedDone <- receiveErr
+				return
+			}
+			if len(recovered) != len(payload) {
+				receivedDone <- fmt.Errorf(
+					"qualification processing payload length = %d, want %d at sequence %d",
+					len(recovered), len(payload), index,
+				)
+				return
+			}
+			sequence := binary.BigEndian.Uint32(recovered[len(recovered)-4:])
+			if sequence != uint32(index) {
+				receivedDone <- fmt.Errorf(
+					"qualification processing sequence = %d, want %d", sequence, index,
+				)
+				return
+			}
+			expected := append([]byte(nil), payload...)
+			binary.BigEndian.PutUint32(expected[len(expected)-4:], uint32(index))
+			if !bytes.Equal(recovered, expected) {
+				receivedDone <- fmt.Errorf("qualification processing payload bytes changed at sequence %d", index)
+				return
+			}
+		}
+		receivedDone <- nil
+	}()
 	var processed int64
-	for processed < targetPackets || time.Since(started) < profile.Duration {
+	var nextRelease time.Time
+	for processed < targetPackets {
+		select {
+		case receiveErr := <-receivedDone:
+			if receiveErr != nil {
+				snapshot, _ := path.process.snapshot()
+				return qualificationProcessingSummary{}, fmt.Errorf("%w; gateway snapshot=%#v", receiveErr, snapshot)
+			}
+			return qualificationProcessingSummary{}, errors.New("qualification processing receiver ended early")
+		default:
+		}
+		release := qualificationBoundedRelease(
+			started.Add(time.Duration(processed)*spacing), nextRelease, time.Now(), spacing,
+		)
+		qualificationWaitUntil(release)
+		nextRelease = release.Add(spacing)
 		current := append([]byte(nil), payload...)
 		binary.BigEndian.PutUint32(current[len(current)-4:], uint32(processed))
 		if _, err := path.emit(t, current); err != nil {
 			return qualificationProcessingSummary{}, err
 		}
-		recovered, err := path.receivePayload(context.Background())
-		if err != nil || !bytes.Equal(recovered, current) {
-			return qualificationProcessingSummary{}, errors.New("qualification processing payload integrity failure")
-		}
 		processed++
 	}
+	if err := <-receivedDone; err != nil {
+		snapshot, _ := path.process.snapshot()
+		return qualificationProcessingSummary{}, fmt.Errorf("%w; gateway snapshot=%#v", err, snapshot)
+	}
+	qualificationWaitUntil(started.Add(profile.Duration))
 	actualDuration := time.Since(started)
 	record, err := path.process.stopRecording()
 	if err != nil {
@@ -1664,8 +1728,8 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
 	if summary.CPUSeconds < 0 || summary.ClockOverhead <= 0 || summary.ClockMethod == "" {
 		return qualificationProcessingSummary{}, errors.New("process CPU usage unavailable")
 	}
-	if actualDuration < profile.Duration || actualDuration > profile.Duration*105/100+250*time.Millisecond {
-		return qualificationProcessingSummary{}, fmt.Errorf("wall-clock duration %s outside [%s,%s]", actualDuration, profile.Duration, profile.Duration*105/100+250*time.Millisecond)
+	if actualDuration < profile.Duration || actualDuration > maximumDuration {
+		return qualificationProcessingSummary{}, fmt.Errorf("wall-clock duration %s outside [%s,%s]", actualDuration, profile.Duration, maximumDuration)
 	}
 	if summary.ObservedBitrateKbps < float64(profile.BitrateKbps)*0.95 || summary.ObservedBitrateKbps > float64(profile.BitrateKbps)*1.05 {
 		return qualificationProcessingSummary{}, fmt.Errorf("observed bitrate %.2f outside profile bounds for %d", summary.ObservedBitrateKbps, profile.BitrateKbps)
@@ -2137,7 +2201,7 @@ func TestSection7Qualification(t *testing.T) {
 		OS:           runtime.GOOS, Architecture: runtime.GOARCH,
 		Topology:        "parent source-shaped encrypted Apollo fixture -> isolated gateway subprocess for processing/resource evidence -> public Verse client decoder; impairment uses the same native recovery/FEC, bounded queue, production pacer, framing, and QUIC path",
 		Direction:       "provider_to_client",
-		QueueDiscipline: "ordered fixed-seed source delay queue with one-serialization-interval catch-up, bounded 16-packet native provider queue, production equal-tier fair pacer",
+		QueueDiscipline: "ordered fixed-seed source delay queue with one-serialization-interval catch-up, bounded 64-packet native video queue, production equal-tier fair pacer",
 		Evidence:        []string{"deterministic source-shaped Apollo recovery", "isolated gateway-process resources", "local real-time production path", "mTLS/QUIC fixture transport", "attributed path impairment", "production fair pacer"},
 		Deferred:        []string{"live Apollo", "macOS client", "physical firewall and packet route", "real encoder fidelity", "multi-host scale"},
 	}