test(gateway): measure qualification wire capacity

gateway/qualification_harness_test.go

@@ -38,7 +38,7 @@ import (
 )
 
 const (
-	qualificationToolVersion                         = "versevdi-gateway-qualification/v8"
+	qualificationToolVersion                         = "versevdi-gateway-qualification/v9"
 	qualificationImpairmentQueuePackets              = nativeApolloVideoQueuePackets
 	qualificationImpairmentMaxPackets                = 100_000
 	qualificationImpairmentPacketCount               = 10_000
@@ -1243,9 +1243,13 @@ func (p *qualificationPath) processingDiagnostics(processed int64) qualification
 }
 
 func (p *qualificationPath) receivePayload(parent context.Context) ([]byte, error) {
+	return p.receivePayloadObserved(parent, nil)
+}
+
+func (p *qualificationPath) receivePayloadObserved(parent context.Context, observe func(time.Time, int)) ([]byte, error) {
 	ctx, cancel := context.WithTimeout(parent, 2*time.Second)
 	defer cancel()
-	return p.client.ReceiveMedia(ctx)
+	return p.client.receiveMedia(ctx, observe)
 }
 
 func qualificationSourceVideoPackets(t *testing.T, key []byte, frame uint32, encoded []byte) [][]byte {
@@ -1503,6 +1507,7 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 	beforeProviderDrops := path.session.mediaDrops.Load()
 	beforeSourceUDP := path.sourceUDP.Load()
 	beforePacer := path.server.pacer.reservations.Load()
+	wireDeliveries := make([]qualificationDeliverySample, 0, len(jobs)*2)
 	grace := max(2*profile.RTT+2*profile.Jitter, 2*time.Second)
 	lastTarget := time.Duration(packetCount) * spacing
 	if len(jobs) > 0 {
@@ -1522,7 +1527,9 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 		metrics := beforeMetrics
 		seen := make([]bool, packetCount)
 		for len(result.packets) < len(jobs) {
-			recovered, receiveErr := path.receivePayload(receiveCtx)
+			recovered, receiveErr := path.receivePayloadObserved(receiveCtx, func(receivedAt time.Time, bytes int) {
+				wireDeliveries = append(wireDeliveries, qualificationDeliverySample{At: receivedAt, Bytes: int64(bytes)})
+			})
 			if receiveErr != nil {
 				if errors.Is(receiveErr, context.DeadlineExceeded) || errors.Is(receiveErr, context.Canceled) {
 					break
@@ -1595,7 +1602,6 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 		return qualificationImpairmentObservation{}, received.err
 	}
 
-	var deliveries []qualificationDeliverySample
 	var totalLatency, totalJitter, previousLatency time.Duration
 	previousDelivered := -1
 	for _, packet := range received.packets {
@@ -1620,10 +1626,6 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 			observation.ObservedOutOfOrder++
 		}
 		previousDelivered = packet.index
-		datagrams := (media.PacketBytes + frameV2PayloadSize - 1) / frameV2PayloadSize
-		deliveries = append(deliveries, qualificationDeliverySample{
-			At: packet.deliveredAt, Bytes: int64(media.PacketBytes + datagrams*frameV2HeaderSize),
-		})
 		observation.MaxQueuePackets = max(observation.MaxQueuePackets, packet.queuePackets)
 	}
 	observation.Delivered = len(received.packets)
@@ -1712,17 +1714,11 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 		return qualificationImpairmentObservation{}, err
 	}
 	for _, reduction := range profile.CapacitySteps {
-		bytesPerSecond := media.BitrateKbps * int64(100-reduction) * 1000 / 100 / 8
-		stepDeliveries := qualificationDeliveriesAfter(deliveries, stepAt[reduction])
-		convergence := qualificationMeasuredConvergence(stepDeliveries, stepAt[reduction], bytesPerSecond)
-		maximum := qualificationMaximumDeliveryBytes(stepDeliveries, 5*time.Second)
-		observation.CapacityStepObservations = append(observation.CapacityStepObservations, qualificationCapacityStep{
-			ReductionPercent: reduction, Convergence: convergence,
-			MaximumFiveSecond: maximum, FiveSecondCap: bytesPerSecond * 5,
-		})
+		step := qualificationCapacityStepObservation(wireDeliveries, stepAt[reduction], media, reduction)
+		observation.CapacityStepObservations = append(observation.CapacityStepObservations, step)
 		if packetCount >= qualificationImpairmentPacketCount &&
-			(convergence > 10*time.Second || maximum > bytesPerSecond*5*105/100) {
-			return qualificationImpairmentObservation{}, fmt.Errorf("capacity step %d failed measured convergence=%s five-second=%d", reduction, convergence, maximum)
+			(step.Convergence > 10*time.Second || step.MaximumFiveSecond > step.FiveSecondCap*105/100) {
+			return qualificationImpairmentObservation{}, fmt.Errorf("capacity step %d failed measured convergence=%s five-second=%d", reduction, step.Convergence, step.MaximumFiveSecond)
 		}
 	}
 	if observation.Delivered+observation.Dropped != observation.Sent ||
@@ -1733,6 +1729,17 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
 	return observation, nil
 }
 
+func qualificationCapacityStepObservation(wireDeliveries []qualificationDeliverySample, start time.Time, media qualificationMediaProfile, reduction int) qualificationCapacityStep {
+	bytesPerSecond := qualificationMediaPacerKbps(media, reduction) * 1000 / 8
+	deliveries := qualificationDeliveriesAfter(wireDeliveries, start)
+	return qualificationCapacityStep{
+		ReductionPercent:  reduction,
+		Convergence:       qualificationMeasuredConvergence(deliveries, start, bytesPerSecond),
+		MaximumFiveSecond: qualificationMaximumDeliveryBytes(deliveries, 5*time.Second),
+		FiveSecondCap:     bytesPerSecond * 5,
+	}
+}
+
 func qualificationKnownImpairment(profile qualificationImpairmentProfile) bool {
 	for _, known := range qualificationImpairmentProfiles() {
 		if profile.Name != known.Name || profile.RTT != known.RTT || profile.Jitter != known.Jitter ||
@@ -1759,9 +1766,16 @@ func qualificationDeliveriesAfter(deliveries []qualificationDeliverySample, star
 func qualificationMeasuredConvergence(deliveries []qualificationDeliverySample, start time.Time, targetBytesPerSecond int64) time.Duration {
 	const window = 250 * time.Millisecond
 	const requiredWindows = 4
+	if len(deliveries) == 0 {
+		return 11 * time.Second
+	}
+	windowOrigin := start
+	if deliveries[0].At.After(windowOrigin) {
+		windowOrigin = deliveries[0].At
+	}
 	consecutive := 0
 	for offset := time.Duration(0); offset <= 10*time.Second; offset += window {
-		windowStart := start.Add(offset)
+		windowStart := windowOrigin.Add(offset)
 		var total int64
 		for _, delivery := range deliveries {
 			if !delivery.At.Before(windowStart) && delivery.At.Before(windowStart.Add(window)) {
@@ -1772,7 +1786,7 @@ func qualificationMeasuredConvergence(deliveries []qualificationDeliverySample,
 		if rate >= targetBytesPerSecond*90/100 && rate <= targetBytesPerSecond*105/100 {
 			consecutive++
 			if consecutive == requiredWindows {
-				return offset + window
+				return windowStart.Add(window).Sub(start)
 			}
 		} else {
 			consecutive = 0