fix(gateway): recover bounded pacing debt

2026-08-09 20:54:15 +07:00
parent 55afea72a1
commit 122080ab34
8 changed files with 334 additions and 3 deletions
@@ -719,6 +719,184 @@ func TestQualificationCarriesCompleteLargeFramesThroughPublicPath(t *testing.T)
 	}
 }

+func TestQualificationPacerRepaysThreeMediaLoopStallsThroughPublicPath(t *testing.T) {
+	profile := qualificationMediaProfiles()[0]
+	const totalFrames = 480
+	barriers := []int64{30, 180, 330}
+	type stall struct {
+		public  chan struct{}
+		blocked chan uint64
+		release chan struct{}
+	}
+	stalls := make([]stall, len(barriers))
+	for index := range stalls {
+		stalls[index] = stall{public: make(chan struct{}), blocked: make(chan uint64, 1), release: make(chan struct{})}
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
+	defer cancel()
+	var path *qualificationPath
+	var completed int64
+	var observationMu sync.Mutex
+	var maximumQueueDelay time.Duration
+	observer := func(observation mediaTimingObservation) {
+		observationMu.Lock()
+		maximumQueueDelay = max(maximumQueueDelay, observation.QueueDelay)
+		completed++
+		current := completed
+		observationMu.Unlock()
+		for index, barrier := range barriers {
+			if current != barrier {
+				continue
+			}
+			select {
+			case <-stalls[index].public:
+			case <-ctx.Done():
+				return
+			}
+			baseline := path.session.mediaRecovered.Load()
+			select {
+			case stalls[index].blocked <- baseline:
+			case <-ctx.Done():
+				return
+			}
+			select {
+			case <-stalls[index].release:
+			case <-ctx.Done():
+			}
+			return
+		}
+	}
+	path = newQualificationPathWithNativeBackendAndObserver(
+		t, profile, qualificationFramePacerKbps(profile), nil, NewNativeApolloBackend(), observer,
+	)
+	defer path.Close()
+
+	spacing := time.Second / time.Duration(profile.FPS)
+	started := time.Now().Add(10 * time.Millisecond)
+	producerDone := make(chan error, 1)
+	go func() {
+		var nextRelease time.Time
+		for index := int64(0); index < totalFrames; index++ {
+			release := qualificationBoundedRelease(
+				started.Add(time.Duration(index)*spacing), nextRelease, time.Now(), spacing,
+			)
+			if err := qualificationWaitContext(ctx, release); err != nil {
+				producerDone <- err
+				return
+			}
+			nextRelease = release.Add(spacing)
+			if _, err := path.emit(t, qualificationFramePayload(profile, index)); err != nil {
+				producerDone <- err
+				return
+			}
+		}
+		producerDone <- nil
+	}()
+
+	var expectedVersePackets uint64
+	receiverDone := make(chan error, 1)
+	go func() {
+		for index := int64(0); index < totalFrames; index++ {
+			payload, err := path.receivePayload(ctx)
+			if err != nil {
+				receiverDone <- err
+				return
+			}
+			expected := qualificationFramePayload(profile, index)
+			if !bytes.Equal(payload, expected) {
+				receiverDone <- fmt.Errorf("public payload changed at frame %d", index)
+				return
+			}
+			fragments := (len(payload) + frameV2PayloadSize - 1) / frameV2PayloadSize
+			expectedVersePackets += uint64(fragments)
+			for barrierIndex, barrier := range barriers {
+				if index+1 == barrier {
+					close(stalls[barrierIndex].public)
+				}
+			}
+		}
+		receiverDone <- nil
+	}()
+
+	for index := range stalls {
+		var baseline uint64
+		select {
+		case baseline = <-stalls[index].blocked:
+		case <-ctx.Done():
+			t.Fatalf("media loop did not block at public frame %d: %v", barriers[index], ctx.Err())
+		}
+		target := baseline + 6
+		for path.session.mediaRecovered.Load() < target {
+			select {
+			case <-ctx.Done():
+				t.Fatalf("stall %d recovered %d, want at least %d: %v", index, path.session.mediaRecovered.Load(), target, ctx.Err())
+			default:
+				runtime.Gosched()
+			}
+		}
+		close(stalls[index].release)
+	}
+	if err := <-producerDone; err != nil {
+		t.Fatalf("source fixture: %v", err)
+	}
+	if err := <-receiverDone; err != nil {
+		t.Fatalf("independent client: %v; expected=%d sent=%d source=%d ingress=%d recovered=%d enqueued=%d provider_drops=%d gateway_drops=%d queue_count=%d queue_bytes=%d",
+			err, path.expectedSourceUDP.Load(), path.fixture.sentPackets.Load(), path.sourceUDP.Load(),
+			path.session.mediaIngress.Load(), path.session.mediaRecovered.Load(), path.session.mediaEnqueued.Load(),
+			path.session.Telemetry().MediaDrops, path.server.Metrics().MediaDrops,
+			path.session.mediaQueueMaximum.Load(), path.session.mediaQueueMaximumBytes.Load())
+	}
+	for {
+		observationMu.Lock()
+		observed := completed
+		observationMu.Unlock()
+		if observed == totalFrames {
+			break
+		}
+		select {
+		case <-ctx.Done():
+			t.Fatalf("media observer completed %d frames, want %d: %v", observed, totalFrames, ctx.Err())
+		default:
+			runtime.Gosched()
+		}
+	}
+
+	if source := path.expectedSourceUDP.Load(); source != path.fixture.sentPackets.Load() || source != path.sourceUDP.Load() || source != path.session.mediaIngress.Load() {
+		t.Fatalf("source accounting expected=%d sent=%d source=%d ingress=%d", source, path.fixture.sentPackets.Load(), path.sourceUDP.Load(), path.session.mediaIngress.Load())
+	}
+	metrics := path.server.Metrics()
+	observationMu.Lock()
+	completedFrames := completed
+	queueDelay := maximumQueueDelay
+	observationMu.Unlock()
+	if path.backend.setups.Load() != 1 || path.backend.opens.Load() != 1 || completedFrames != totalFrames ||
+		path.session.mediaRecovered.Load() != totalFrames || path.session.mediaEnqueued.Load() != totalFrames ||
+		metrics.ProcessingSamples != totalFrames || metrics.MediaPackets != expectedVersePackets ||
+		path.server.pacer.reservations.Load() != expectedVersePackets ||
+		path.session.Telemetry().MediaDrops != 0 || path.server.Metrics().MediaDrops != 0 {
+		t.Fatalf("media accounting setup=%d open=%d completed=%d recovered=%d enqueued=%d processing=%d media=%d pacer=%d provider_drops=%d gateway_drops=%d",
+			path.backend.setups.Load(), path.backend.opens.Load(), completedFrames,
+			path.session.mediaRecovered.Load(), path.session.mediaEnqueued.Load(), metrics.ProcessingSamples,
+			metrics.MediaPackets, path.server.pacer.reservations.Load(), path.session.Telemetry().MediaDrops, metrics.MediaDrops)
+	}
+	if path.session.mediaQueueMaximum.Load() > nativeApolloVideoQueuePackets ||
+		path.session.mediaQueueMaximumBytes.Load() > nativeApolloVideoQueueBytes || queueDelay > nativeApolloVideoQueueLatency {
+		t.Fatalf("queue bounds count=%d bytes=%d residence=%s", path.session.mediaQueueMaximum.Load(), path.session.mediaQueueMaximumBytes.Load(), queueDelay)
+	}
+	t.Logf("three-stall public path: frames=%d source=%d ingress=%d recovered=%d enqueued=%d drops=%d/%d queue=%d/%d residence=%s verse_packets=%d",
+		totalFrames, path.sourceUDP.Load(), path.session.mediaIngress.Load(), path.session.mediaRecovered.Load(),
+		path.session.mediaEnqueued.Load(), path.session.Telemetry().MediaDrops, metrics.MediaDrops,
+		path.session.mediaQueueMaximum.Load(), path.session.mediaQueueMaximumBytes.Load(), queueDelay, metrics.MediaPackets)
+
+	path.Close()
+	select {
+	case <-path.session.readDone:
+	case <-time.After(time.Second):
+		t.Fatal("native media workers did not stop")
+	}
+}
+
 func TestQualificationLateSourceBatchDoesNotCollapseThroughPublicPath(t *testing.T) {
 	profile := qualificationMediaProfiles()[2]
 	path := newQualificationPath(t, profile, profile.BitrateKbps)