fix(gateway): bound Apollo video ingress

gateway/qualification_contract_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"compress/gzip"
 	"context"
+	"crypto/cipher"
 	"encoding/binary"
 	"errors"
 	"fmt"
@@ -15,10 +16,35 @@ import (
 	"runtime"
 	"strconv"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 )
 
+type qualificationBlockingAEAD struct {
+	cipher.AEAD
+	ctx     context.Context
+	blocked chan struct{}
+	release chan struct{}
+	once    sync.Once
+	waitErr error
+}
+
+func (a *qualificationBlockingAEAD) Open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
+	a.once.Do(func() {
+		close(a.blocked)
+		select {
+		case <-a.release:
+		case <-a.ctx.Done():
+			a.waitErr = a.ctx.Err()
+		}
+	})
+	if a.waitErr != nil {
+		return nil, a.waitErr
+	}
+	return a.AEAD.Open(dst, nonce, ciphertext, additionalData)
+}
+
 func TestQualificationCatalogMatchesSection7(t *testing.T) {
 	media := qualificationMediaProfiles()
 	if len(media) != 3 {
@@ -757,6 +783,94 @@ func TestQualificationLateSourceBatchDoesNotCollapseThroughPublicPath(t *testing
 		path.session.mediaRecovered.Load(), path.session.mediaEnqueued.Load())
 }
 
+func TestQualificationBlockedVideoAEADDoesNotBlockProviderIngress(t *testing.T) {
+	profile := qualificationMediaProfiles()[2]
+	blockCtx, cancelBlock := context.WithCancel(context.Background())
+	defer cancelBlock()
+	blocked := make(chan struct{})
+	release := make(chan struct{})
+	var releaseOnce sync.Once
+	releaseAEAD := func() { releaseOnce.Do(func() { close(release) }) }
+	t.Cleanup(releaseAEAD)
+	native := NewNativeApolloBackend()
+	native.configureMedia = func(media *apolloMediaCodec) {
+		media.aead = &qualificationBlockingAEAD{
+			AEAD: media.aead, ctx: blockCtx, blocked: blocked, release: release,
+		}
+	}
+	path := newQualificationPathWithNativeBackend(t, profile, profile.BitrateKbps, nil, native)
+	t.Cleanup(func() {
+		path.Close()
+		select {
+		case <-path.session.readDone:
+		case <-time.After(time.Second):
+			t.Error("native media workers did not stop during qualification cleanup")
+		}
+	})
+
+	payload := qualificationFramePayload(profile, 0)
+	packets := qualificationSourceVideoPackets(t, path.key, 1, payload)
+	if len(payload) != 666_664 || len(packets) != 662 {
+		t.Fatalf("4K60 keyframe = %d bytes/%d shards, want 666664/662", len(payload), len(packets))
+	}
+
+	beforeMetrics := path.server.Metrics()
+	beforeIngress := path.session.mediaIngress.Load()
+	beforeRecovered := path.session.mediaRecovered.Load()
+	beforeEnqueued := path.session.mediaEnqueued.Load()
+	beforePacer := path.server.pacer.reservations.Load()
+	if _, err := path.emit(t, payload); err != nil {
+		t.Fatal(err)
+	}
+	select {
+	case <-blocked:
+	case <-time.After(time.Second):
+		t.Fatal("video AEAD did not block")
+	}
+	if sent := path.fixture.sentPackets.Load(); sent != 662 {
+		t.Fatalf("fixture sent packets = %d, want 662", sent)
+	}
+	deadline := time.NewTimer(250 * time.Millisecond)
+	defer deadline.Stop()
+	for path.session.mediaIngress.Load()-beforeIngress != 662 {
+		select {
+		case <-deadline.C:
+			t.Fatalf("blocked-AEAD ingress = %d, want 662 after 662 successful fixture writes",
+				path.session.mediaIngress.Load()-beforeIngress)
+		default:
+			runtime.Gosched()
+		}
+	}
+
+	releaseAEAD()
+	recovered, err := path.receivePayload(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	afterMetrics := path.server.Metrics()
+	stageDeadline := time.Now().Add(2 * time.Second)
+	for (path.session.mediaRecovered.Load() <= beforeRecovered ||
+		path.session.mediaEnqueued.Load() <= beforeEnqueued ||
+		afterMetrics.ProcessingSamples <= beforeMetrics.ProcessingSamples ||
+		afterMetrics.MediaPackets <= beforeMetrics.MediaPackets) && time.Now().Before(stageDeadline) {
+		runtime.Gosched()
+		afterMetrics = path.server.Metrics()
+	}
+	if path.backend.setups.Load() != 1 || path.backend.opens.Load() != 1 ||
+		path.session.mediaRecovered.Load()-beforeRecovered != 1 ||
+		path.session.mediaEnqueued.Load()-beforeEnqueued != 1 ||
+		afterMetrics.ProcessingSamples <= beforeMetrics.ProcessingSamples ||
+		path.server.pacer.reservations.Load() <= beforePacer ||
+		afterMetrics.MediaPackets <= beforeMetrics.MediaPackets || !bytes.Equal(recovered, payload) {
+		t.Fatalf("blocked-AEAD public path: setup=%d open=%d ingress=%d recovered=%d enqueued=%d processing=%d pacer=%d media=%d payload=%t",
+			path.backend.setups.Load(), path.backend.opens.Load(), path.session.mediaIngress.Load()-beforeIngress,
+			path.session.mediaRecovered.Load()-beforeRecovered, path.session.mediaEnqueued.Load()-beforeEnqueued,
+			afterMetrics.ProcessingSamples-beforeMetrics.ProcessingSamples,
+			path.server.pacer.reservations.Load()-beforePacer, afterMetrics.MediaPackets-beforeMetrics.MediaPackets,
+			bytes.Equal(recovered, payload))
+	}
+}
+
 func TestQualificationProcessingRetainsProviderIngressDiagnostics(t *testing.T) {
 	profile := qualificationMediaProfiles()[2]
 	path := newQualificationProcessingPath(t, profile, profile.BitrateKbps)