2253 lines
88 KiB
Go
2253 lines
88 KiB
Go
package gateway
|
|
|
|
import (
|
|
"bufio"
|
|
"bytes"
|
|
"compress/gzip"
|
|
"context"
|
|
"crypto/cipher"
|
|
"encoding/binary"
|
|
"encoding/csv"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net"
|
|
"os"
|
|
"path/filepath"
|
|
"reflect"
|
|
"runtime"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
type qualificationBlockingAEAD struct {
|
|
cipher.AEAD
|
|
ctx context.Context
|
|
blocked chan struct{}
|
|
release chan struct{}
|
|
once sync.Once
|
|
waitErr error
|
|
}
|
|
|
|
func (a *qualificationBlockingAEAD) Open(dst, nonce, ciphertext, additionalData []byte) ([]byte, error) {
|
|
a.once.Do(func() {
|
|
close(a.blocked)
|
|
select {
|
|
case <-a.release:
|
|
case <-a.ctx.Done():
|
|
a.waitErr = a.ctx.Err()
|
|
}
|
|
})
|
|
if a.waitErr != nil {
|
|
return nil, a.waitErr
|
|
}
|
|
return a.AEAD.Open(dst, nonce, ciphertext, additionalData)
|
|
}
|
|
|
|
func TestQualificationCatalogMatchesSection7(t *testing.T) {
|
|
media := qualificationMediaProfiles()
|
|
if len(media) != 3 {
|
|
t.Fatalf("media profile count = %d, want 3", len(media))
|
|
}
|
|
wantMedia := []qualificationMediaProfile{
|
|
{Name: "1080p60-h264", Codec: "h264", BitrateKbps: 20000, FPS: 60, Duration: 10 * time.Minute, Warmup: time.Second, PacketBytes: 1179},
|
|
{Name: "1440p120-hevc", Codec: "hevc", BitrateKbps: 50000, FPS: 120, Duration: 10 * time.Minute, Warmup: time.Second, PacketBytes: 1179},
|
|
{Name: "4k60-hevc", Codec: "hevc", BitrateKbps: 80000, FPS: 60, Duration: 10 * time.Minute, Warmup: time.Second, PacketBytes: 1179},
|
|
}
|
|
if !reflect.DeepEqual(media, wantMedia) {
|
|
t.Fatalf("media profiles = %#v, want %#v", media, wantMedia)
|
|
}
|
|
|
|
impairments := qualificationImpairmentProfiles()
|
|
wantImpairments := []qualificationImpairmentProfile{
|
|
{Name: "baseline", RTT: 20 * time.Millisecond},
|
|
{Name: "latency", RTT: 150 * time.Millisecond},
|
|
{Name: "jitter", RTT: 50 * time.Millisecond, Jitter: 30 * time.Millisecond},
|
|
{Name: "loss", RTT: 50 * time.Millisecond, Jitter: 10 * time.Millisecond, LossPercent: 5},
|
|
{Name: "reorder", RTT: 100 * time.Millisecond, Jitter: 10 * time.Millisecond, LossPercent: 1, Reorder: true},
|
|
{Name: "constrained", RTT: 50 * time.Millisecond, Jitter: 10 * time.Millisecond, LossPercent: 2, Reorder: true, CapacitySteps: []int{25, 50}},
|
|
}
|
|
if !reflect.DeepEqual(impairments, wantImpairments) {
|
|
t.Fatalf("impairment profiles = %#v, want %#v", impairments, wantImpairments)
|
|
}
|
|
}
|
|
|
|
func TestQualificationProtocolVersionIsExplicitAndImmutable(t *testing.T) {
|
|
const version = "v1.0.0-phase3c-gateway-rc.8"
|
|
t.Setenv("VERSEVDI_QUALIFICATION_PROTOCOL_VERSION", version)
|
|
got, err := qualificationProtocolVersion()
|
|
if err != nil || got != version {
|
|
t.Fatalf("qualificationProtocolVersion() = %q, want %q", got, version)
|
|
}
|
|
for _, invalid := range []string{"", "unknown", "v1", " v1.0.0", "v1.0.0+mutable"} {
|
|
t.Setenv("VERSEVDI_QUALIFICATION_PROTOCOL_VERSION", invalid)
|
|
if _, err := qualificationProtocolVersion(); err == nil {
|
|
t.Fatalf("qualificationProtocolVersion() accepted %q", invalid)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestQualificationRecordsLinkedToolVersions(t *testing.T) {
|
|
versions, err := qualificationToolVersions()
|
|
if err != nil || versions["qualification"] != qualificationToolVersion ||
|
|
versions["go"] == "" || versions["quic-go"] == "" {
|
|
t.Fatalf("qualification tool versions = %#v, %v", versions, err)
|
|
}
|
|
}
|
|
|
|
func TestQualificationApolloFixturePacesSourceShapedVideo(t *testing.T) {
|
|
key := bytes.Repeat([]byte{0x3c}, 16)
|
|
encoded := make([]byte, 1000*apolloVideoShardPayloadSize-8)
|
|
packets := qualificationSourceVideoPackets(t, key, 1, encoded)
|
|
if len(packets) != 1000 || len(packets[0]) != 1072 {
|
|
t.Fatalf("source vector = %d packets of %d bytes, want 1000 packets of 1072 bytes", len(packets), len(packets[0]))
|
|
}
|
|
packetsPerMillisecond, batchSize := qualificationApolloVideoPacing(apolloVideoRawPacketSize)
|
|
if apolloVideoRawPacketSize != 1040 || packetsPerMillisecond != 96 || batchSize != 63 {
|
|
t.Fatalf("Apollo raw pacing vector = %d bytes, %d packets/ms, batch %d; want 1040, 96, and 63", apolloVideoRawPacketSize, packetsPerMillisecond, batchSize)
|
|
}
|
|
wantOffsets := []time.Duration{0, 656250 * time.Nanosecond, 1312500 * time.Nanosecond, 10416666 * time.Nanosecond}
|
|
for index, sent := range []int{0, 63, 126, 1000} {
|
|
if got := qualificationApolloVideoOffset(sent, packetsPerMillisecond); got != wantOffsets[index] {
|
|
t.Fatalf("Apollo pacing offset after %d packets = %s, want %s", sent, got, wantOffsets[index])
|
|
}
|
|
}
|
|
|
|
receiver, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer receiver.Close()
|
|
sender, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer sender.Close()
|
|
|
|
fixture := &qualificationApolloFixture{video: sender, failures: make(chan error, 1)}
|
|
remote := *receiver.LocalAddr().(*net.UDPAddr)
|
|
fixture.videoRemote.Store(&remote)
|
|
var batchIndices []int
|
|
var batchStarts []time.Duration
|
|
var started time.Time
|
|
fixture.observeVideoBatch = func(index int, at time.Time) {
|
|
if len(batchIndices) < 3 {
|
|
batchIndices = append(batchIndices, index)
|
|
batchStarts = append(batchStarts, at.Sub(started))
|
|
}
|
|
}
|
|
drained := make(chan struct{})
|
|
go func() {
|
|
defer close(drained)
|
|
buffer := make([]byte, 2048)
|
|
for range len(packets) + 1 {
|
|
if _, _, readErr := receiver.ReadFromUDP(buffer); readErr != nil {
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
|
|
started = time.Now()
|
|
if err := fixture.sendVideo(context.Background(), 1, packets); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := fixture.sendVideo(context.Background(), 2, packets[:1]); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
elapsed := time.Since(started)
|
|
wantCarry := qualificationApolloVideoOffset(len(packets), packetsPerMillisecond)
|
|
if wantCarry != 10416666*time.Nanosecond {
|
|
t.Fatalf("1000-packet carry = %s, want 10.416666 ms", wantCarry)
|
|
}
|
|
if elapsed < wantCarry {
|
|
t.Fatalf("source fixture sent the next frame after %s, before Apollo pacing carry %s", elapsed, wantCarry)
|
|
}
|
|
wantIndices := []int{0, 63, 126}
|
|
wantStarts := []time.Duration{0, 656250 * time.Nanosecond, 1312500 * time.Nanosecond}
|
|
if len(batchIndices) != len(wantIndices) {
|
|
t.Fatalf("observed %d batch starts, want %d", len(batchIndices), len(wantIndices))
|
|
}
|
|
for index := range wantIndices {
|
|
if batchIndices[index] != wantIndices[index] || batchStarts[index] < wantStarts[index] {
|
|
t.Fatalf("batch starts = indices %v at %v; want indices %v no earlier than %v", batchIndices, batchStarts, wantIndices, wantStarts)
|
|
}
|
|
}
|
|
select {
|
|
case <-drained:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("source-shaped UDP receiver did not drain the fixed vector")
|
|
}
|
|
fixture.videoNext = time.Now().Add(time.Second)
|
|
beforeCancel := fixture.sentPackets.Load()
|
|
cancelled, cancel := context.WithCancel(context.Background())
|
|
cancel()
|
|
if err := fixture.sendVideo(cancelled, 3, packets[:1]); !errors.Is(err, context.Canceled) {
|
|
t.Fatalf("cancelled pacing wait returned %v, want context.Canceled", err)
|
|
}
|
|
if fixture.sentPackets.Load() != beforeCancel {
|
|
t.Fatal("cancelled pacing wait emitted a UDP shard")
|
|
}
|
|
}
|
|
|
|
func TestQualificationOutputAndStatisticsFailClosed(t *testing.T) {
|
|
if err := validateQualificationOutputDir("relative/evidence"); err == nil {
|
|
t.Fatal("relative evidence directory was accepted")
|
|
}
|
|
if err := validateQualificationOutputDir(filepath.Join(t.TempDir(), "evidence")); err != nil {
|
|
t.Fatalf("absolute evidence directory rejected: %v", err)
|
|
}
|
|
|
|
summary, err := summarizeQualificationSamples([]time.Duration{
|
|
time.Millisecond, 2 * time.Millisecond, 3 * time.Millisecond,
|
|
4 * time.Millisecond, 5 * time.Millisecond,
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if summary.Count != 5 || summary.Min != time.Millisecond || summary.Median != 3*time.Millisecond ||
|
|
summary.P90 != 5*time.Millisecond || summary.P95 != 5*time.Millisecond ||
|
|
summary.P99 != 5*time.Millisecond || summary.Max != 5*time.Millisecond ||
|
|
summary.Mean != 3*time.Millisecond || summary.Histogram["le_5ms"] != 5 {
|
|
t.Fatalf("summary = %#v", summary)
|
|
}
|
|
if err := enforceQualificationProcessingGate(summary); err != nil {
|
|
t.Fatalf("5 ms p95 rejected: %v", err)
|
|
}
|
|
summary.P95++
|
|
if err := enforceQualificationProcessingGate(summary); err == nil {
|
|
t.Fatal("p95 above 5 ms was accepted")
|
|
}
|
|
}
|
|
|
|
func TestQualificationShortProcessingWritesRawArtifact(t *testing.T) {
|
|
profile := qualificationMediaProfile{
|
|
Name: "smoke", Codec: "h264", BitrateKbps: 20000,
|
|
Duration: time.Second, Warmup: time.Millisecond, PacketBytes: 1000,
|
|
}
|
|
rawPath := filepath.Join(t.TempDir(), "processing.csv.gz")
|
|
summary, err := runQualificationProcessing(t, profile, rawPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if summary.Count < 1 || summary.RawSamplesSHA256 == "" || summary.RawSamplesBytes < 1 ||
|
|
summary.ResourceSamples < 2 || summary.RawResourcesSHA256 == "" || summary.RawResourcesBytes < 1 ||
|
|
summary.CPUScope != qualificationGatewayCPUScope || summary.ResourceMethod != qualificationResourceMethod ||
|
|
summary.ClockOverhead <= 0 || summary.ClockMethod == "" {
|
|
t.Fatalf("processing summary = %#v", summary)
|
|
}
|
|
file, err := os.Open(rawPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer file.Close()
|
|
reader, err := gzip.NewReader(file)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
raw, err := io.ReadAll(reader)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := reader.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.HasPrefix(string(raw), "elapsed_ns,queue_ns,processing_ns,pacing_ns\n") ||
|
|
strings.Count(string(raw), "\n") != int(summary.Count)+1 {
|
|
t.Fatalf("raw sample rows do not match summary count: %q", raw)
|
|
}
|
|
}
|
|
|
|
func TestQualificationShortProcessingSubprocessCoversFixedProfiles(t *testing.T) {
|
|
for _, profile := range qualificationMediaProfiles() {
|
|
profile.Duration = time.Second
|
|
profile.Warmup = 10 * time.Millisecond
|
|
summary, err := runQualificationProcessing(t, profile, filepath.Join(t.TempDir(), profile.Name+".csv.gz"))
|
|
if err != nil {
|
|
t.Fatalf("%s: %v", profile.Name, err)
|
|
}
|
|
if summary.Count < 1 || summary.CPUScope != qualificationGatewayCPUScope ||
|
|
summary.ClockOverhead <= 0 || summary.ClockMethod != qualificationClockOverheadMethod ||
|
|
summary.Count != int64(profile.FPS) ||
|
|
summary.ConfiguredFPS != profile.FPS ||
|
|
summary.ObservedFPS < float64(profile.FPS)*0.95 ||
|
|
summary.ObservedFPS > float64(profile.FPS)*1.05 ||
|
|
summary.PayloadBytes != profile.BitrateKbps*1000/8 ||
|
|
summary.MaximumFrameBytes <= summary.MinimumFrameBytes ||
|
|
summary.MaximumFrameBytes <= 18_864 ||
|
|
summary.PayloadSHA256 == "" ||
|
|
summary.ObservedBitrateKbps < float64(profile.BitrateKbps)*0.95 ||
|
|
summary.ObservedBitrateKbps > float64(profile.BitrateKbps)*1.05 {
|
|
t.Fatalf("%s subprocess summary = %#v", profile.Name, summary)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestQualificationShortProcessingUsesCompleteFrameCadence(t *testing.T) {
|
|
profile := qualificationMediaProfiles()[0]
|
|
profile.Duration = 100 * time.Millisecond
|
|
profile.Warmup = time.Millisecond
|
|
summary, err := runQualificationProcessing(t, profile, filepath.Join(t.TempDir(), "frames.csv.gz"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if summary.Count != 6 {
|
|
t.Fatalf("100 ms of 1080p60 processed %d units, want 6 complete encoded frames", summary.Count)
|
|
}
|
|
}
|
|
|
|
func TestQualificationSustainedProcessingKeepsCleanPathBounded(t *testing.T) {
|
|
profile := qualificationMediaProfiles()[2]
|
|
profile.Duration = 2 * time.Minute
|
|
profile.Warmup = 100 * time.Millisecond
|
|
summary, err := runQualificationProcessing(t, profile, filepath.Join(t.TempDir(), "4k60-hevc.csv.gz"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if summary.Count < 1 || summary.ObservedBitrateKbps < float64(profile.BitrateKbps)*0.95 ||
|
|
summary.ObservedBitrateKbps > float64(profile.BitrateKbps)*1.05 {
|
|
t.Fatalf("sustained processing summary = %#v", summary)
|
|
}
|
|
}
|
|
|
|
func TestQualificationProcessingResourcesExcludeParentDriverCPU(t *testing.T) {
|
|
profile := qualificationMediaProfile{
|
|
Name: "resource-isolation", Codec: "h264", BitrateKbps: 1000,
|
|
Duration: 300 * time.Millisecond, Warmup: time.Millisecond, PacketBytes: 1000,
|
|
}
|
|
baseline, err := runQualificationProcessing(t, profile, filepath.Join(t.TempDir(), "baseline.csv.gz"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
stop := make(chan struct{})
|
|
done := make(chan struct{})
|
|
go func() {
|
|
defer close(done)
|
|
var value uint64 = 1
|
|
for {
|
|
select {
|
|
case <-stop:
|
|
if value == 0 {
|
|
panic("bounded parent work was optimized away")
|
|
}
|
|
return
|
|
default:
|
|
value = value*2862933555777941757 + 3037000493
|
|
}
|
|
}
|
|
}()
|
|
busy, err := runQualificationProcessing(t, profile, filepath.Join(t.TempDir(), "busy.csv.gz"))
|
|
close(stop)
|
|
<-done
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if busy.CPUSeconds > baseline.CPUSeconds+50*time.Millisecond.Seconds() {
|
|
t.Fatalf("parent CPU leaked into gateway sample: baseline=%.6fs busy=%.6fs", baseline.CPUSeconds, busy.CPUSeconds)
|
|
}
|
|
}
|
|
|
|
func TestQualificationProcessingPreservesPayload(t *testing.T) {
|
|
profile := qualificationMediaProfiles()[0]
|
|
payload := qualificationPayload(profile)
|
|
trace, elapsed, err := newQualificationPath(t, profile, profile.BitrateKbps).traverse(t, payload)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !trace.PayloadPreserved || !trace.ApolloRecovered || !trace.VerseQUIC {
|
|
t.Fatalf("production path trace = %#v", trace)
|
|
}
|
|
if elapsed <= 0 {
|
|
t.Fatalf("processing duration = %s", elapsed)
|
|
}
|
|
}
|
|
|
|
func TestQualificationRepeatedTraversalTracksEveryProductionStage(t *testing.T) {
|
|
profile := qualificationMediaProfiles()[1]
|
|
pacerKbps := (profile.BitrateKbps*int64(profile.PacketBytes+frameHeaderSize) + int64(profile.PacketBytes) - 1) / int64(profile.PacketBytes)
|
|
path := newQualificationPath(t, profile, pacerKbps)
|
|
payload := qualificationPayload(profile)
|
|
if err := runQualificationWarmup(t, path, profile, payload); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for index := 0; index < 25_000; index++ {
|
|
current := append([]byte(nil), payload...)
|
|
binary.BigEndian.PutUint32(current[len(current)-4:], uint32(index))
|
|
trace, _, err := path.traverse(t, current)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !trace.NativeUDPIngress || !trace.ApolloRecovered || !trace.ProductionQueue ||
|
|
!trace.ProductionMediaLoop || !trace.ProductionPacer || !trace.VerseQUIC ||
|
|
!trace.PublicClientDecode || !trace.PayloadPreserved {
|
|
t.Fatalf("traversal %d missed a production stage: %#v", index, trace)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestQualificationZeroLossBaselinesAreAttributedAtNormativeScale(t *testing.T) {
|
|
for _, media := range qualificationMediaProfiles() {
|
|
observation, err := runQualificationImpairment(t, qualificationImpairmentProfiles()[0], media,
|
|
qualificationImpairmentPacketCount, filepath.Join(t.TempDir(), media.Name+".csv.gz"))
|
|
if err != nil {
|
|
t.Fatalf("%s: %v", media.Name, err)
|
|
}
|
|
if observation.InjectedDropped != 0 || observation.Dropped != 0 ||
|
|
observation.ProviderFECDropped != 0 || observation.ProviderEnqueueDropped != 0 ||
|
|
observation.ProviderQueueReplaced != 0 || observation.GatewayDropped != 0 ||
|
|
observation.QUICSendDropped != 0 || observation.ClientDeliveryDropped != 0 ||
|
|
observation.UnexplainedDropped != 0 {
|
|
t.Fatalf("%s clean-path loss attribution = %#v", media.Name, observation)
|
|
}
|
|
if observation.SourceEmitted != qualificationImpairmentPacketCount ||
|
|
observation.ProviderRecovered != qualificationImpairmentPacketCount ||
|
|
observation.ProviderEnqueued != qualificationImpairmentPacketCount ||
|
|
observation.GatewayForwarded != qualificationImpairmentPacketCount ||
|
|
observation.QUICSent != qualificationImpairmentPacketCount ||
|
|
observation.Delivered != qualificationImpairmentPacketCount {
|
|
t.Fatalf("%s stage counts = %#v", media.Name, observation)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestQualificationImpairmentIsDeterministicAndBounded(t *testing.T) {
|
|
profile := qualificationImpairmentProfiles()[3]
|
|
first, err := runQualificationImpairment(t, profile, qualificationMediaProfiles()[0], 1000, filepath.Join(t.TempDir(), "first.csv.gz"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
second, err := runQualificationImpairment(t, profile, qualificationMediaProfiles()[0], 1000, filepath.Join(t.TempDir(), "second.csv.gz"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if first.Dropped != second.Dropped || first.InjectedReordered != second.InjectedReordered {
|
|
t.Fatalf("deterministic impairment selection differs:\n%#v\n%#v", first, second)
|
|
}
|
|
if first.Sent != 1000 || first.Delivered+first.Dropped != first.Sent ||
|
|
first.ObservedLossPercent < 3.5 || first.ObservedLossPercent > 6.5 ||
|
|
first.MaxQueuePackets > qualificationImpairmentQueuePackets || first.RawSamplesSHA256 == "" {
|
|
t.Fatalf("impairment observation = %#v", first)
|
|
}
|
|
if _, err := runQualificationImpairment(t, profile, qualificationMediaProfiles()[0], qualificationImpairmentMaxPackets+1, filepath.Join(t.TempDir(), "invalid.csv.gz")); err == nil {
|
|
t.Fatal("unbounded impairment packet count was accepted")
|
|
}
|
|
unknown := profile
|
|
unknown.Name = "private-simulator"
|
|
if _, err := runQualificationImpairment(t, unknown, qualificationMediaProfiles()[0], 1, filepath.Join(t.TempDir(), "unknown.csv.gz")); err == nil {
|
|
t.Fatal("unregistered impairment profile was accepted")
|
|
}
|
|
}
|
|
|
|
func TestQualificationRTTIsNotSyntheticDoubleOneWayCompletion(t *testing.T) {
|
|
rawPath := filepath.Join(t.TempDir(), "latency.csv.gz")
|
|
observation, err := runQualificationImpairment(
|
|
t,
|
|
qualificationImpairmentProfiles()[1],
|
|
qualificationMediaProfiles()[0],
|
|
40,
|
|
rawPath,
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
meanLatency := qualificationRawMeanLatency(t, rawPath)
|
|
delta := observation.ObservedRTT - 2*meanLatency
|
|
if delta < 0 {
|
|
delta = -delta
|
|
}
|
|
if delta < 5*time.Millisecond {
|
|
t.Fatalf("RTT %s was synthesized as twice one-way completion %s", observation.ObservedRTT, meanLatency)
|
|
}
|
|
}
|
|
|
|
func TestQualificationFixedSeedJitterIsObservableOnTraversedTraffic(t *testing.T) {
|
|
profile := qualificationImpairmentProfiles()[2]
|
|
observation, err := runQualificationImpairment(
|
|
t,
|
|
profile,
|
|
qualificationMediaProfiles()[0],
|
|
200,
|
|
filepath.Join(t.TempDir(), "jitter.csv.gz"),
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if observation.RTTSource != "apollo_enet_acknowledge" || observation.ObservedRTT <= 0 {
|
|
t.Fatalf("RTT observation is not transport-acknowledged: %#v", observation)
|
|
}
|
|
if observation.ObservedLatency < 5*time.Millisecond || observation.ObservedLatency > 100*time.Millisecond {
|
|
t.Fatalf("observed one-way latency %s does not reflect configured traversal", observation.ObservedLatency)
|
|
}
|
|
if observation.AppliedJitter < 10*time.Millisecond || observation.AppliedJitter > 25*time.Millisecond {
|
|
t.Fatalf("applied fixed-seed jitter %s is outside the uniform-delay tolerance", observation.AppliedJitter)
|
|
}
|
|
if observation.ObservedJitter <= 0 || observation.ObservedJitter > observation.AppliedJitter {
|
|
t.Fatalf("ordered traversal jitter %s is not bounded by applied jitter %s", observation.ObservedJitter, observation.AppliedJitter)
|
|
}
|
|
if observation.InjectedReordered != 0 || observation.ObservedOutOfOrder != 0 {
|
|
t.Fatalf("reorder-off jitter changed source order: %#v", observation)
|
|
}
|
|
}
|
|
|
|
func TestQualificationLossOnlyDoesNotImplicitlyReorder(t *testing.T) {
|
|
observation, err := runQualificationImpairment(
|
|
t,
|
|
qualificationImpairmentProfiles()[3],
|
|
qualificationMediaProfiles()[0],
|
|
400,
|
|
filepath.Join(t.TempDir(), "loss.csv.gz"),
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if observation.InjectedReordered != 0 || observation.ObservedOutOfOrder != 0 {
|
|
t.Fatalf("loss-only profile changed source order: %#v", observation)
|
|
}
|
|
}
|
|
|
|
func TestQualificationSourceShaperCarriesAtMostOnePacketOfCatchup(t *testing.T) {
|
|
spacing := time.Millisecond
|
|
started := time.Unix(0, 0)
|
|
now := started.Add(100 * spacing)
|
|
first := qualificationBoundedRelease(started, time.Time{}, now, spacing)
|
|
if first != now.Add(-spacing) {
|
|
t.Fatalf("first catch-up release = %s, want %s", first, now.Add(-spacing))
|
|
}
|
|
second := qualificationBoundedRelease(started.Add(spacing), first.Add(spacing), now, spacing)
|
|
if second != now {
|
|
t.Fatalf("second catch-up release = %s, want %s", second, now)
|
|
}
|
|
third := qualificationBoundedRelease(started.Add(2*spacing), second.Add(spacing), now, spacing)
|
|
if third != now.Add(spacing) {
|
|
t.Fatalf("catch-up debt was reset: third release = %s, want %s", third, now.Add(spacing))
|
|
}
|
|
}
|
|
|
|
func TestQualificationExplicitReorderIsBoundedAndAttributed(t *testing.T) {
|
|
observation, err := runQualificationImpairment(
|
|
t,
|
|
qualificationImpairmentProfiles()[4],
|
|
qualificationMediaProfiles()[0],
|
|
400,
|
|
filepath.Join(t.TempDir(), "reorder.csv.gz"),
|
|
)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if observation.InjectedReordered == 0 || observation.ObservedOutOfOrder != observation.InjectedReordered {
|
|
t.Fatalf("explicit reorder attribution = %#v", observation)
|
|
}
|
|
}
|
|
|
|
func TestQualificationGatewaySubprocessResourcesResetAndTrackWork(t *testing.T) {
|
|
profile := qualificationMediaProfile{
|
|
Name: "resource-process", Codec: "h264", BitrateKbps: 100000,
|
|
Duration: time.Second, Warmup: time.Millisecond, PacketBytes: 1000,
|
|
}
|
|
path := newQualificationProcessingPath(t, profile, qualificationFramePacerKbps(profile))
|
|
defer path.Close()
|
|
output := t.TempDir()
|
|
record := func(name string, work func() error) qualificationProcessRecordResult {
|
|
t.Helper()
|
|
if err := path.process.startRecording(
|
|
filepath.Join(output, name+".csv.gz"),
|
|
filepath.Join(output, name+"-resources.csv.gz"),
|
|
); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := work(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
result, err := path.process.stopRecording()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return result
|
|
}
|
|
idle := record("idle", func() error {
|
|
time.Sleep(150 * time.Millisecond)
|
|
return nil
|
|
})
|
|
payload := qualificationPayload(profile)
|
|
work := record("work", func() error {
|
|
for index := 0; index < 500; index++ {
|
|
if _, err := path.emit(t, payload); err != nil {
|
|
return err
|
|
}
|
|
recovered, err := path.receivePayload(context.Background())
|
|
if err != nil || !bytes.Equal(recovered, payload) {
|
|
return errors.New("gateway subprocess work payload mismatch")
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
secondIdle := record("idle-again", func() error {
|
|
time.Sleep(150 * time.Millisecond)
|
|
return nil
|
|
})
|
|
if idle.CPUSeconds > 50*time.Millisecond.Seconds() || secondIdle.CPUSeconds > 50*time.Millisecond.Seconds() {
|
|
t.Fatalf("idle gateway CPU was reported as consumed work: first=%.6fs second=%.6fs", idle.CPUSeconds, secondIdle.CPUSeconds)
|
|
}
|
|
if work.CPUSeconds <= idle.CPUSeconds || work.Count != 500 || work.AllocatedObjects == 0 ||
|
|
work.AllocatedBytes == 0 || work.PeakHeapBytes == 0 || work.PeakGoroutines == 0 {
|
|
t.Fatalf("gateway work resource sample = %#v idle=%#v", work, idle)
|
|
}
|
|
if secondIdle.AllocatedObjects >= work.AllocatedObjects || secondIdle.AllocatedBytes >= work.AllocatedBytes {
|
|
t.Fatalf("successive recording inherited counters: work=%#v second=%#v", work, secondIdle)
|
|
}
|
|
if work.ClockOverhead <= 0 || work.ClockMethod != qualificationClockOverheadMethod {
|
|
t.Fatalf("clock overhead evidence = %#v", work)
|
|
}
|
|
}
|
|
|
|
func qualificationRawMeanLatency(t *testing.T, path string) time.Duration {
|
|
t.Helper()
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer file.Close()
|
|
compressed, err := gzip.NewReader(file)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
raw, err := io.ReadAll(compressed)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := compressed.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var total time.Duration
|
|
var count int
|
|
for _, line := range strings.Split(string(raw), "\n")[1:] {
|
|
fields := strings.Split(line, ",")
|
|
if len(fields) < 5 || fields[4] != "delivered" {
|
|
continue
|
|
}
|
|
sent, sentErr := strconv.ParseInt(fields[1], 10, 64)
|
|
delivered, deliveredErr := strconv.ParseInt(fields[2], 10, 64)
|
|
if sentErr != nil || deliveredErr != nil || delivered < sent {
|
|
t.Fatalf("invalid raw latency row %q", line)
|
|
}
|
|
total += time.Duration(delivered - sent)
|
|
count++
|
|
}
|
|
if count == 0 {
|
|
t.Fatal("no delivered raw latency rows")
|
|
}
|
|
return total / time.Duration(count)
|
|
}
|
|
|
|
func TestQualificationSixImpairmentProfilesTraverseProductionPath(t *testing.T) {
|
|
profiles := qualificationImpairmentProfiles()
|
|
if len(profiles) != 6 {
|
|
t.Fatalf("impairment profile count = %d, want exactly 6", len(profiles))
|
|
}
|
|
for _, profile := range profiles {
|
|
observation, err := runQualificationImpairment(t, profile, qualificationMediaProfiles()[0], 40,
|
|
filepath.Join(t.TempDir(), profile.Name+".csv.gz"))
|
|
if err != nil {
|
|
t.Fatalf("%s: %v", profile.Name, err)
|
|
}
|
|
if observation.Profile != profile.Name || observation.Delivered+observation.Dropped != 40 ||
|
|
observation.RawSamplesSHA256 == "" || observation.MaxQueuePackets > qualificationImpairmentQueuePackets {
|
|
t.Fatalf("%s observation = %#v", profile.Name, observation)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestQualificationLossAndSteppedThroughputBounds(t *testing.T) {
|
|
profiles := qualificationImpairmentProfiles()
|
|
for _, profile := range profiles[3:] {
|
|
directory := t.TempDir()
|
|
observation, err := runQualificationImpairment(t, profile, qualificationMediaProfiles()[0],
|
|
qualificationImpairmentPacketCount, filepath.Join(directory, profile.Name+".csv.gz"))
|
|
if err != nil {
|
|
t.Fatalf("%s: %v", profile.Name, err)
|
|
}
|
|
if observation.ObservedThroughputKbps <= 0 ||
|
|
profile.Name == "constrained" && len(observation.CapacityStepObservations) != 2 {
|
|
t.Fatalf("%s observation = %#v", profile.Name, observation)
|
|
}
|
|
for _, step := range observation.CapacityStepObservations {
|
|
t.Logf("%s %d%%: convergence=%s wire_max_5s=%d wire_cap=%d", profile.Name, step.ReductionPercent, step.Convergence, step.MaximumFiveSecond, step.FiveSecondCap)
|
|
}
|
|
if profile.Name == "constrained" {
|
|
if err := validateQualificationWireBundle(directory, observation, qualificationMediaProfiles()[0], qualificationEvidenceNormative); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestQualificationBundleRetainsIndependentlyRecomputableWireEvidence(t *testing.T) {
|
|
profile := qualificationImpairmentProfiles()[5]
|
|
directory := t.TempDir()
|
|
rawPath := filepath.Join(directory, "impairment-constrained-1080p60-h264.csv.gz")
|
|
observation, err := runQualificationImpairment(t, profile, qualificationMediaProfiles()[0], 40, rawPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := validateQualificationWireBundle(directory, observation, qualificationMediaProfiles()[0], qualificationEvidenceSmoke); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestQualificationV9AggregateOnlyBundleIsRejectedAsIncomplete(t *testing.T) {
|
|
observation := qualificationImpairmentObservation{
|
|
ConfiguredCapacitySteps: []int{25, 50},
|
|
CapacityStepObservations: []qualificationCapacityStep{
|
|
{ReductionPercent: 25, Convergence: 2 * time.Second, MaximumFiveSecond: 1, FiveSecondCap: 1},
|
|
{ReductionPercent: 50, Convergence: 2 * time.Second, MaximumFiveSecond: 1, FiveSecondCap: 1},
|
|
},
|
|
RawSamples: "impairment-constrained-1080p60-h264.csv.gz",
|
|
}
|
|
if err := validateQualificationWireBundle(t.TempDir(), observation, qualificationMediaProfiles()[0], qualificationEvidenceSmoke); err == nil || !strings.Contains(err.Error(), "missing retained public-wire samples") {
|
|
t.Fatalf("v9 aggregate-only bundle rejection = %v", err)
|
|
}
|
|
}
|
|
|
|
type qualificationEvidenceMode uint8
|
|
|
|
const (
|
|
qualificationEvidenceSmoke qualificationEvidenceMode = iota
|
|
qualificationEvidenceNormative
|
|
)
|
|
|
|
const qualificationCanonicalCapacityStepCount = 2
|
|
|
|
var qualificationCanonicalCapacitySteps = [qualificationCanonicalCapacityStepCount]int{25, 50}
|
|
|
|
type qualificationCSVLimits struct {
|
|
compressedBytes int64
|
|
decompressedBytes int64
|
|
headerBytes []int
|
|
fieldBytes []int
|
|
}
|
|
|
|
type qualificationBoundedCSVReader struct {
|
|
file *os.File
|
|
compressed *gzip.Reader
|
|
limited *io.LimitedReader
|
|
reader *csv.Reader
|
|
headerBytes []int
|
|
fieldBytes []int
|
|
first bool
|
|
}
|
|
|
|
func qualificationMaximumWireOffset(media qualificationMediaProfile) time.Duration {
|
|
minimumKbps := qualificationMediaPacerKbps(media, 50)
|
|
spacing := time.Duration(int64(time.Second) * int64(media.PacketBytes) * 8 / (minimumKbps * 1000))
|
|
maximumNetworkDelay := time.Duration(0)
|
|
for _, profile := range qualificationImpairmentProfiles() {
|
|
maximumNetworkDelay = max(maximumNetworkDelay, profile.RTT+profile.Jitter)
|
|
}
|
|
return time.Duration(qualificationImpairmentMaxPackets)*spacing + maximumNetworkDelay + nativeApolloVideoQueueLatency + 10*time.Second
|
|
}
|
|
|
|
func qualificationMaximumFairnessOffset() time.Duration {
|
|
return 60*time.Second + 2*10*time.Second + nativeApolloVideoQueueLatency
|
|
}
|
|
|
|
func qualificationGzipMaximumBytes(decompressedBytes int64) int64 {
|
|
const maximumStoredBlockBytes = 16_383
|
|
return decompressedBytes + ((decompressedBytes+maximumStoredBlockBytes-1)/maximumStoredBlockBytes)*5 + 64
|
|
}
|
|
|
|
func qualificationCSVLimitsFor(header []string, maximumRows int, maximumFieldBytes []int) qualificationCSVLimits {
|
|
rowBytes := int64(len(maximumFieldBytes))
|
|
fieldBytes := make([]int, len(maximumFieldBytes))
|
|
headerBytes := make([]int, len(header))
|
|
for index, size := range maximumFieldBytes {
|
|
headerBytes[index] = len(header[index])
|
|
fieldBytes[index] = size
|
|
rowBytes += int64(max(size, headerBytes[index]))
|
|
}
|
|
headerLineBytes := int64(len(strings.Join(header, ",")) + 1)
|
|
decompressedBytes := headerLineBytes + int64(maximumRows)*rowBytes
|
|
return qualificationCSVLimits{
|
|
compressedBytes: qualificationGzipMaximumBytes(decompressedBytes), decompressedBytes: decompressedBytes,
|
|
headerBytes: headerBytes, fieldBytes: fieldBytes,
|
|
}
|
|
}
|
|
|
|
func qualificationWireCSVLimits(media qualificationMediaProfile, maximumRows int) qualificationCSVLimits {
|
|
maxOffsetBytes := len(strconv.FormatInt(qualificationMaximumWireOffset(media).Nanoseconds(), 10))
|
|
return qualificationCSVLimitsFor(
|
|
[]string{"record_type", "reduction_percent", "transition_after_ns", "received_after_ns", "encoded_bytes"},
|
|
maximumRows,
|
|
[]int{len("transition"), len("100"), maxOffsetBytes, maxOffsetBytes, len(strconv.Itoa(frameV2HeaderSize + frameV2PayloadSize))},
|
|
)
|
|
}
|
|
|
|
func qualificationFairnessCSVLimits() qualificationCSVLimits {
|
|
maxFlowBytes := 0
|
|
for _, flow := range qualificationFairnessFlows() {
|
|
maxFlowBytes = max(maxFlowBytes, len(flow))
|
|
}
|
|
return qualificationCSVLimitsFor(
|
|
[]string{"elapsed_ns", "flow", "bytes"}, qualificationImpairmentMaxPackets,
|
|
[]int{
|
|
len(strconv.FormatInt(qualificationMaximumFairnessOffset().Nanoseconds(), 10)),
|
|
maxFlowBytes,
|
|
len(strconv.Itoa(1000 + frameHeaderSize)),
|
|
},
|
|
)
|
|
}
|
|
|
|
func openQualificationBoundedCSV(path string, limits qualificationCSVLimits, fieldsPerRecord int) (*qualificationBoundedCSVReader, error) {
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if info.Size() > limits.compressedBytes {
|
|
return nil, errors.New("qualification CSV compressed byte bound exceeded")
|
|
}
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
compressed, err := gzip.NewReader(file)
|
|
if err != nil {
|
|
_ = file.Close()
|
|
return nil, err
|
|
}
|
|
limited := &io.LimitedReader{R: compressed, N: limits.decompressedBytes + 1}
|
|
reader := csv.NewReader(limited)
|
|
reader.FieldsPerRecord = fieldsPerRecord
|
|
return &qualificationBoundedCSVReader{
|
|
file: file, compressed: compressed, limited: limited, reader: reader,
|
|
headerBytes: limits.headerBytes, fieldBytes: limits.fieldBytes, first: true,
|
|
}, nil
|
|
}
|
|
|
|
func (reader *qualificationBoundedCSVReader) Read() ([]string, error) {
|
|
row, err := reader.reader.Read()
|
|
if reader.limited.N == 0 {
|
|
return nil, errors.New("qualification CSV decompressed byte bound exceeded")
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
bounds := reader.fieldBytes
|
|
if reader.first {
|
|
reader.first = false
|
|
bounds = reader.headerBytes
|
|
}
|
|
for index, field := range row {
|
|
if index >= len(bounds) || len(field) > bounds[index] {
|
|
return nil, errors.New("qualification CSV field bound exceeded")
|
|
}
|
|
}
|
|
return row, nil
|
|
}
|
|
|
|
func (reader *qualificationBoundedCSVReader) Close() {
|
|
_ = reader.compressed.Close()
|
|
_ = reader.file.Close()
|
|
}
|
|
|
|
func validateQualificationWireBundle(directory string, observation qualificationImpairmentObservation, media qualificationMediaProfile, mode qualificationEvidenceMode) error {
|
|
if mode != qualificationEvidenceSmoke && mode != qualificationEvidenceNormative {
|
|
return errors.New("qualification public-wire validation mode invalid")
|
|
}
|
|
if mode == qualificationEvidenceNormative && observation.Sent != qualificationImpairmentPacketCount {
|
|
return fmt.Errorf("qualification normative sent=%d want=%d", observation.Sent, qualificationImpairmentPacketCount)
|
|
}
|
|
if len(observation.ConfiguredCapacitySteps) != len(qualificationCanonicalCapacitySteps) {
|
|
return errors.New("qualification public-wire capacity steps invalid")
|
|
}
|
|
for index, reduction := range qualificationCanonicalCapacitySteps {
|
|
if observation.ConfiguredCapacitySteps[index] != reduction {
|
|
return errors.New("qualification public-wire capacity steps invalid")
|
|
}
|
|
}
|
|
if observation.RawWireSamples == "" || filepath.Base(observation.RawWireSamples) != observation.RawWireSamples {
|
|
return errors.New("qualification bundle missing retained public-wire samples")
|
|
}
|
|
if observation.RawWireTimebase != qualificationWireTimebase {
|
|
return errors.New("qualification public-wire timebase invalid")
|
|
}
|
|
fragmentsPerUnit := (media.PacketBytes + frameV2PayloadSize - 1) / frameV2PayloadSize
|
|
maximumRows := qualificationImpairmentMaxPackets*fragmentsPerUnit + qualificationCanonicalCapacityStepCount
|
|
limits := qualificationWireCSVLimits(media, maximumRows)
|
|
if observation.RawWireDeliveryRows <= 0 || observation.RawWireTransitionRows != qualificationCanonicalCapacityStepCount ||
|
|
observation.RawWireRows < 1 || observation.RawWireRows > maximumRows ||
|
|
observation.RawWireDeliveryRows > maximumRows-qualificationCanonicalCapacityStepCount ||
|
|
observation.RawWireRows != observation.RawWireDeliveryRows+qualificationCanonicalCapacityStepCount {
|
|
return errors.New("qualification public-wire row counts invalid")
|
|
}
|
|
path := filepath.Join(directory, observation.RawWireSamples)
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.Size() > limits.compressedBytes {
|
|
return errors.New("qualification public-wire compressed byte bound exceeded")
|
|
}
|
|
digest, size, err := qualificationFileSHA256(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if digest != observation.RawWireSamplesSHA256 || size != observation.RawWireSamplesBytes {
|
|
return errors.New("qualification public-wire hash or size mismatch")
|
|
}
|
|
reader, err := openQualificationBoundedCSV(path, limits, 5)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer reader.Close()
|
|
header, err := reader.Read()
|
|
if err != nil || !reflect.DeepEqual(header, []string{"record_type", "reduction_percent", "transition_after_ns", "received_after_ns", "encoded_bytes"}) {
|
|
return errors.New("qualification public-wire schema invalid")
|
|
}
|
|
|
|
epoch := time.Unix(0, 0)
|
|
transitions := make(map[int]time.Duration, qualificationCanonicalCapacityStepCount)
|
|
deliveries := make([]qualificationDeliverySample, 0, observation.RawWireDeliveryRows)
|
|
expectedLengths := make([]int64, 0, fragmentsPerUnit)
|
|
remaining := media.PacketBytes
|
|
for remaining > 0 {
|
|
payloadBytes := min(remaining, frameV2PayloadSize)
|
|
expectedLengths = append(expectedLengths, int64(payloadBytes+frameV2HeaderSize))
|
|
remaining -= payloadBytes
|
|
}
|
|
lastAfter := int64(-1)
|
|
rows, deliveryRows, transitionRows := 0, 0, 0
|
|
for {
|
|
row, readErr := reader.Read()
|
|
if errors.Is(readErr, io.EOF) {
|
|
break
|
|
}
|
|
if readErr != nil {
|
|
return readErr
|
|
}
|
|
rows++
|
|
if rows > maximumRows {
|
|
return errors.New("qualification public-wire row bound exceeded")
|
|
}
|
|
switch row[0] {
|
|
case "transition":
|
|
if row[1] == "" || row[2] == "" || row[3] != "" || row[4] != "" {
|
|
return errors.New("qualification public-wire transition fields invalid")
|
|
}
|
|
reduction, reductionErr := strconv.Atoi(row[1])
|
|
after, afterErr := strconv.ParseInt(row[2], 10, 64)
|
|
if reductionErr != nil || afterErr != nil || after < 0 || time.Duration(after) > qualificationMaximumWireOffset(media) {
|
|
return errors.New("qualification public-wire transition invalid")
|
|
}
|
|
if _, duplicate := transitions[reduction]; duplicate {
|
|
return errors.New("qualification public-wire transition duplicated")
|
|
}
|
|
transitions[reduction] = time.Duration(after)
|
|
transitionRows++
|
|
if after < lastAfter {
|
|
return errors.New("qualification public-wire records not monotonic")
|
|
}
|
|
lastAfter = after
|
|
case "delivery":
|
|
if row[1] != "" || row[2] != "" || row[3] == "" || row[4] == "" {
|
|
return errors.New("qualification public-wire delivery fields invalid")
|
|
}
|
|
after, afterErr := strconv.ParseInt(row[3], 10, 64)
|
|
encodedBytes, bytesErr := strconv.ParseInt(row[4], 10, 64)
|
|
if afterErr != nil || bytesErr != nil || after < 0 || time.Duration(after) > qualificationMaximumWireOffset(media) || encodedBytes <= 0 {
|
|
return errors.New("qualification public-wire delivery invalid")
|
|
}
|
|
if after < lastAfter {
|
|
return errors.New("qualification public-wire records not monotonic")
|
|
}
|
|
if encodedBytes != expectedLengths[deliveryRows%len(expectedLengths)] {
|
|
return errors.New("qualification public-wire encoded length or logical-frame substitution invalid")
|
|
}
|
|
lastAfter = after
|
|
deliveries = append(deliveries, qualificationDeliverySample{At: epoch.Add(time.Duration(after)), Bytes: encodedBytes})
|
|
deliveryRows++
|
|
default:
|
|
return errors.New("qualification public-wire record type invalid")
|
|
}
|
|
}
|
|
if rows != observation.RawWireRows || deliveryRows != observation.RawWireDeliveryRows ||
|
|
transitionRows != observation.RawWireTransitionRows || deliveryRows != observation.QUICDatagramsSent ||
|
|
transitionRows != qualificationCanonicalCapacityStepCount || deliveryRows%len(expectedLengths) != 0 {
|
|
return errors.New("qualification public-wire retained counts mismatch")
|
|
}
|
|
if len(observation.CapacityStepObservations) != qualificationCanonicalCapacityStepCount {
|
|
return errors.New("qualification public-wire capacity summary count mismatch")
|
|
}
|
|
if _, ok := transitions[25]; !ok {
|
|
return errors.New("qualification public-wire transition missing")
|
|
}
|
|
if _, ok := transitions[50]; !ok {
|
|
return errors.New("qualification public-wire transition missing")
|
|
}
|
|
if transitions[25] >= transitions[50] {
|
|
return errors.New("qualification public-wire transition order invalid")
|
|
}
|
|
for index, reduction := range qualificationCanonicalCapacitySteps {
|
|
transition, ok := transitions[reduction]
|
|
if !ok {
|
|
return errors.New("qualification public-wire transition missing")
|
|
}
|
|
step := observation.CapacityStepObservations[index]
|
|
if step.ReductionPercent != reduction || step.TransitionAfter != transition ||
|
|
step.RecomputationSource != observation.RawWireSamples {
|
|
return errors.New("qualification public-wire manifest transition mismatch")
|
|
}
|
|
start := epoch.Add(transition)
|
|
first := sort.Search(len(deliveries), func(index int) bool { return !deliveries[index].At.Before(start) })
|
|
afterStep := deliveries[first:]
|
|
target := qualificationMediaPacerKbps(media, reduction) * 1000 / 8
|
|
convergence := qualificationMeasuredConvergence(afterStep, start, target)
|
|
maximum := qualificationMaximumDeliveryBytes(afterStep, 5*time.Second)
|
|
if step.Convergence != convergence || step.MaximumFiveSecond != maximum || step.FiveSecondCap != target*5 {
|
|
return fmt.Errorf(
|
|
"qualification public-wire capacity summary mismatch reduction=%d convergence=%s/%s maximum=%d/%d cap=%d/%d",
|
|
reduction, step.Convergence, convergence, step.MaximumFiveSecond, maximum, step.FiveSecondCap, target*5,
|
|
)
|
|
}
|
|
if mode == qualificationEvidenceNormative &&
|
|
(step.Convergence > 10*time.Second || step.MaximumFiveSecond > step.FiveSecondCap*105/100) {
|
|
return fmt.Errorf("qualification normative capacity gate failed reduction=%d convergence=%s maximum=%d cap=%d",
|
|
reduction, step.Convergence, step.MaximumFiveSecond, step.FiveSecondCap)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func validateQualificationFairnessBundle(directory string, evidence qualificationFairnessEvidence) error {
|
|
if evidence.RawSamples == "" || filepath.Base(evidence.RawSamples) != evidence.RawSamples {
|
|
return errors.New("qualification fairness raw samples missing")
|
|
}
|
|
path := filepath.Join(directory, evidence.RawSamples)
|
|
limits := qualificationFairnessCSVLimits()
|
|
info, err := os.Stat(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if info.Size() > limits.compressedBytes {
|
|
return errors.New("qualification fairness compressed byte bound exceeded")
|
|
}
|
|
digest, size, err := qualificationFileSHA256(path)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if digest != evidence.RawSamplesSHA256 || size != evidence.RawSamplesBytes {
|
|
return errors.New("qualification fairness hash or size mismatch")
|
|
}
|
|
reader, err := openQualificationBoundedCSV(path, limits, 3)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer reader.Close()
|
|
header, err := reader.Read()
|
|
if err != nil || !reflect.DeepEqual(header, []string{"elapsed_ns", "flow", "bytes"}) {
|
|
return errors.New("qualification fairness schema invalid")
|
|
}
|
|
epoch := time.Unix(0, 0)
|
|
deliveries := make([]qualificationFlowDelivery, 0, qualificationImpairmentMaxPackets)
|
|
flowSet := make(map[string]struct{}, 8)
|
|
allowedFlows := make(map[string]struct{}, 8)
|
|
for _, flow := range qualificationFairnessFlows() {
|
|
allowedFlows[flow] = struct{}{}
|
|
}
|
|
lastElapsed := int64(-1)
|
|
for {
|
|
row, readErr := reader.Read()
|
|
if errors.Is(readErr, io.EOF) {
|
|
break
|
|
}
|
|
if readErr != nil {
|
|
return readErr
|
|
}
|
|
if len(deliveries) >= qualificationImpairmentMaxPackets {
|
|
return errors.New("qualification fairness row bound exceeded")
|
|
}
|
|
elapsed, elapsedErr := strconv.ParseInt(row[0], 10, 64)
|
|
encodedBytes, bytesErr := strconv.ParseInt(row[2], 10, 64)
|
|
_, knownFlow := allowedFlows[row[1]]
|
|
if elapsedErr != nil || bytesErr != nil || elapsed < 0 || time.Duration(elapsed) > qualificationMaximumFairnessOffset() ||
|
|
elapsed < lastElapsed || !knownFlow || encodedBytes != int64(1000+frameHeaderSize) {
|
|
return errors.New("qualification fairness row invalid")
|
|
}
|
|
lastElapsed = elapsed
|
|
flowSet[row[1]] = struct{}{}
|
|
deliveries = append(deliveries, qualificationFlowDelivery{at: epoch.Add(time.Duration(elapsed)), flow: row[1], bytes: encodedBytes})
|
|
}
|
|
if len(deliveries) == 0 || len(flowSet) != 8 || len(evidence.CapacitySteps) != 2 {
|
|
return errors.New("qualification fairness retained counts invalid")
|
|
}
|
|
flows := make([]string, 0, len(flowSet))
|
|
for flow := range flowSet {
|
|
flows = append(flows, flow)
|
|
}
|
|
sort.Strings(flows)
|
|
previousTransition := time.Duration(-1)
|
|
for index, reduction := range []int{25, 50} {
|
|
step := evidence.CapacitySteps[index]
|
|
if step.ReductionPercent != reduction || step.TransitionAfter < 0 || step.TransitionAfter <= previousTransition ||
|
|
step.RecomputationSource != evidence.RawSamples {
|
|
return errors.New("qualification fairness transition manifest invalid")
|
|
}
|
|
previousTransition = step.TransitionAfter
|
|
start := epoch.Add(step.TransitionAfter)
|
|
first := sort.Search(len(deliveries), func(index int) bool { return !deliveries[index].at.Before(start) })
|
|
last := len(deliveries)
|
|
if index+1 < len(evidence.CapacitySteps) {
|
|
next := epoch.Add(evidence.CapacitySteps[index+1].TransitionAfter)
|
|
last = sort.Search(len(deliveries), func(index int) bool { return !deliveries[index].at.Before(next) })
|
|
}
|
|
afterStep := append([]qualificationFlowDelivery(nil), deliveries[first:last]...)
|
|
target := int64(8_000*1000/8) * int64(100-reduction) / 100
|
|
convergence := qualificationPacerConvergence(afterStep, start, flows, target)
|
|
maximum := qualificationMaximumFiveSecondBytes(append([]qualificationFlowDelivery(nil), afterStep...))
|
|
if convergence != 2*time.Second || step.Convergence != convergence ||
|
|
step.MaximumFiveSecond != maximum || step.FiveSecondCap != target*5 ||
|
|
step.MaximumFiveSecond > step.FiveSecondCap*105/100 {
|
|
return fmt.Errorf(
|
|
"qualification fairness capacity summary mismatch reduction=%d transition=%s convergence=%s/%s maximum=%d/%d cap=%d/%d",
|
|
reduction, step.TransitionAfter, step.Convergence, convergence,
|
|
step.MaximumFiveSecond, maximum, step.FiveSecondCap, target*5,
|
|
)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func TestQualificationFairnessTransitionsAreIndependentlyRecomputable(t *testing.T) {
|
|
epoch := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC)
|
|
flows := []string{"one", "two", "three", "four", "five", "six", "seven", "eight"}
|
|
deliveries := make([]qualificationFlowDelivery, 0, 5_520)
|
|
for _, stage := range []struct {
|
|
start, end time.Duration
|
|
packets int
|
|
}{
|
|
{0, 2 * time.Second, 12},
|
|
{2 * time.Second, 5 * time.Second, 9},
|
|
{5 * time.Second, 8 * time.Second, 6},
|
|
} {
|
|
for elapsed := stage.start; elapsed < stage.end; elapsed += 100 * time.Millisecond {
|
|
for packet := 0; packet < stage.packets; packet++ {
|
|
for _, flow := range flows {
|
|
deliveries = append(deliveries, qualificationFlowDelivery{
|
|
at: epoch.Add(elapsed + time.Duration(packet)*time.Millisecond), flow: flow, bytes: int64(1000 + frameHeaderSize),
|
|
})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
directory := t.TempDir()
|
|
path := filepath.Join(directory, "fairness.csv.gz")
|
|
if err := writeQualificationPacerSamples(path, deliveries, epoch); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
evidence := qualificationFairnessEvidence{RawSamples: filepath.Base(path)}
|
|
steps := []struct {
|
|
reduction int
|
|
transition time.Duration
|
|
}{
|
|
{25, 2 * time.Second},
|
|
{50, 5 * time.Second},
|
|
}
|
|
for stepIndex, step := range steps {
|
|
start := epoch.Add(step.transition)
|
|
first := sort.Search(len(deliveries), func(index int) bool { return !deliveries[index].at.Before(start) })
|
|
last := len(deliveries)
|
|
if stepIndex+1 < len(steps) {
|
|
next := epoch.Add(steps[stepIndex+1].transition)
|
|
last = sort.Search(len(deliveries), func(index int) bool { return !deliveries[index].at.Before(next) })
|
|
}
|
|
afterStep := append([]qualificationFlowDelivery(nil), deliveries[first:last]...)
|
|
target := int64(8_000*1000/8) * int64(100-step.reduction) / 100
|
|
evidence.CapacitySteps = append(evidence.CapacitySteps, qualificationCapacityStep{
|
|
ReductionPercent: step.reduction, TransitionAfter: step.transition,
|
|
Convergence: qualificationPacerConvergence(afterStep, start, flows, target),
|
|
MaximumFiveSecond: qualificationMaximumFiveSecondBytes(append([]qualificationFlowDelivery(nil), afterStep...)),
|
|
FiveSecondCap: target * 5, RecomputationSource: filepath.Base(path),
|
|
})
|
|
}
|
|
evidence.RawSamplesSHA256, evidence.RawSamplesBytes, _ = qualificationFileSHA256(path)
|
|
if err := validateQualificationFairnessBundle(directory, evidence); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func readQualificationTestCSV(t *testing.T, path string) [][]string {
|
|
t.Helper()
|
|
file, err := os.Open(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer file.Close()
|
|
compressed, err := gzip.NewReader(file)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer compressed.Close()
|
|
rows, err := csv.NewReader(compressed).ReadAll()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return rows
|
|
}
|
|
|
|
func writeQualificationTestCSV(t *testing.T, path string, rows [][]string) {
|
|
t.Helper()
|
|
file, err := os.Create(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
compressed := gzip.NewWriter(file)
|
|
buffered := bufio.NewWriter(compressed)
|
|
writer := csv.NewWriter(buffered)
|
|
writer.WriteAll(rows)
|
|
if err := writer.Error(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := buffered.Flush(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := compressed.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := file.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func qualificationTestWireBundle(t *testing.T, sent int) (string, qualificationImpairmentObservation, qualificationMediaProfile, []qualificationDeliverySample, map[int]time.Time) {
|
|
t.Helper()
|
|
media := qualificationMediaProfiles()[0]
|
|
epoch := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC)
|
|
transitions := map[int]time.Time{25: epoch.Add(time.Second), 50: epoch.Add(2 * time.Second)}
|
|
deliveries := []qualificationDeliverySample{
|
|
{At: transitions[25], Bytes: 1200},
|
|
{At: transitions[25].Add(time.Nanosecond), Bytes: 25},
|
|
{At: transitions[50], Bytes: 1200},
|
|
{At: transitions[50].Add(time.Nanosecond), Bytes: 25},
|
|
}
|
|
directory := t.TempDir()
|
|
path := filepath.Join(directory, "impairment-constrained-1080p60-h264-wire.csv.gz")
|
|
evidence, err := writeQualificationWireSamples(path, epoch, []int{25, 50}, transitions, deliveries, len(deliveries)+2)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
observation := qualificationImpairmentObservation{
|
|
Sent: sent, ConfiguredCapacitySteps: []int{25, 50}, QUICDatagramsSent: len(deliveries),
|
|
RawWireSamples: evidence.Name, RawWireSamplesSHA256: evidence.SHA256, RawWireSamplesBytes: evidence.Bytes,
|
|
RawWireRows: evidence.Rows, RawWireDeliveryRows: evidence.DeliveryRows,
|
|
RawWireTransitionRows: evidence.TransitionRows, RawWireTimebase: qualificationWireTimebase,
|
|
}
|
|
for _, reduction := range observation.ConfiguredCapacitySteps {
|
|
step := qualificationCapacityStepObservation(deliveries, transitions[reduction], media, reduction)
|
|
step.TransitionAfter = transitions[reduction].Sub(epoch)
|
|
step.RecomputationSource = evidence.Name
|
|
observation.CapacityStepObservations = append(observation.CapacityStepObservations, step)
|
|
}
|
|
return directory, observation, media, deliveries, transitions
|
|
}
|
|
|
|
func TestQualificationCapacityStepRetainsFullPostTransitionTail(t *testing.T) {
|
|
media := qualificationMediaProfiles()[0]
|
|
epoch := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC)
|
|
quarter := epoch.Add(time.Second)
|
|
half := epoch.Add(3 * time.Second)
|
|
deliveries := make([]qualificationDeliverySample, 0, 120)
|
|
for at := quarter; at.Before(epoch.Add(7 * time.Second)); at = at.Add(100 * time.Millisecond) {
|
|
deliveries = append(deliveries,
|
|
qualificationDeliverySample{At: at, Bytes: 1200},
|
|
qualificationDeliverySample{At: at.Add(time.Nanosecond), Bytes: 25},
|
|
)
|
|
}
|
|
got := qualificationCapacityStepObservation(deliveries, quarter, media, 25)
|
|
fullTail := qualificationDeliveriesAfter(deliveries, quarter)
|
|
wantMaximum := qualificationMaximumDeliveryBytes(fullTail, 5*time.Second)
|
|
wantConvergence := qualificationMeasuredConvergence(fullTail, quarter, qualificationMediaPacerKbps(media, 25)*1000/8)
|
|
stageEnd := sort.Search(len(deliveries), func(index int) bool { return !deliveries[index].At.Before(half) })
|
|
stageMaximum := qualificationMaximumDeliveryBytes(deliveries[:stageEnd], 5*time.Second)
|
|
if wantMaximum <= stageMaximum {
|
|
t.Fatalf("test fixture full-tail maximum=%d stage-only=%d", wantMaximum, stageMaximum)
|
|
}
|
|
if got.MaximumFiveSecond != wantMaximum || got.Convergence != wantConvergence {
|
|
t.Fatalf("25%% summary used shortened stage: got maximum=%d convergence=%s, full-tail maximum=%d convergence=%s, stage-only=%d",
|
|
got.MaximumFiveSecond, got.Convergence, wantMaximum, wantConvergence, stageMaximum)
|
|
}
|
|
}
|
|
|
|
func TestQualificationNormativeValidationCannotTrustPersistedSent(t *testing.T) {
|
|
directory, observation, media, _, _ := qualificationTestWireBundle(t, qualificationImpairmentPacketCount-1)
|
|
if err := validateQualificationWireBundle(directory, observation, media, qualificationEvidenceNormative); err == nil {
|
|
t.Fatal("normative validation accepted persisted sent=9999 and 11-second convergence sentinel")
|
|
}
|
|
}
|
|
|
|
func TestQualificationWireBundleRejectsReversedCapacitySteps(t *testing.T) {
|
|
directory, observation, media, _, _ := qualificationTestWireBundle(t, 40)
|
|
observation.ConfiguredCapacitySteps = []int{50, 25}
|
|
observation.CapacityStepObservations[0], observation.CapacityStepObservations[1] =
|
|
observation.CapacityStepObservations[1], observation.CapacityStepObservations[0]
|
|
if err := validateQualificationWireBundle(directory, observation, media, qualificationEvidenceSmoke); err == nil {
|
|
t.Fatal("qualification wire bundle accepted reversed capacity steps")
|
|
}
|
|
}
|
|
|
|
func TestQualificationWireBundleRejectsSwappedTransitionChronology(t *testing.T) {
|
|
directory, observation, media, deliveries, transitions := qualificationTestWireBundle(t, 40)
|
|
rows := readQualificationTestCSV(t, filepath.Join(directory, observation.RawWireSamples))
|
|
for _, row := range rows[1:] {
|
|
if row[0] != "transition" {
|
|
continue
|
|
}
|
|
switch row[1] {
|
|
case "25":
|
|
row[1] = "50"
|
|
case "50":
|
|
row[1] = "25"
|
|
}
|
|
}
|
|
mutatedDirectory := t.TempDir()
|
|
path := filepath.Join(mutatedDirectory, observation.RawWireSamples)
|
|
writeQualificationTestCSV(t, path, rows)
|
|
observation.RawWireSamplesSHA256, observation.RawWireSamplesBytes, _ = qualificationFileSHA256(path)
|
|
transitions[25], transitions[50] = transitions[50], transitions[25]
|
|
observation.CapacityStepObservations = nil
|
|
epoch := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC)
|
|
for _, reduction := range observation.ConfiguredCapacitySteps {
|
|
step := qualificationCapacityStepObservation(deliveries, transitions[reduction], media, reduction)
|
|
step.TransitionAfter = transitions[reduction].Sub(epoch)
|
|
step.RecomputationSource = observation.RawWireSamples
|
|
observation.CapacityStepObservations = append(observation.CapacityStepObservations, step)
|
|
}
|
|
if err := validateQualificationWireBundle(mutatedDirectory, observation, media, qualificationEvidenceSmoke); err == nil ||
|
|
!strings.Contains(err.Error(), "transition order invalid") {
|
|
t.Fatalf("qualification swapped transition chronology rejection = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestQualificationWireBundleRejectsSelfConsistentNoncanonicalSteps(t *testing.T) {
|
|
directory, observation, media, deliveries, transitions := qualificationTestWireBundle(t, 40)
|
|
rows := readQualificationTestCSV(t, filepath.Join(directory, observation.RawWireSamples))
|
|
for _, row := range rows[1:] {
|
|
if row[0] != "transition" {
|
|
continue
|
|
}
|
|
switch row[1] {
|
|
case "25":
|
|
row[1] = "10"
|
|
case "50":
|
|
row[1] = "20"
|
|
}
|
|
}
|
|
mutatedDirectory := t.TempDir()
|
|
path := filepath.Join(mutatedDirectory, observation.RawWireSamples)
|
|
writeQualificationTestCSV(t, path, rows)
|
|
observation.RawWireSamplesSHA256, observation.RawWireSamplesBytes, _ = qualificationFileSHA256(path)
|
|
observation.ConfiguredCapacitySteps = []int{10, 20}
|
|
observation.CapacityStepObservations = nil
|
|
for index, reduction := range observation.ConfiguredCapacitySteps {
|
|
transition := transitions[[]int{25, 50}[index]]
|
|
step := qualificationCapacityStepObservation(deliveries, transition, media, reduction)
|
|
step.TransitionAfter = transition.Sub(time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC))
|
|
step.RecomputationSource = observation.RawWireSamples
|
|
observation.CapacityStepObservations = append(observation.CapacityStepObservations, step)
|
|
}
|
|
if err := validateQualificationWireBundle(mutatedDirectory, observation, media, qualificationEvidenceSmoke); err == nil {
|
|
t.Fatal("qualification wire bundle accepted self-consistent noncanonical capacity steps")
|
|
}
|
|
}
|
|
|
|
func TestQualificationWireBundleRejectsNegativeDeliveryRowsWithoutPanic(t *testing.T) {
|
|
directory, observation, media, _, _ := qualificationTestWireBundle(t, 40)
|
|
observation.RawWireRows = 1
|
|
observation.RawWireDeliveryRows = -1
|
|
observation.RawWireTransitionRows = 2
|
|
defer func() {
|
|
if recovered := recover(); recovered != nil {
|
|
t.Fatalf("qualification wire bundle panicked for negative delivery rows: %v", recovered)
|
|
}
|
|
}()
|
|
if err := validateQualificationWireBundle(directory, observation, media, qualificationEvidenceSmoke); err == nil {
|
|
t.Fatal("qualification wire bundle accepted negative delivery rows")
|
|
}
|
|
}
|
|
|
|
func TestQualificationNormativeRejectsExactSentConvergenceFailure(t *testing.T) {
|
|
directory, observation, media, _, _ := qualificationTestWireBundle(t, qualificationImpairmentPacketCount)
|
|
err := validateQualificationWireBundle(directory, observation, media, qualificationEvidenceNormative)
|
|
if err == nil || !strings.Contains(err.Error(), "normative capacity gate failed") || strings.Contains(err.Error(), "sent=") {
|
|
t.Fatalf("normative exact-sent convergence rejection = %v", err)
|
|
}
|
|
}
|
|
|
|
func TestQualificationEvidenceReadersRejectOversizedFieldsBeforeParsing(t *testing.T) {
|
|
t.Run("wire", func(t *testing.T) {
|
|
directory, observation, media, _, _ := qualificationTestWireBundle(t, 40)
|
|
rows := readQualificationTestCSV(t, filepath.Join(directory, observation.RawWireSamples))
|
|
for index := 1; index < len(rows); index++ {
|
|
if rows[index][0] == "delivery" {
|
|
rows[index][3] = strings.Repeat("9", 4096)
|
|
break
|
|
}
|
|
}
|
|
mutatedDirectory := t.TempDir()
|
|
path := filepath.Join(mutatedDirectory, observation.RawWireSamples)
|
|
writeQualificationTestCSV(t, path, rows)
|
|
observation.RawWireSamplesSHA256, observation.RawWireSamplesBytes, _ = qualificationFileSHA256(path)
|
|
if err := validateQualificationWireBundle(mutatedDirectory, observation, media, qualificationEvidenceSmoke); err == nil || !strings.Contains(err.Error(), "field bound") {
|
|
t.Fatalf("wire oversized field rejection = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("fairness", func(t *testing.T) {
|
|
epoch := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC)
|
|
deliveries := make([]qualificationFlowDelivery, 0, 8)
|
|
for _, flow := range []string{"one", "two", "three", "four", "five", "six", "seven", "eight"} {
|
|
deliveries = append(deliveries, qualificationFlowDelivery{at: epoch, flow: flow, bytes: 1023})
|
|
}
|
|
directory := t.TempDir()
|
|
path := filepath.Join(directory, "fairness.csv.gz")
|
|
if err := writeQualificationPacerSamples(path, deliveries, epoch); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
rows := readQualificationTestCSV(t, path)
|
|
rows[1][1] = strings.Repeat("f", 4096)
|
|
mutatedDirectory := t.TempDir()
|
|
mutatedPath := filepath.Join(mutatedDirectory, "fairness.csv.gz")
|
|
writeQualificationTestCSV(t, mutatedPath, rows)
|
|
evidence := qualificationFairnessEvidence{RawSamples: "fairness.csv.gz", CapacitySteps: []qualificationCapacityStep{{}, {}}}
|
|
evidence.RawSamplesSHA256, evidence.RawSamplesBytes, _ = qualificationFileSHA256(mutatedPath)
|
|
if err := validateQualificationFairnessBundle(mutatedDirectory, evidence); err == nil || !strings.Contains(err.Error(), "field bound") {
|
|
t.Fatalf("fairness oversized field rejection = %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestQualificationBoundedCSVReaderRejectsCompressedAndDecompressedOverflow(t *testing.T) {
|
|
t.Run("compressed", func(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "oversized.csv.gz")
|
|
file, err := os.Create(path)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := file.Truncate(33); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := file.Close(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
_, err = openQualificationBoundedCSV(path, qualificationCSVLimits{compressedBytes: 32, decompressedBytes: 32, headerBytes: []int{5}, fieldBytes: []int{64}}, 1)
|
|
if err == nil || !strings.Contains(err.Error(), "compressed byte bound") {
|
|
t.Fatalf("compressed overflow rejection = %v", err)
|
|
}
|
|
})
|
|
|
|
t.Run("decompressed", func(t *testing.T) {
|
|
path := filepath.Join(t.TempDir(), "oversized.csv.gz")
|
|
writeQualificationTestCSV(t, path, [][]string{{"value"}, {strings.Repeat("x", 64)}})
|
|
reader, err := openQualificationBoundedCSV(path, qualificationCSVLimits{
|
|
compressedBytes: 1024, decompressedBytes: 32, headerBytes: []int{5}, fieldBytes: []int{64},
|
|
}, 1)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer reader.Close()
|
|
if _, err := reader.Read(); err == nil || !strings.Contains(err.Error(), "decompressed byte bound") {
|
|
t.Fatalf("decompressed overflow rejection = %v", err)
|
|
}
|
|
})
|
|
}
|
|
|
|
func TestQualificationWireEvidenceRejectsIncompleteOrMalformedBundles(t *testing.T) {
|
|
profile := qualificationImpairmentProfiles()[5]
|
|
media := qualificationMediaProfiles()[0]
|
|
sourceDirectory := t.TempDir()
|
|
observation, err := runQualificationImpairment(t, profile, media, 40,
|
|
filepath.Join(sourceDirectory, "impairment-constrained-1080p60-h264.csv.gz"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := validateQualificationWireBundle(sourceDirectory, observation, media, qualificationEvidenceSmoke); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
wireRows := readQualificationTestCSV(t, filepath.Join(sourceDirectory, observation.RawWireSamples))
|
|
|
|
metadataCases := []struct {
|
|
name string
|
|
mutate func(*qualificationImpairmentObservation)
|
|
}{
|
|
{"missing-artifact", func(value *qualificationImpairmentObservation) { value.RawWireSamples = "" }},
|
|
{"wrong-hash", func(value *qualificationImpairmentObservation) { value.RawWireSamplesSHA256 = strings.Repeat("0", 64) }},
|
|
{"wrong-size", func(value *qualificationImpairmentObservation) { value.RawWireSamplesBytes++ }},
|
|
{"wrong-count", func(value *qualificationImpairmentObservation) { value.RawWireRows++ }},
|
|
{"manifest-mismatch", func(value *qualificationImpairmentObservation) { value.CapacityStepObservations[0].Convergence++ }},
|
|
}
|
|
for _, testCase := range metadataCases {
|
|
t.Run(testCase.name, func(t *testing.T) {
|
|
mutated := observation
|
|
mutated.CapacityStepObservations = append([]qualificationCapacityStep(nil), observation.CapacityStepObservations...)
|
|
testCase.mutate(&mutated)
|
|
if err := validateQualificationWireBundle(sourceDirectory, mutated, media, qualificationEvidenceSmoke); err == nil {
|
|
t.Fatal("malformed qualification wire bundle accepted")
|
|
}
|
|
})
|
|
}
|
|
|
|
type rowMutation struct {
|
|
name string
|
|
mutate func([][]string, *qualificationImpairmentObservation) [][]string
|
|
}
|
|
rowCases := []rowMutation{
|
|
{"duplicate-transition", func(rows [][]string, value *qualificationImpairmentObservation) [][]string {
|
|
for index := 1; index < len(rows); index++ {
|
|
if rows[index][0] == "transition" {
|
|
duplicate := append([]string(nil), rows[index]...)
|
|
rows = append(rows, nil)
|
|
copy(rows[index+1:], rows[index:])
|
|
rows[index] = duplicate
|
|
value.RawWireRows++
|
|
value.RawWireTransitionRows++
|
|
break
|
|
}
|
|
}
|
|
return rows
|
|
}},
|
|
{"missing-transition", func(rows [][]string, value *qualificationImpairmentObservation) [][]string {
|
|
for index := 1; index < len(rows); index++ {
|
|
if rows[index][0] == "transition" {
|
|
copy(rows[index:], rows[index+1:])
|
|
rows[len(rows)-1] = nil
|
|
value.RawWireRows--
|
|
value.RawWireTransitionRows--
|
|
break
|
|
}
|
|
}
|
|
return rows
|
|
}},
|
|
{"delivery-before-epoch", func(rows [][]string, _ *qualificationImpairmentObservation) [][]string {
|
|
for index := 1; index < len(rows); index++ {
|
|
if rows[index][0] == "delivery" {
|
|
rows[index][3] = "-1"
|
|
break
|
|
}
|
|
}
|
|
return rows
|
|
}},
|
|
{"invalid-encoded-length", func(rows [][]string, _ *qualificationImpairmentObservation) [][]string {
|
|
for index := 1; index < len(rows); index++ {
|
|
if rows[index][0] == "delivery" {
|
|
rows[index][4] = "0"
|
|
break
|
|
}
|
|
}
|
|
return rows
|
|
}},
|
|
{"logical-frame-substitution", func(rows [][]string, _ *qualificationImpairmentObservation) [][]string {
|
|
for index := 1; index < len(rows); index++ {
|
|
if rows[index][0] == "delivery" {
|
|
rows[index][4] = "1179"
|
|
break
|
|
}
|
|
}
|
|
return rows
|
|
}},
|
|
{"invalid-empty-fields", func(rows [][]string, _ *qualificationImpairmentObservation) [][]string {
|
|
for index := 1; index < len(rows); index++ {
|
|
if rows[index][0] == "delivery" {
|
|
rows[index][1] = "25"
|
|
break
|
|
}
|
|
}
|
|
return rows
|
|
}},
|
|
{"nonmonotonic", func(rows [][]string, _ *qualificationImpairmentObservation) [][]string {
|
|
first := int64(-1)
|
|
for index := 1; index < len(rows); index++ {
|
|
if rows[index][0] != "delivery" {
|
|
continue
|
|
}
|
|
if first < 0 {
|
|
first, _ = strconv.ParseInt(rows[index][3], 10, 64)
|
|
continue
|
|
}
|
|
rows[index][3] = strconv.FormatInt(max(first-1, 0), 10)
|
|
break
|
|
}
|
|
return rows
|
|
}},
|
|
}
|
|
for _, testCase := range rowCases {
|
|
t.Run(testCase.name, func(t *testing.T) {
|
|
rows := make([][]string, len(wireRows))
|
|
for index := range wireRows {
|
|
rows[index] = append([]string(nil), wireRows[index]...)
|
|
}
|
|
mutated := observation
|
|
mutated.CapacityStepObservations = append([]qualificationCapacityStep(nil), observation.CapacityStepObservations...)
|
|
rows = testCase.mutate(rows, &mutated)
|
|
trimmed := rows[:0]
|
|
for _, row := range rows {
|
|
if row != nil {
|
|
trimmed = append(trimmed, row)
|
|
}
|
|
}
|
|
directory := t.TempDir()
|
|
path := filepath.Join(directory, mutated.RawWireSamples)
|
|
writeQualificationTestCSV(t, path, trimmed)
|
|
mutated.RawWireSamplesSHA256, mutated.RawWireSamplesBytes, _ = qualificationFileSHA256(path)
|
|
if err := validateQualificationWireBundle(directory, mutated, media, qualificationEvidenceSmoke); err == nil {
|
|
t.Fatal("malformed qualification wire bundle accepted")
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestQualificationCapacityStepsMeasurePublicWireDatagrams(t *testing.T) {
|
|
const packetCount = qualificationImpairmentPacketCount
|
|
profile := qualificationMediaProfiles()[0]
|
|
started := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC)
|
|
spacing := time.Duration(int64(time.Second) * int64(profile.PacketBytes) * 8 / (profile.BitrateKbps * 1000))
|
|
stepAt := map[int]time.Time{
|
|
25: started.Add(time.Duration(packetCount/3) * spacing),
|
|
50: started.Add(time.Duration(packetCount*2/3) * spacing),
|
|
}
|
|
if phase := stepAt[50].Sub(stepAt[25]); phase != 1_571_842_800*time.Nanosecond {
|
|
t.Fatalf("25%% phase = %s, want exact source-shaped transition", phase)
|
|
}
|
|
|
|
pacer := newFairPacer(qualificationMediaPacerKbps(profile, 0))
|
|
const flow = "qualification-wire-flow"
|
|
now := started
|
|
stalled := false
|
|
quarterApplied := false
|
|
halfApplied := false
|
|
debtBeforeQuarter := time.Duration(0)
|
|
logicalDeliveries := make([]qualificationDeliverySample, 0, packetCount)
|
|
type wireSample struct {
|
|
reserved time.Time
|
|
delivery qualificationDeliverySample
|
|
}
|
|
wireSamples := make([]wireSample, 0, packetCount*2)
|
|
for index := 0; index < packetCount; index++ {
|
|
sourceAt := started.Add(time.Duration(index) * spacing)
|
|
if now.Before(sourceAt) {
|
|
now = sourceAt
|
|
}
|
|
if !stalled && !sourceAt.Before(stepAt[25].Add(-150*time.Millisecond)) {
|
|
now = now.Add(42 * time.Millisecond)
|
|
stalled = true
|
|
}
|
|
for _, encodedBytes := range []int{1200, 25} {
|
|
if !quarterApplied && !now.Before(stepAt[25]) {
|
|
pacer.mu.Lock()
|
|
debtBeforeQuarter = pacer.flows[flow].debt
|
|
pacer.mu.Unlock()
|
|
pacer.setKbps(qualificationMediaPacerKbps(profile, 25))
|
|
quarterApplied = true
|
|
}
|
|
if !halfApplied && !now.Before(stepAt[50]) {
|
|
pacer.setKbps(qualificationMediaPacerKbps(profile, 50))
|
|
halfApplied = true
|
|
}
|
|
reserved := pacer.reserveAt(now, flow, encodedBytes)
|
|
if reserved.After(now) {
|
|
now = reserved
|
|
}
|
|
wireSamples = append(wireSamples, wireSample{
|
|
reserved: reserved,
|
|
delivery: qualificationDeliverySample{At: now, Bytes: int64(encodedBytes)},
|
|
})
|
|
}
|
|
logicalDeliveries = append(logicalDeliveries, qualificationDeliverySample{At: now, Bytes: 1225})
|
|
}
|
|
if !quarterApplied || !halfApplied || debtBeforeQuarter <= 0 || debtBeforeQuarter > nativeApolloVideoQueueLatency-fairPacerMaximumCatchup {
|
|
t.Fatalf("capacity transition state quarter=%t half=%t debt=%s", quarterApplied, halfApplied, debtBeforeQuarter)
|
|
}
|
|
pacer.mu.Lock()
|
|
remainingDebt := pacer.flows[flow].debt
|
|
pacer.mu.Unlock()
|
|
if remainingDebt != 0 {
|
|
t.Fatalf("remaining debt = %s, want zero", remainingDebt)
|
|
}
|
|
t.Logf("wire model: datagrams=%d debt_before_25=%s remaining_debt=%s", len(wireSamples), debtBeforeQuarter, remainingDebt)
|
|
wireDeliveries := make([]qualificationDeliverySample, len(wireSamples))
|
|
for index, sample := range wireSamples {
|
|
if sample.delivery.At.Before(sample.reserved) {
|
|
t.Fatalf("wire datagram %d delivered at %s before reservation %s", index, sample.delivery.At, sample.reserved)
|
|
}
|
|
wireDeliveries[index] = sample.delivery
|
|
}
|
|
directory := t.TempDir()
|
|
wirePath := filepath.Join(directory, "impairment-constrained-1080p60-h264-wire.csv.gz")
|
|
bundle := qualificationImpairmentObservation{
|
|
Sent: packetCount,
|
|
ConfiguredCapacitySteps: []int{25, 50},
|
|
QUICDatagramsSent: len(wireDeliveries),
|
|
RawWireTimebase: qualificationWireTimebase,
|
|
}
|
|
|
|
for _, reduction := range []int{25, 50} {
|
|
wireBytesPerSecond := qualificationMediaPacerKbps(profile, reduction) * 1000 / 8
|
|
wireAfterStep := qualificationDeliveriesAfter(wireDeliveries, stepAt[reduction])
|
|
maximum := qualificationMaximumDeliveryBytes(wireAfterStep, 5*time.Second)
|
|
if maximum > wireBytesPerSecond*5*105/100 {
|
|
t.Fatalf("%d%% wire five-second maximum = %d, cap = %d", reduction, maximum, wireBytesPerSecond*5*105/100)
|
|
}
|
|
payloadBytesPerSecond := profile.BitrateKbps * int64(100-reduction) * 1000 / 100 / 8
|
|
legacy := qualificationMeasuredConvergence(
|
|
qualificationDeliveriesAfter(logicalDeliveries, stepAt[reduction]), stepAt[reduction], payloadBytesPerSecond,
|
|
)
|
|
if reduction == 25 && legacy != 11*time.Second {
|
|
t.Fatalf("25%% complete-frame classifier convergence = %s, want 11s sentinel reproduction", legacy)
|
|
}
|
|
stepObservation := qualificationCapacityStepObservation(wireDeliveries, stepAt[reduction], profile, reduction)
|
|
stepObservation.TransitionAfter = stepAt[reduction].Sub(started)
|
|
stepObservation.RecomputationSource = filepath.Base(wirePath)
|
|
if stepObservation.Convergence > 10*time.Second {
|
|
for offset := time.Duration(0); offset < 2*time.Second; offset += 250 * time.Millisecond {
|
|
var bucket int64
|
|
for _, delivery := range wireAfterStep {
|
|
if !delivery.At.Before(stepAt[reduction].Add(offset)) && delivery.At.Before(stepAt[reduction].Add(offset+250*time.Millisecond)) {
|
|
bucket += delivery.Bytes
|
|
}
|
|
}
|
|
t.Logf("%d%% wire bucket %s = %d bytes (%d B/s)", reduction, offset, bucket, bucket*4)
|
|
}
|
|
t.Fatalf("%d%% wire convergence = %s sentinel with wire maximum %d under cap %d", reduction, stepObservation.Convergence, maximum, wireBytesPerSecond*5*105/100)
|
|
}
|
|
t.Logf("%d%%: payload_target=%d wire_target=%d legacy=%s wire_convergence=%s wire_max_5s=%d wire_cap_105=%d",
|
|
reduction, payloadBytesPerSecond, wireBytesPerSecond, legacy, stepObservation.Convergence, maximum, wireBytesPerSecond*5*105/100)
|
|
bundle.CapacityStepObservations = append(bundle.CapacityStepObservations, stepObservation)
|
|
}
|
|
wireEvidence, err := writeQualificationWireSamples(wirePath, started, []int{25, 50}, stepAt, wireDeliveries, len(wireDeliveries)+2)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
bundle.RawWireSamples = wireEvidence.Name
|
|
bundle.RawWireSamplesSHA256 = wireEvidence.SHA256
|
|
bundle.RawWireSamplesBytes = wireEvidence.Bytes
|
|
bundle.RawWireRows = wireEvidence.Rows
|
|
bundle.RawWireDeliveryRows = wireEvidence.DeliveryRows
|
|
bundle.RawWireTransitionRows = wireEvidence.TransitionRows
|
|
if err := validateQualificationWireBundle(directory, bundle, profile, qualificationEvidenceNormative); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestQualificationUsesPublicQUICAndProductionPacer(t *testing.T) {
|
|
qualificationTraverseProfiles(t, qualificationMediaProfiles())
|
|
directory := t.TempDir()
|
|
evidence, err := qualificationPacerEvidence(t, filepath.Join(directory, "fairness.csv.gz"), 2*time.Second, 4*time.Second)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(evidence.PerFlowBytes) != 8 || len(evidence.CapacitySteps) != 2 ||
|
|
len(evidence.Series) != 10 || evidence.RawSamplesSHA256 == "" || evidence.JainIndex < 0.99 {
|
|
t.Fatalf("pacer evidence = %#v", evidence)
|
|
}
|
|
for _, step := range evidence.CapacitySteps {
|
|
if step.Convergence > 10*time.Second || step.MaximumFiveSecond > step.FiveSecondCap*105/100 {
|
|
t.Fatalf("capacity step = %#v", step)
|
|
}
|
|
}
|
|
if err := validateQualificationFairnessBundle(directory, evidence); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestQualificationSmokeTraversesNativeApolloRecoveryQueuePacerAndQUIC(t *testing.T) {
|
|
trace := qualificationProductionPathSmoke(t, qualificationMediaProfiles()[0])
|
|
if !trace.ApolloRecovered || !trace.ProductionQueue || !trace.ProductionPacer ||
|
|
!trace.VerseQUIC || !trace.PayloadPreserved {
|
|
t.Fatalf("qualification production-path trace = %#v", trace)
|
|
}
|
|
}
|
|
|
|
func TestQualificationCarriesCompleteLargeFramesThroughPublicPath(t *testing.T) {
|
|
profile := qualificationMediaProfiles()[2]
|
|
path := newQualificationPath(t, profile, 200000)
|
|
defer path.Close()
|
|
|
|
for _, size := range []int{24 * 1024, 96 * 1024, 384 * 1024} {
|
|
payload := make([]byte, size)
|
|
for index := range payload {
|
|
payload[index] = byte(index*31 + size)
|
|
}
|
|
trace, _, err := path.traverse(t, payload)
|
|
if err != nil {
|
|
t.Fatalf("frame bytes=%d: %v", size, err)
|
|
}
|
|
if !trace.NativeUDPIngress || !trace.ApolloRecovered || !trace.ProductionQueue ||
|
|
!trace.ProductionMediaLoop || !trace.ProductionPacer || !trace.VerseQUIC ||
|
|
!trace.PublicClientDecode || !trace.PayloadPreserved {
|
|
t.Fatalf("frame bytes=%d skipped path: %#v", size, trace)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestQualificationPacerRepaysThreeMediaLoopStallsThroughPublicPath(t *testing.T) {
|
|
profile := qualificationMediaProfiles()[0]
|
|
const totalFrames = 480
|
|
barriers := []int64{30, 180, 330}
|
|
type stall struct {
|
|
public chan struct{}
|
|
blocked chan uint64
|
|
release chan struct{}
|
|
}
|
|
stalls := make([]stall, len(barriers))
|
|
for index := range stalls {
|
|
stalls[index] = stall{public: make(chan struct{}), blocked: make(chan uint64, 1), release: make(chan struct{})}
|
|
}
|
|
|
|
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
|
defer cancel()
|
|
var path *qualificationPath
|
|
var completed int64
|
|
var observationMu sync.Mutex
|
|
var maximumQueueDelay time.Duration
|
|
observer := func(observation mediaTimingObservation) {
|
|
observationMu.Lock()
|
|
maximumQueueDelay = max(maximumQueueDelay, observation.QueueDelay)
|
|
completed++
|
|
current := completed
|
|
observationMu.Unlock()
|
|
for index, barrier := range barriers {
|
|
if current != barrier {
|
|
continue
|
|
}
|
|
select {
|
|
case <-stalls[index].public:
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
baseline := path.session.mediaRecovered.Load()
|
|
select {
|
|
case stalls[index].blocked <- baseline:
|
|
case <-ctx.Done():
|
|
return
|
|
}
|
|
select {
|
|
case <-stalls[index].release:
|
|
case <-ctx.Done():
|
|
}
|
|
return
|
|
}
|
|
}
|
|
path = newQualificationPathWithNativeBackendAndObserver(
|
|
t, profile, qualificationFramePacerKbps(profile), nil, NewNativeApolloBackend(), observer,
|
|
)
|
|
defer path.Close()
|
|
|
|
spacing := time.Second / time.Duration(profile.FPS)
|
|
started := time.Now().Add(10 * time.Millisecond)
|
|
producerDone := make(chan error, 1)
|
|
go func() {
|
|
var nextRelease time.Time
|
|
for index := int64(0); index < totalFrames; index++ {
|
|
release := qualificationBoundedRelease(
|
|
started.Add(time.Duration(index)*spacing), nextRelease, time.Now(), spacing,
|
|
)
|
|
if err := qualificationWaitContext(ctx, release); err != nil {
|
|
producerDone <- err
|
|
return
|
|
}
|
|
nextRelease = release.Add(spacing)
|
|
if _, err := path.emit(t, qualificationFramePayload(profile, index)); err != nil {
|
|
producerDone <- err
|
|
return
|
|
}
|
|
}
|
|
producerDone <- nil
|
|
}()
|
|
|
|
var expectedVersePackets uint64
|
|
receiverDone := make(chan error, 1)
|
|
go func() {
|
|
for index := int64(0); index < totalFrames; index++ {
|
|
payload, err := path.receivePayload(ctx)
|
|
if err != nil {
|
|
receiverDone <- err
|
|
return
|
|
}
|
|
expected := qualificationFramePayload(profile, index)
|
|
if !bytes.Equal(payload, expected) {
|
|
receiverDone <- fmt.Errorf("public payload changed at frame %d", index)
|
|
return
|
|
}
|
|
fragments := (len(payload) + frameV2PayloadSize - 1) / frameV2PayloadSize
|
|
expectedVersePackets += uint64(fragments)
|
|
for barrierIndex, barrier := range barriers {
|
|
if index+1 == barrier {
|
|
close(stalls[barrierIndex].public)
|
|
}
|
|
}
|
|
}
|
|
receiverDone <- nil
|
|
}()
|
|
|
|
for index := range stalls {
|
|
var baseline uint64
|
|
select {
|
|
case baseline = <-stalls[index].blocked:
|
|
case <-ctx.Done():
|
|
t.Fatalf("media loop did not block at public frame %d: %v", barriers[index], ctx.Err())
|
|
}
|
|
target := baseline + 6
|
|
for path.session.mediaRecovered.Load() < target {
|
|
select {
|
|
case <-ctx.Done():
|
|
t.Fatalf("stall %d recovered %d, want at least %d: %v", index, path.session.mediaRecovered.Load(), target, ctx.Err())
|
|
default:
|
|
runtime.Gosched()
|
|
}
|
|
}
|
|
close(stalls[index].release)
|
|
}
|
|
if err := <-producerDone; err != nil {
|
|
t.Fatalf("source fixture: %v", err)
|
|
}
|
|
if err := <-receiverDone; err != nil {
|
|
t.Fatalf("independent client: %v; expected=%d sent=%d source=%d ingress=%d recovered=%d enqueued=%d provider_drops=%d gateway_drops=%d queue_count=%d queue_bytes=%d",
|
|
err, path.expectedSourceUDP.Load(), path.fixture.sentPackets.Load(), path.sourceUDP.Load(),
|
|
path.session.mediaIngress.Load(), path.session.mediaRecovered.Load(), path.session.mediaEnqueued.Load(),
|
|
path.session.Telemetry().MediaDrops, path.server.Metrics().MediaDrops,
|
|
path.session.mediaQueueMaximum.Load(), path.session.mediaQueueMaximumBytes.Load())
|
|
}
|
|
for {
|
|
observationMu.Lock()
|
|
observed := completed
|
|
observationMu.Unlock()
|
|
if observed == totalFrames {
|
|
break
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
t.Fatalf("media observer completed %d frames, want %d: %v", observed, totalFrames, ctx.Err())
|
|
default:
|
|
runtime.Gosched()
|
|
}
|
|
}
|
|
|
|
if source := path.expectedSourceUDP.Load(); source != path.fixture.sentPackets.Load() || source != path.sourceUDP.Load() || source != path.session.mediaIngress.Load() {
|
|
t.Fatalf("source accounting expected=%d sent=%d source=%d ingress=%d", source, path.fixture.sentPackets.Load(), path.sourceUDP.Load(), path.session.mediaIngress.Load())
|
|
}
|
|
metrics := path.server.Metrics()
|
|
observationMu.Lock()
|
|
completedFrames := completed
|
|
queueDelay := maximumQueueDelay
|
|
observationMu.Unlock()
|
|
if path.backend.setups.Load() != 1 || path.backend.opens.Load() != 1 || completedFrames != totalFrames ||
|
|
path.session.mediaRecovered.Load() != totalFrames || path.session.mediaEnqueued.Load() != totalFrames ||
|
|
metrics.ProcessingSamples != totalFrames || metrics.MediaPackets != expectedVersePackets ||
|
|
path.server.pacer.reservations.Load() != expectedVersePackets ||
|
|
path.session.Telemetry().MediaDrops != 0 || path.server.Metrics().MediaDrops != 0 {
|
|
t.Fatalf("media accounting setup=%d open=%d completed=%d recovered=%d enqueued=%d processing=%d media=%d pacer=%d provider_drops=%d gateway_drops=%d",
|
|
path.backend.setups.Load(), path.backend.opens.Load(), completedFrames,
|
|
path.session.mediaRecovered.Load(), path.session.mediaEnqueued.Load(), metrics.ProcessingSamples,
|
|
metrics.MediaPackets, path.server.pacer.reservations.Load(), path.session.Telemetry().MediaDrops, metrics.MediaDrops)
|
|
}
|
|
if path.session.mediaQueueMaximum.Load() > nativeApolloVideoQueuePackets ||
|
|
path.session.mediaQueueMaximumBytes.Load() > nativeApolloVideoQueueBytes || queueDelay > nativeApolloVideoQueueLatency {
|
|
t.Fatalf("queue bounds count=%d bytes=%d residence=%s", path.session.mediaQueueMaximum.Load(), path.session.mediaQueueMaximumBytes.Load(), queueDelay)
|
|
}
|
|
t.Logf("three-stall public path: frames=%d source=%d ingress=%d recovered=%d enqueued=%d drops=%d/%d queue=%d/%d residence=%s verse_packets=%d",
|
|
totalFrames, path.sourceUDP.Load(), path.session.mediaIngress.Load(), path.session.mediaRecovered.Load(),
|
|
path.session.mediaEnqueued.Load(), path.session.Telemetry().MediaDrops, metrics.MediaDrops,
|
|
path.session.mediaQueueMaximum.Load(), path.session.mediaQueueMaximumBytes.Load(), queueDelay, metrics.MediaPackets)
|
|
|
|
path.Close()
|
|
select {
|
|
case <-path.session.readDone:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("native media workers did not stop")
|
|
}
|
|
}
|
|
|
|
func TestQualificationLateSourceBatchDoesNotCollapseThroughPublicPath(t *testing.T) {
|
|
profile := qualificationMediaProfiles()[2]
|
|
path := newQualificationPath(t, profile, profile.BitrateKbps)
|
|
defer path.Close()
|
|
|
|
const shardCount = 662
|
|
payload := qualificationFramePayload(profile, 0)
|
|
if len(payload) != 666_664 {
|
|
t.Fatalf("4K60 keyframe bytes = %d, want 666664", len(payload))
|
|
}
|
|
packets := qualificationSourceVideoPackets(t, path.key, 1, payload)
|
|
if len(packets) != shardCount {
|
|
t.Fatalf("source packet count = %d, want %d", len(packets), shardCount)
|
|
}
|
|
|
|
var batchIndices []int
|
|
var batchStarts []time.Time
|
|
var firstBatch time.Time
|
|
path.fixture.observeVideoBatch = func(index int, at time.Time) {
|
|
if firstBatch.IsZero() {
|
|
firstBatch = at
|
|
}
|
|
batchIndices = append(batchIndices, index)
|
|
batchStarts = append(batchStarts, at)
|
|
}
|
|
stalled := false
|
|
path.fixture.beforeVideoBatch = func(ctx context.Context, index int) error {
|
|
if index != 63 || stalled {
|
|
return nil
|
|
}
|
|
stalled = true
|
|
return qualificationWaitContext(ctx, firstBatch.Add(5*time.Millisecond))
|
|
}
|
|
|
|
trace, _, err := path.traverse(t, payload)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !trace.NativeSetup || !trace.NativeOpen || !trace.NativeUDPIngress || !trace.ApolloRecovered ||
|
|
!trace.ProductionQueue || !trace.ProductionMediaLoop || !trace.ProductionPacer ||
|
|
!trace.VerseQUIC || !trace.PublicClientDecode || !trace.PayloadPreserved {
|
|
t.Fatalf("late-batch path skipped production stages: %#v", trace)
|
|
}
|
|
if got := path.sourceUDP.Load(); got != shardCount {
|
|
t.Fatalf("source UDP count = %d, want %d", got, shardCount)
|
|
}
|
|
if len(batchStarts) != (shardCount+62)/63 {
|
|
t.Fatalf("batch count = %d, want %d", len(batchStarts), (shardCount+62)/63)
|
|
}
|
|
packetsPerMillisecond, _ := qualificationApolloVideoPacing(apolloVideoRawPacketSize)
|
|
for index := 1; index < len(batchStarts); index++ {
|
|
previousPackets := min(63, shardCount-batchIndices[index-1])
|
|
minimum := qualificationApolloVideoOffset(previousPackets, packetsPerMillisecond)
|
|
actual := batchStarts[index].Sub(batchStarts[index-1])
|
|
if actual < minimum {
|
|
t.Fatalf("batch %d at packet %d started %s after packet %d, before raw serialization interval %s; starts=%v",
|
|
index, batchIndices[index], actual, batchIndices[index-1], minimum, batchStarts)
|
|
}
|
|
}
|
|
t.Logf("late-batch public path: packets=%d starts=%v ingress=%d recovered=%d enqueued=%d",
|
|
path.sourceUDP.Load(), batchStarts, path.session.mediaIngress.Load(),
|
|
path.session.mediaRecovered.Load(), path.session.mediaEnqueued.Load())
|
|
}
|
|
|
|
func TestQualificationBlockedVideoAEADDoesNotBlockProviderIngress(t *testing.T) {
|
|
profile := qualificationMediaProfiles()[2]
|
|
blockCtx, cancelBlock := context.WithCancel(context.Background())
|
|
defer cancelBlock()
|
|
blocked := make(chan struct{})
|
|
release := make(chan struct{})
|
|
var releaseOnce sync.Once
|
|
releaseAEAD := func() { releaseOnce.Do(func() { close(release) }) }
|
|
t.Cleanup(releaseAEAD)
|
|
native := NewNativeApolloBackend()
|
|
native.configureMedia = func(media *apolloMediaCodec) {
|
|
media.aead = &qualificationBlockingAEAD{
|
|
AEAD: media.aead, ctx: blockCtx, blocked: blocked, release: release,
|
|
}
|
|
}
|
|
path := newQualificationPathWithNativeBackend(t, profile, profile.BitrateKbps, nil, native)
|
|
t.Cleanup(func() {
|
|
path.Close()
|
|
select {
|
|
case <-path.session.readDone:
|
|
case <-time.After(time.Second):
|
|
t.Error("native media workers did not stop during qualification cleanup")
|
|
}
|
|
})
|
|
|
|
payload := qualificationFramePayload(profile, 0)
|
|
packets := qualificationSourceVideoPackets(t, path.key, 1, payload)
|
|
if len(payload) != 666_664 || len(packets) != 662 {
|
|
t.Fatalf("4K60 keyframe = %d bytes/%d shards, want 666664/662", len(payload), len(packets))
|
|
}
|
|
|
|
beforeMetrics := path.server.Metrics()
|
|
beforeIngress := path.session.mediaIngress.Load()
|
|
beforeRecovered := path.session.mediaRecovered.Load()
|
|
beforeEnqueued := path.session.mediaEnqueued.Load()
|
|
beforePacer := path.server.pacer.reservations.Load()
|
|
if _, err := path.emit(t, payload); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case <-blocked:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("video AEAD did not block")
|
|
}
|
|
if sent := path.fixture.sentPackets.Load(); sent != 662 {
|
|
t.Fatalf("fixture sent packets = %d, want 662", sent)
|
|
}
|
|
deadline := time.NewTimer(250 * time.Millisecond)
|
|
defer deadline.Stop()
|
|
for path.session.mediaIngress.Load()-beforeIngress != 662 {
|
|
select {
|
|
case <-deadline.C:
|
|
t.Fatalf("blocked-AEAD ingress = %d, want 662 after 662 successful fixture writes",
|
|
path.session.mediaIngress.Load()-beforeIngress)
|
|
default:
|
|
runtime.Gosched()
|
|
}
|
|
}
|
|
|
|
releaseAEAD()
|
|
recovered, err := path.receivePayload(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
afterMetrics := path.server.Metrics()
|
|
stageDeadline := time.Now().Add(2 * time.Second)
|
|
for (path.session.mediaRecovered.Load() <= beforeRecovered ||
|
|
path.session.mediaEnqueued.Load() <= beforeEnqueued ||
|
|
afterMetrics.ProcessingSamples <= beforeMetrics.ProcessingSamples ||
|
|
afterMetrics.MediaPackets <= beforeMetrics.MediaPackets) && time.Now().Before(stageDeadline) {
|
|
runtime.Gosched()
|
|
afterMetrics = path.server.Metrics()
|
|
}
|
|
if path.backend.setups.Load() != 1 || path.backend.opens.Load() != 1 ||
|
|
path.session.mediaRecovered.Load()-beforeRecovered != 1 ||
|
|
path.session.mediaEnqueued.Load()-beforeEnqueued != 1 ||
|
|
afterMetrics.ProcessingSamples <= beforeMetrics.ProcessingSamples ||
|
|
path.server.pacer.reservations.Load() <= beforePacer ||
|
|
afterMetrics.MediaPackets <= beforeMetrics.MediaPackets || !bytes.Equal(recovered, payload) {
|
|
t.Fatalf("blocked-AEAD public path: setup=%d open=%d ingress=%d recovered=%d enqueued=%d processing=%d pacer=%d media=%d payload=%t",
|
|
path.backend.setups.Load(), path.backend.opens.Load(), path.session.mediaIngress.Load()-beforeIngress,
|
|
path.session.mediaRecovered.Load()-beforeRecovered, path.session.mediaEnqueued.Load()-beforeEnqueued,
|
|
afterMetrics.ProcessingSamples-beforeMetrics.ProcessingSamples,
|
|
path.server.pacer.reservations.Load()-beforePacer, afterMetrics.MediaPackets-beforeMetrics.MediaPackets,
|
|
bytes.Equal(recovered, payload))
|
|
}
|
|
}
|
|
|
|
func TestQualificationProcessingRetainsProviderIngressDiagnostics(t *testing.T) {
|
|
profile := qualificationMediaProfiles()[2]
|
|
path := newQualificationProcessingPath(t, profile, profile.BitrateKbps)
|
|
defer path.Close()
|
|
|
|
payload := qualificationFramePayload(profile, 0)
|
|
packets := qualificationSourceVideoPackets(t, path.key, 1, payload)
|
|
if len(payload) != 666_664 || len(packets) != 662 {
|
|
t.Fatalf("4K60 keyframe = %d bytes/%d shards, want 666664/662", len(payload), len(packets))
|
|
}
|
|
before, err := path.process.snapshot()
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := path.emit(t, payload); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
recovered, err := path.receivePayload(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
diagnostics := path.processingDiagnostics(1)
|
|
if diagnostics.SnapshotError != "" {
|
|
t.Fatal(diagnostics.SnapshotError)
|
|
}
|
|
after := diagnostics.Gateway
|
|
batches := diagnostics.BatchHistory
|
|
if diagnostics.ProcessedFrames != 1 || diagnostics.SourceFrames != 1 ||
|
|
diagnostics.ExpectedWritesThroughLastFrame != 662 ||
|
|
diagnostics.FixtureSentPackets != 662 || diagnostics.SourceUDP != 662 ||
|
|
after.MediaIngress-before.MediaIngress != 662 ||
|
|
after.MediaRecovered-before.MediaRecovered != 1 ||
|
|
after.MediaEnqueued-before.MediaEnqueued != 1 || !bytes.Equal(recovered, payload) {
|
|
t.Fatalf("provider ingress diagnostics: sent=%d source=%d ingress=%d recovered=%d enqueued=%d payload=%t",
|
|
diagnostics.FixtureSentPackets, diagnostics.SourceUDP, after.MediaIngress-before.MediaIngress,
|
|
after.MediaRecovered-before.MediaRecovered, after.MediaEnqueued-before.MediaEnqueued,
|
|
bytes.Equal(recovered, payload))
|
|
}
|
|
if len(batches) != 11 {
|
|
t.Fatalf("provider batch history = %d entries, want 11", len(batches))
|
|
}
|
|
for index := range batches {
|
|
wantPacket := index * 63
|
|
if batches[index].ProviderFrame != 1 || batches[index].PacketWithinFrame != wantPacket ||
|
|
batches[index].SourcePacket != uint64(wantPacket) {
|
|
t.Fatalf("provider batch %d = %#v, want frame 1 source/within %d", index, batches[index], wantPacket)
|
|
}
|
|
if index > 0 {
|
|
previousPackets := min(63, 662-batches[index-1].PacketWithinFrame)
|
|
minimum := qualificationApolloVideoOffset(previousPackets, 96)
|
|
if batches[index].StartedAfter-batches[index-1].StartedAfter < minimum {
|
|
t.Fatalf("provider batch %d collapsed: %#v", index, batches)
|
|
}
|
|
}
|
|
}
|
|
if (runtime.GOOS == "darwin" || runtime.GOOS == "linux") && !after.VideoReceiveBufferAvailable {
|
|
t.Fatal("provider video SO_RCVBUF unavailable on a supported diagnostic platform")
|
|
}
|
|
if after.VideoReceiveBufferAvailable && after.VideoReceiveBuffer <= 0 {
|
|
t.Fatalf("provider video SO_RCVBUF = %d available=%t, want measured >0",
|
|
after.VideoReceiveBuffer, after.VideoReceiveBufferAvailable)
|
|
}
|
|
if after.KernelDropsAvailable && after.KernelDrops != 0 {
|
|
t.Fatalf("provider UDP kernel drops = %d, want 0", after.KernelDrops)
|
|
}
|
|
t.Logf("provider ingress diagnostics: %#v", diagnostics)
|
|
}
|
|
|
|
func TestQualificationActualEmissionDoesNotCatchUpAfterPreWritePause(t *testing.T) {
|
|
key := bytes.Repeat([]byte{0x4d}, 16)
|
|
packets := qualificationSourceVideoPackets(t, key, 1, make([]byte, 189*apolloVideoShardPayloadSize-8))
|
|
receiver, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer receiver.Close()
|
|
sender, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.ParseIP("127.0.0.1")})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer sender.Close()
|
|
fixture := &qualificationApolloFixture{video: sender, failures: make(chan error, 1)}
|
|
remote := *receiver.LocalAddr().(*net.UDPAddr)
|
|
fixture.videoRemote.Store(&remote)
|
|
stalled := false
|
|
fixture.beforeVideoFirstWrite = func(ctx context.Context, packetWithinFrame int) error {
|
|
if packetWithinFrame != 63 || stalled {
|
|
return nil
|
|
}
|
|
stalled = true
|
|
return qualificationWaitContext(ctx, time.Now().Add(5*time.Millisecond))
|
|
}
|
|
var emitted []time.Time
|
|
fixture.observeVideoBatch = func(_ int, at time.Time) { emitted = append(emitted, at) }
|
|
drained := make(chan struct{})
|
|
go func() {
|
|
defer close(drained)
|
|
buffer := make([]byte, 2048)
|
|
for range len(packets) {
|
|
if _, _, readErr := receiver.ReadFromUDP(buffer); readErr != nil {
|
|
return
|
|
}
|
|
}
|
|
}()
|
|
if err := fixture.sendVideo(context.Background(), 1, packets); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
select {
|
|
case <-drained:
|
|
case <-time.After(time.Second):
|
|
t.Fatal("actual-emission receiver did not drain")
|
|
}
|
|
if len(emitted) != 3 {
|
|
t.Fatalf("actual emitted batch starts = %d, want 3", len(emitted))
|
|
}
|
|
minimum := qualificationApolloVideoOffset(63, 96)
|
|
if actual := emitted[2].Sub(emitted[1]); actual < minimum {
|
|
t.Fatalf("post-write batch 126 started %s after batch 63, before %s; emitted=%v", actual, minimum, emitted)
|
|
}
|
|
t.Logf("post-write emitted batch starts: %v", emitted)
|
|
}
|
|
|
|
func TestQualificationProcessingDiagnosticsFormatUsesMonotonicOffsets(t *testing.T) {
|
|
diagnostics := qualificationProcessingDiagnostics{
|
|
BatchHistory: []qualificationVideoBatchObservation{{StartedAfter: 1234567 * time.Nanosecond}},
|
|
}
|
|
failureText := fmt.Errorf("qualification failed; diagnostics=%#v", diagnostics).Error()
|
|
if !strings.Contains(failureText, "StartedAfter") || !strings.Contains(failureText, "1234567") ||
|
|
strings.Contains(failureText, "time.Date(") {
|
|
t.Fatalf("processing failure text does not retain explicit monotonic offsets: %s", failureText)
|
|
}
|
|
}
|