sechmachine/VerseVDI-Data-Plane
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(gateway): isolate qualification evidence

Browse Source
This commit is contained in:
sechmachine
2026-07-30 15:06:23 +07:00
parent c0aabf2a72
commit b5aaff9e39
10 changed files with 1004 additions and 160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gateway/qualification_harness_test.go
+216 -85
View File
@@ -38,12 +38,14 @@ import (
)
const (
qualificationToolVersion = "versevdi-gateway-qualification/v4"
qualificationToolVersion = "versevdi-gateway-qualification/v5"
qualificationImpairmentQueuePackets = 64
qualificationImpairmentMaxPackets = 100_000
qualificationImpairmentPacketCount = 10_000
qualificationProcessingLimit = 5 * time.Millisecond
qualificationImpairmentSeed uint64 = 0x3c6a11ce
qualificationClockOverheadMethod = "median of 1000 batches of 100 monotonic time reads"
qualificationGatewayCPUScope = "isolated gateway subprocess; bounded recorder/control included, fixture and client driver excluded"
)
type qualificationMediaProfile struct {
@@ -74,10 +76,12 @@ type qualificationPath struct {
session *nativeApolloSession
fixture *qualificationApolloFixture
backend *qualificationTracingBackend
process *qualificationGatewayProcess
key []byte
flow string
frame uint32
bootTrace atomic.Bool
sourceUDP atomic.Uint64
closeOnce sync.Once
shutdown func()
}
@@ -109,6 +113,7 @@ type qualificationProcessingSummary struct {
Mean time.Duration `json:"mean_ns"`
StandardDeviation time.Duration `json:"standard_deviation_ns"`
ClockOverhead time.Duration `json:"clock_overhead_ns"`
ClockMethod string `json:"clock_overhead_method"`
Histogram map[string]int `json:"histogram"`
PayloadSHA256 string `json:"payload_sha256"`
RawSamples string `json:"raw_samples"`
@@ -135,6 +140,20 @@ type qualificationImpairmentObservation struct {
Dropped int `json:"dropped"`
InjectedDropped int `json:"injected_dropped"`
InjectedReordered int `json:"injected_reordered"`
SourceEmitted int `json:"source_emitted"`
SourceDatagrams uint64 `json:"source_datagrams"`
ProviderIngressDatagrams uint64 `json:"provider_ingress_datagrams"`
ProviderRecovered int `json:"provider_recovered"`
ProviderFECDropped int `json:"provider_fec_dropped"`
ProviderEnqueued int `json:"provider_enqueued"`
ProviderEnqueueDropped int `json:"provider_enqueue_dropped"`
ProviderQueueReplaced int `json:"provider_queue_replaced"`
GatewayForwarded int `json:"gateway_forwarded"`
GatewayDropped int `json:"gateway_dropped"`
QUICSent int `json:"quic_sent"`
QUICSendDropped int `json:"quic_send_dropped"`
ClientDeliveryDropped int `json:"client_delivery_dropped"`
UnexplainedDropped int `json:"unexplained_dropped"`
ObservedOutOfOrder int `json:"observed_out_of_order"`
ObservedLatency time.Duration `json:"observed_one_way_latency_ns"`
ObservedRTT time.Duration `json:"observed_rtt_ns"`
@@ -146,6 +165,7 @@ type qualificationImpairmentObservation struct {
MaxQueuePackets int `json:"max_queue_packets"`
ConfiguredRTT time.Duration `json:"configured_rtt_ns"`
ConfiguredJitter time.Duration `json:"configured_jitter_ns"`
AppliedJitter time.Duration `json:"applied_jitter_ns"`
ConfiguredLossPercent float64 `json:"configured_loss_percent"`
ConfiguredReorder bool `json:"configured_reorder"`
ConfiguredCapacitySteps []int `json:"configured_capacity_steps_percent"`
@@ -255,6 +275,11 @@ func qualificationPayload(profile qualificationMediaProfile) []byte {
return payload
}
func qualificationMediaPacerKbps(profile qualificationMediaProfile, reduction int) int64 {
payloadKbps := profile.BitrateKbps * int64(100-reduction) / 100
return (payloadKbps*int64(profile.PacketBytes+frameHeaderSize) + int64(profile.PacketBytes) - 1) / int64(profile.PacketBytes)
}
type qualificationTracingBackend struct {
native *NativeApolloBackend
setups atomic.Uint64
@@ -867,6 +892,46 @@ func newQualificationPathWithImpairment(t *testing.T, profile qualificationMedia
return path
}
func newQualificationProcessingPath(t *testing.T, profile qualificationMediaProfile, pacerKbps int64) *qualificationPath {
t.Helper()
serverTLS, clientTLS := testTLS(t)
fixture := newQualificationApolloFixture(t, serverTLS, clientTLS, "qualification-session", profile)
authority := protocol.SessionAuthority{
Version: "1", SessionID: "qualification-session", GatewayID: "gateway-1",
Audience: "versevdi-gateway", ReconnectSequence: 0,
ExpiresAt: time.Now().Add(45 * time.Minute).UTC().Format(time.RFC3339Nano),
Capabilities: DefaultCapabilities(), ProviderProfile: ProviderProfileApollo,
ProviderIdentity: fixture.work.ProviderIdentity,
}
work := fixture.work
work.ExpiresAt = authority.ExpiresAt
process := startQualificationGatewayProcess(t, serverTLS, authority, work, pacerKbps)
request := protocol.TunnelAdmissionRequest{
Version: "1", SessionID: authority.SessionID, GatewayID: authority.GatewayID,
Audience: authority.Audience, Grant: strings.Repeat("g", 64),
ClientNonce: "nonce-qualification", DeviceSignature: strings.Repeat("s", 86),
Capabilities: DefaultCapabilities(),
}
client, err := Dial(context.Background(), process.ready.GatewayAddress, clientTLS, request)
if err != nil {
process.Close()
t.Fatal(err)
}
key, err := fixture.streamKey()
if err != nil {
_ = client.Close()
process.Close()
t.Fatal(err)
}
path := &qualificationPath{client: client, fixture: fixture, process: process, key: key}
path.shutdown = func() {
_ = client.Close()
process.Close()
}
t.Cleanup(path.Close)
return path
}
func (p *qualificationPath) Close() {
if p != nil {
p.closeOnce.Do(p.shutdown)
@@ -913,7 +978,7 @@ func (p *qualificationPath) traverse(t *testing.T, payload []byte) (qualificatio
func (p *qualificationPath) emit(t *testing.T, payload []byte) (qualificationPathTrace, error) {
t.Helper()
if p == nil || p.session == nil || len(payload) == 0 || len(payload) > 2*apolloVideoShardPayloadSize-8 {
if p == nil || p.fixture == nil || len(payload) == 0 || len(payload) > 2*apolloVideoShardPayloadSize-8 {
return qualificationPathTrace{}, ErrProviderMalformed
}
p.frame++
@@ -923,6 +988,7 @@ func (p *qualificationPath) emit(t *testing.T, payload []byte) (qualificationPat
if err := p.fixture.sendVideo(ctx, packets); err != nil {
return qualificationPathTrace{}, err
}
p.sourceUDP.Add(uint64(len(packets)))
return qualificationPathTrace{}, nil
}
@@ -1140,7 +1206,7 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
ConfiguredCapacitySteps: append([]int(nil), profile.CapacitySteps...),
RTTSource: "apollo_enet_acknowledge",
}
path := newQualificationImpairedPath(t, media, media.BitrateKbps, profile)
path := newQualificationImpairedPath(t, media, qualificationMediaPacerKbps(media, 0), profile)
defer path.Close()
payload := qualificationPayload(media)
rawSamples := make([]rawSample, packetCount)
@@ -1166,31 +1232,36 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
jobs = append(jobs, scheduledPacket{index: index, target: time.Duration(index)*spacing + delay})
rawSamples[index].outcome = "traversal_dropped"
}
var jitterMean, jitterM2 float64
var jitterSamples int
for _, packet := range jobs {
applied := float64(packet.target - time.Duration(packet.index)*spacing - profile.RTT/2)
jitterSamples++
delta := applied - jitterMean
jitterMean += delta / float64(jitterSamples)
jitterM2 += delta * (applied - jitterMean)
}
if jitterSamples > 0 {
observation.AppliedJitter = time.Duration(math.Sqrt(jitterM2 / float64(jitterSamples)))
}
if profile.Reorder {
for index := 18; index+1 < packetCount; index += 20 {
first, second := jobByIndex[index], jobByIndex[index+1]
if first < 0 || second < 0 {
continue
}
earlier := min(jobs[first].target, jobs[second].target)
later := max(jobs[first].target, jobs[second].target)
jobs[second].target = earlier
jobs[first].target = later + time.Nanosecond
jobs[first], jobs[second] = jobs[second], jobs[first]
observation.InjectedReordered++
}
}
sort.SliceStable(jobs, func(first, second int) bool {
if jobs[first].target == jobs[second].target {
return jobs[first].index < jobs[second].index
}
return jobs[first].target < jobs[second].target
})
started := time.Now()
beforeMetrics := path.server.Metrics()
beforeIngress := path.session.mediaIngress.Load()
beforeRecovered := path.session.mediaRecovered.Load()
beforeEnqueued := path.session.mediaEnqueued.Load()
beforeProviderDrops := path.session.mediaDrops.Load()
beforeSourceUDP := path.sourceUDP.Load()
beforePacer := path.server.pacer.reservations.Load()
grace := max(2*profile.RTT+2*profile.Jitter, 2*time.Second)
lastTarget := time.Duration(packetCount) * spacing
@@ -1254,17 +1325,28 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
}()
stepAt := make(map[int]time.Time, len(profile.CapacitySteps))
emitted := 0
var previousRelease time.Time
maxCatchup := spacing
for _, packet := range jobs {
if delay := time.Until(started.Add(packet.target)); delay > 0 {
release := started.Add(packet.target)
if minimum := previousRelease.Add(spacing); !previousRelease.IsZero() && release.Before(minimum) {
release = minimum
}
if lag := time.Since(release); lag > maxCatchup {
release = release.Add(lag - maxCatchup)
}
if delay := time.Until(release); delay > 0 {
time.Sleep(delay)
}
previousRelease = release
if len(profile.CapacitySteps) == 2 {
switch {
case packet.index >= packetCount*2/3 && stepAt[profile.CapacitySteps[1]].IsZero():
path.server.pacer.setKbps(media.BitrateKbps * int64(100-profile.CapacitySteps[1]) / 100)
path.server.pacer.setKbps(qualificationMediaPacerKbps(media, profile.CapacitySteps[1]))
stepAt[profile.CapacitySteps[1]] = time.Now()
case packet.index >= packetCount/3 && stepAt[profile.CapacitySteps[0]].IsZero():
path.server.pacer.setKbps(media.BitrateKbps * int64(100-profile.CapacitySteps[0]) / 100)
path.server.pacer.setKbps(qualificationMediaPacerKbps(media, profile.CapacitySteps[0]))
stepAt[profile.CapacitySteps[0]] = time.Now()
}
}
@@ -1274,6 +1356,7 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
receiveCancel()
return qualificationImpairmentObservation{}, err
}
emitted++
}
received := <-receivedDone
receiveCancel()
@@ -1316,6 +1399,23 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
completedAt = received.packets[len(received.packets)-1].deliveredAt
}
afterMetrics := path.server.Metrics()
observation.SourceEmitted = emitted
observation.SourceDatagrams = path.sourceUDP.Load() - beforeSourceUDP
observation.ProviderIngressDatagrams = path.session.mediaIngress.Load() - beforeIngress
observation.ProviderRecovered = int(path.session.mediaRecovered.Load() - beforeRecovered)
observation.ProviderEnqueued = int(path.session.mediaEnqueued.Load() - beforeEnqueued)
observation.ProviderQueueReplaced = int(path.session.mediaDrops.Load() - beforeProviderDrops)
observation.GatewayForwarded = int(afterMetrics.ProcessingSamples - beforeMetrics.ProcessingSamples)
observation.QUICSent = int(afterMetrics.MediaPackets - beforeMetrics.MediaPackets)
observation.ProviderFECDropped = max(observation.SourceEmitted-observation.ProviderRecovered, 0)
observation.ProviderEnqueueDropped = max(observation.ProviderRecovered-observation.ProviderEnqueued, 0)
observation.GatewayDropped = max(observation.ProviderEnqueued-observation.ProviderQueueReplaced-observation.GatewayForwarded, 0)
observation.QUICSendDropped = max(observation.GatewayForwarded-observation.QUICSent, 0)
observation.ClientDeliveryDropped = max(observation.QUICSent-observation.Delivered, 0)
observation.UnexplainedDropped = observation.Dropped - observation.InjectedDropped -
observation.ProviderFECDropped - observation.ProviderEnqueueDropped -
observation.ProviderQueueReplaced - observation.GatewayDropped -
observation.QUICSendDropped - observation.ClientDeliveryDropped
if observation.Delivered > 0 && (path.session.mediaIngress.Load() <= beforeIngress ||
path.session.mediaRecovered.Load() <= beforeRecovered ||
path.session.mediaEnqueued.Load() <= beforeEnqueued ||
@@ -1391,6 +1491,7 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
}
}
if observation.Delivered+observation.Dropped != observation.Sent ||
observation.UnexplainedDropped != 0 ||
observation.MaxQueuePackets > qualificationImpairmentQueuePackets {
return qualificationImpairmentObservation{}, errors.New("qualification impairment accounting invalid")
}
@@ -1469,75 +1570,58 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
if len(payload) < 4 {
return qualificationProcessingSummary{}, errors.New("qualification payload too small")
}
pacerKbps := (profile.BitrateKbps*int64(profile.PacketBytes+frameHeaderSize) + int64(profile.PacketBytes) - 1) / int64(profile.PacketBytes)
path := newQualificationPath(t, profile, pacerKbps)
path := newQualificationProcessingPath(t, profile, qualificationMediaPacerKbps(profile, 0))
defer path.Close()
if err := runQualificationWarmup(t, path, profile, payload); err != nil {
if err := runQualificationProcessWarmup(t, path, profile, payload); err != nil {
return qualificationProcessingSummary{}, err
}
file, err := os.OpenFile(rawPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o640)
resourcePath := strings.TrimSuffix(rawPath, ".csv.gz") + "-resources.csv.gz"
if err := path.process.startRecording(rawPath, resourcePath); err != nil {
return qualificationProcessingSummary{}, err
}
before, err := path.process.snapshot()
if err != nil {
return qualificationProcessingSummary{}, err
}
compressed, err := gzip.NewWriterLevel(file, gzip.BestSpeed)
if err != nil {
_ = file.Close()
return qualificationProcessingSummary{}, err
}
buffered := bufio.NewWriterSize(compressed, 1<<20)
closed := false
defer func() {
if !closed {
_ = buffered.Flush()
_ = compressed.Close()
_ = file.Close()
}
}()
if _, err := buffered.WriteString("elapsed_ns,processing_ns\n"); err != nil {
return qualificationProcessingSummary{}, err
}
bytesPerSecond := profile.BitrateKbps * 1000 / 8
targetBytes := bytesPerSecond * profile.Duration.Nanoseconds() / int64(time.Second)
targetPackets := (targetBytes + int64(profile.PacketBytes) - 1) / int64(profile.PacketBytes)
samples := make([]time.Duration, 0, int(targetPackets))
started := time.Now()
resources := []qualificationResourceSample{qualificationRuntimeSample(started)}
lastResourceSample := started
var processed int64
for processed < targetPackets || time.Since(started) < profile.Duration {
current := append([]byte(nil), payload...)
binary.BigEndian.PutUint32(current[len(current)-4:], uint32(processed))
trace, sample, traverseErr := path.traverse(t, current)
if traverseErr != nil {
return qualificationProcessingSummary{}, traverseErr
if _, err := path.emit(t, current); err != nil {
return qualificationProcessingSummary{}, err
}
if !trace.NativeUDPIngress || !trace.ApolloRecovered || !trace.ProductionQueue ||
!trace.ProductionMediaLoop || !trace.ProductionPacer || !trace.VerseQUIC ||
!trace.PublicClientDecode || !trace.PayloadPreserved {
return qualificationProcessingSummary{}, fmt.Errorf("qualification bypassed the production provider-to-client path: %#v", trace)
}
samples = append(samples, sample)
if _, writeErr := fmt.Fprintf(buffered, "%d,%d\n", time.Since(started).Nanoseconds(), sample.Nanoseconds()); writeErr != nil {
return qualificationProcessingSummary{}, writeErr
recovered, err := path.receivePayload(context.Background())
if err != nil || !bytes.Equal(recovered, current) {
return qualificationProcessingSummary{}, errors.New("qualification processing payload integrity failure")
}
processed++
if time.Since(lastResourceSample) >= time.Second {
resources = append(resources, qualificationRuntimeSample(started))
lastResourceSample = time.Now()
}
}
actualDuration := time.Since(started)
resources = append(resources, qualificationRuntimeSample(started))
if err := buffered.Flush(); err != nil {
record, err := path.process.stopRecording()
if err != nil {
return qualificationProcessingSummary{}, err
}
if err := compressed.Close(); err != nil {
after, err := path.process.snapshot()
if err != nil {
return qualificationProcessingSummary{}, err
}
if err := file.Close(); err != nil {
if after.NativeSetups != 1 || after.NativeOpens != 1 ||
after.MediaRecovered-before.MediaRecovered != uint64(processed) ||
after.MediaEnqueued-before.MediaEnqueued != uint64(processed) ||
after.MediaDrops != before.MediaDrops ||
after.Metrics.ProcessingSamples-before.Metrics.ProcessingSamples != uint64(processed) ||
after.PacerReservations <= before.PacerReservations ||
after.Metrics.MediaPackets-before.Metrics.MediaPackets != uint64(processed) {
return qualificationProcessingSummary{}, fmt.Errorf("qualification subprocess bypassed a production stage: before=%#v after=%#v processed=%d", before, after, processed)
}
samples, err := readQualificationProcessingSamples(rawPath, record.Count)
if err != nil {
return qualificationProcessingSummary{}, err
}
closed = true
summary, err := summarizeQualificationSamples(samples)
if err != nil {
return qualificationProcessingSummary{}, err
@@ -1553,15 +1637,12 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
summary.Warmup = profile.Warmup
summary.ConfiguredDuration = profile.Duration
summary.ActualDuration = actualDuration
summary.ClockOverhead = qualificationClockOverhead()
summary.ClockOverhead = record.ClockOverhead
summary.ClockMethod = record.ClockMethod
summary.PayloadSHA256 = fmt.Sprintf("%x", sha256.Sum256(payload))
summary.RawSamples = filepath.Base(rawPath)
summary.RawSamplesSHA256 = sum
summary.RawSamplesBytes = size
resourcePath := strings.TrimSuffix(rawPath, ".csv.gz") + "-resources.csv.gz"
if err := writeQualificationResourceSamples(resourcePath, resources); err != nil {
return qualificationProcessingSummary{}, err
}
resourceSum, resourceSize, err := qualificationFileSHA256(resourcePath)
if err != nil {
return qualificationProcessingSummary{}, err
@@ -1569,23 +1650,16 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
summary.RawResources = filepath.Base(resourcePath)
summary.RawResourcesSHA256 = resourceSum
summary.RawResourcesBytes = resourceSize
summary.ResourceSamples = len(resources)
summary.CPUScope = "isolated gateway qualification process (gateway plus bounded fixture/client driver)"
firstResource, lastResource := resources[0], resources[len(resources)-1]
if firstResource.CPUSeconds < 0 || lastResource.CPUSeconds < 0 {
summary.ResourceSamples = record.ResourceSamples
summary.CPUScope = qualificationGatewayCPUScope
summary.CPUSeconds = record.CPUSeconds
summary.PeakHeapBytes = record.PeakHeapBytes
summary.PeakGoroutines = record.PeakGoroutines
summary.Mallocs = record.Mallocs
summary.AllocatedBytes = record.AllocatedBytes
if summary.CPUSeconds < 0 || summary.ClockOverhead <= 0 || summary.ClockMethod == "" {
return qualificationProcessingSummary{}, errors.New("process CPU usage unavailable")
}
summary.CPUSeconds = math.Max(0, lastResource.CPUSeconds-firstResource.CPUSeconds)
summary.Mallocs = lastResource.Mallocs - firstResource.Mallocs
summary.AllocatedBytes = lastResource.Allocated - firstResource.Allocated
for _, resource := range resources {
if resource.HeapBytes > summary.PeakHeapBytes {
summary.PeakHeapBytes = resource.HeapBytes
}
if resource.Goroutines > summary.PeakGoroutines {
summary.PeakGoroutines = resource.Goroutines
}
}
if actualDuration < profile.Duration || actualDuration > profile.Duration*105/100+250*time.Millisecond {
return qualificationProcessingSummary{}, fmt.Errorf("wall-clock duration %s outside [%s,%s]", actualDuration, profile.Duration, profile.Duration*105/100+250*time.Millisecond)
}
@@ -1598,6 +1672,57 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
return summary, nil
}
func readQualificationProcessingSamples(path string, expected int) ([]time.Duration, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
compressed, err := gzip.NewReader(file)
if err != nil {
return nil, err
}
defer compressed.Close()
scanner := bufio.NewScanner(compressed)
if !scanner.Scan() || scanner.Text() != "elapsed_ns,queue_ns,processing_ns,pacing_ns" {
return nil, errors.New("qualification processing header invalid")
}
samples := make([]time.Duration, 0, expected)
for scanner.Scan() {
fields := strings.Split(scanner.Text(), ",")
if len(fields) != 4 {
return nil, errors.New("qualification processing row invalid")
}
value, err := strconv.ParseInt(fields[2], 10, 64)
if err != nil || value < 0 {
return nil, errors.New("qualification processing sample invalid")
}
samples = append(samples, time.Duration(value))
}
if err := scanner.Err(); err != nil {
return nil, err
}
if len(samples) != expected {
return nil, fmt.Errorf("qualification processing rows=%d want=%d", len(samples), expected)
}
return samples, nil
}
func runQualificationProcessWarmup(t *testing.T, path *qualificationPath, profile qualificationMediaProfile, payload []byte) error {
t.Helper()
started := time.Now()
for time.Since(started) < profile.Warmup {
if _, err := path.emit(t, payload); err != nil {
return err
}
recovered, err := path.receivePayload(context.Background())
if err != nil || !bytes.Equal(recovered, payload) {
return errors.New("qualification warmup payload integrity failure")
}
}
return nil
}
func runQualificationWarmup(t *testing.T, path *qualificationPath, profile qualificationMediaProfile, payload []byte) error {
t.Helper()
started := time.Now()
@@ -1657,11 +1782,17 @@ func writeQualificationResourceSamples(path string, samples []qualificationResou
}
func qualificationClockOverhead() time.Duration {
samples := make([]time.Duration, 10_000)
const readsPerBatch = 100
samples := make([]time.Duration, 1000)
var observed time.Time
for index := range samples {
started := time.Now()
samples[index] = time.Since(started)
for range readsPerBatch {
observed = time.Now()
}
samples[index] = time.Since(started) / readsPerBatch
}
runtime.KeepAlive(observed)
summary, _ := summarizeQualificationSamples(samples)
return summary.Median
}
@@ -2000,10 +2131,10 @@ func TestSection7Qualification(t *testing.T) {
StartedAt: started.Format(time.RFC3339Nano), GoVersion: runtime.Version(),
ToolVersions: toolVersions,
OS: runtime.GOOS, Architecture: runtime.GOARCH,
Topology: "source-shaped encrypted Apollo fixture -> native recovery/FEC -> bounded provider queue -> production fair pacer -> Verse framing over mTLS/QUIC -> public Verse client decoder",
Topology: "parent source-shaped encrypted Apollo fixture -> isolated gateway subprocess for processing/resource evidence -> public Verse client decoder; impairment uses the same native recovery/FEC, bounded queue, production pacer, framing, and QUIC path",
Direction: "provider_to_client",
QueueDiscipline: "bounded 16-packet native provider queue, production equal-tier fair pacer, deterministic fixed-seed source impairment",
Evidence: []string{"deterministic source-shaped Apollo recovery", "local real-time production path", "mTLS/QUIC fixture transport", "path impairment", "production fair pacer"},
QueueDiscipline: "ordered fixed-seed source delay queue with one-serialization-interval catch-up, bounded 16-packet native provider queue, production equal-tier fair pacer",
Evidence: []string{"deterministic source-shaped Apollo recovery", "isolated gateway-process resources", "local real-time production path", "mTLS/QUIC fixture transport", "attributed path impairment", "production fair pacer"},
Deferred: []string{"live Apollo", "macOS client", "physical firewall and packet route", "real encoder fidelity", "multi-host scale"},
}
for _, profile := range media {