sechmachine/VerseVDI-Data-Plane
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test(gateway): freeze v8 qualification harness
Verify Data Plane / gateway (push) Failing after 3m12s
Details

Browse Source
This commit is contained in:
sechmachine
2026-08-09 17:50:02 +07:00
parent a491c4f733
commit a0ca194691
8 changed files with 489 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
gateway/qualification_harness_test.go
+137 -58
View File
@@ -31,7 +31,6 @@ import (
"strings"
"sync"
"sync/atomic"
"syscall"
"testing"
"time"
@@ -39,7 +38,7 @@ import (
)
const (
qualificationToolVersion = "versevdi-gateway-qualification/v7"
qualificationToolVersion = "versevdi-gateway-qualification/v8"
qualificationImpairmentQueuePackets = nativeApolloVideoQueuePackets
qualificationImpairmentMaxPackets = 100_000
qualificationImpairmentPacketCount = 10_000
@@ -76,20 +75,39 @@ type qualificationPathTrace struct {
PayloadPreserved bool
}
type qualificationVideoBatchObservation struct {
ProviderFrame uint32
SourcePacket uint64
PacketWithinFrame int
StartedAfter time.Duration
}
type qualificationPath struct {
client *independentGatewayClient
server *Server
session *nativeApolloSession
fixture *qualificationApolloFixture
backend *qualificationTracingBackend
process *qualificationGatewayProcess
key []byte
flow string
frame uint32
bootTrace atomic.Bool
sourceUDP atomic.Uint64
closeOnce sync.Once
shutdown func()
client *independentGatewayClient
server *Server
session *nativeApolloSession
fixture *qualificationApolloFixture
backend *qualificationTracingBackend
process *qualificationGatewayProcess
key []byte
flow string
frame uint32
bootTrace atomic.Bool
sourceUDP atomic.Uint64
expectedSourceUDP atomic.Uint64
closeOnce sync.Once
shutdown func()
}
type qualificationProcessingDiagnostics struct {
ProcessedFrames int64
SourceFrames uint32
ExpectedWritesThroughLastFrame uint64
FixtureSentPackets uint64
SourceUDP uint64
BatchHistory []qualificationVideoBatchObservation
Gateway qualificationGatewayProcessSnapshot
SnapshotError string
}
type qualificationImpairmentProfile struct {
@@ -414,23 +432,29 @@ func (b *qualificationTracingBackend) session(sessionID string) *nativeApolloSes
}
type qualificationApolloFixture struct {
sessionID string
management *httptest.Server
stream net.Listener
control *net.UDPConn
audio *net.UDPConn
video *net.UDPConn
videoRemote atomic.Pointer[net.UDPAddr]
key atomic.Pointer[[]byte]
keyReady chan []byte
failures chan error
closed atomic.Bool
closeOnce sync.Once
sentPackets atomic.Uint64
work protocol.ProviderSessionWork
videoPaceMu sync.Mutex
videoNext time.Time
observeVideoBatch func(int, time.Time)
sessionID string
management *httptest.Server
stream net.Listener
control *net.UDPConn
audio *net.UDPConn
video *net.UDPConn
videoRemote atomic.Pointer[net.UDPAddr]
key atomic.Pointer[[]byte]
keyReady chan []byte
failures chan error
closed atomic.Bool
closeOnce sync.Once
sentPackets atomic.Uint64
work protocol.ProviderSessionWork
videoPaceMu sync.Mutex
videoNext time.Time
beforeVideoBatch func(context.Context, int) error
beforeVideoFirstWrite func(context.Context, int) error
observeVideoBatch func(int, time.Time)
videoBatchMu sync.Mutex
videoBatchEpoch time.Time
videoBatchCount uint64
videoBatches [128]qualificationVideoBatchObservation
controlImpairmentMu sync.Mutex
controlRTT time.Duration
@@ -438,6 +462,30 @@ type qualificationApolloFixture struct {
controlRandom uint64
}
func (f *qualificationApolloFixture) videoBatchHistory() []qualificationVideoBatchObservation {
f.videoBatchMu.Lock()
defer f.videoBatchMu.Unlock()
count := min(f.videoBatchCount, uint64(len(f.videoBatches)))
result := make([]qualificationVideoBatchObservation, 0, count)
for index := f.videoBatchCount - count; index < f.videoBatchCount; index++ {
result = append(result, f.videoBatches[index%uint64(len(f.videoBatches))])
}
return result
}
func (f *qualificationApolloFixture) recordVideoBatch(providerFrame uint32, sourcePacket uint64, packetWithinFrame int, started time.Time) {
f.videoBatchMu.Lock()
if f.videoBatchEpoch.IsZero() {
f.videoBatchEpoch = started
}
f.videoBatches[f.videoBatchCount%uint64(len(f.videoBatches))] = qualificationVideoBatchObservation{
ProviderFrame: providerFrame, SourcePacket: sourcePacket,
PacketWithinFrame: packetWithinFrame, StartedAfter: started.Sub(f.videoBatchEpoch),
}
f.videoBatchCount++
f.videoBatchMu.Unlock()
}
func newQualificationApolloFixture(t *testing.T, serverTLS, clientTLS *tls.Config, sessionID string, profile qualificationMediaProfile) *qualificationApolloFixture {
t.Helper()
fixture := &qualificationApolloFixture{sessionID: sessionID, keyReady: make(chan []byte, 1), failures: make(chan error, 8)}
@@ -749,7 +797,7 @@ func (f *qualificationApolloFixture) serveMedia(socket *net.UDPConn, video bool)
}
}
func (f *qualificationApolloFixture) sendVideo(ctx context.Context, packets [][]byte) error {
func (f *qualificationApolloFixture) sendVideo(ctx context.Context, providerFrame uint32, packets [][]byte) error {
for f.videoRemote.Load() == nil {
select {
case err := <-f.failures:
@@ -770,21 +818,37 @@ func (f *qualificationApolloFixture) sendVideo(ctx context.Context, packets [][]
f.videoPaceMu.Lock()
defer f.videoPaceMu.Unlock()
frameStart := time.Now()
if f.videoNext.After(frameStart) {
frameStart = f.videoNext
}
framePackets := 0
for batchStart := 0; batchStart < len(packets); batchStart += batchSize {
batchEnd := min(batchStart+batchSize, len(packets))
due := frameStart.Add(qualificationApolloVideoOffset(framePackets, packetsPerMillisecond))
if err := qualificationWaitContext(ctx, due); err != nil {
sourceStart := f.sentPackets.Load()
if err := qualificationWaitContext(ctx, f.videoNext); err != nil {
return err
}
if f.observeVideoBatch != nil {
f.observeVideoBatch(framePackets, time.Now())
if f.beforeVideoBatch != nil {
if err := f.beforeVideoBatch(ctx, framePackets); err != nil {
return err
}
}
for _, packet := range packets[batchStart:batchEnd] {
if f.beforeVideoFirstWrite != nil {
if err := f.beforeVideoFirstWrite(ctx, framePackets); err != nil {
return err
}
}
firstPacket := packets[batchStart]
if len(firstPacket) != len(packets[0]) {
return ErrProviderMalformed
}
if _, err := f.video.WriteToUDP(firstPacket, remote); err != nil {
return err
}
batchStarted := time.Now()
f.sentPackets.Add(1)
f.recordVideoBatch(providerFrame, sourceStart, framePackets, batchStarted)
if f.observeVideoBatch != nil {
f.observeVideoBatch(framePackets, batchStarted)
}
for _, packet := range packets[batchStart+1 : batchEnd] {
if len(packet) != len(packets[0]) {
return ErrProviderMalformed
}
@@ -795,8 +859,8 @@ func (f *qualificationApolloFixture) sendVideo(ctx context.Context, packets [][]
}
currentBatch := batchEnd - batchStart
framePackets += currentBatch
f.videoNext = batchStarted.Add(qualificationApolloVideoOffset(currentBatch, packetsPerMillisecond))
}
f.videoNext = frameStart.Add(qualificationApolloVideoOffset(framePackets, packetsPerMillisecond))
return nil
}
@@ -1145,15 +1209,31 @@ func (p *qualificationPath) emit(t *testing.T, payload []byte) (qualificationPat
}
p.frame++
packets := qualificationSourceVideoPackets(t, p.key, p.frame, payload)
p.expectedSourceUDP.Add(uint64(len(packets)))
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := p.fixture.sendVideo(ctx, packets); err != nil {
if err := p.fixture.sendVideo(ctx, p.frame, packets); err != nil {
return qualificationPathTrace{}, err
}
p.sourceUDP.Add(uint64(len(packets)))
return qualificationPathTrace{}, nil
}
func (p *qualificationPath) processingDiagnostics(processed int64) qualificationProcessingDiagnostics {
diagnostics := qualificationProcessingDiagnostics{
ProcessedFrames: processed, SourceFrames: p.frame,
ExpectedWritesThroughLastFrame: p.expectedSourceUDP.Load(),
FixtureSentPackets: p.fixture.sentPackets.Load(), SourceUDP: p.sourceUDP.Load(),
BatchHistory: p.fixture.videoBatchHistory(),
}
var err error
diagnostics.Gateway, err = p.process.snapshot()
if err != nil {
diagnostics.SnapshotError = err.Error()
}
return diagnostics
}
func (p *qualificationPath) receivePayload(parent context.Context) ([]byte, error) {
ctx, cancel := context.WithTimeout(parent, 2*time.Second)
defer cancel()
@@ -1711,14 +1791,21 @@ func qualificationMaximumDeliveryBytes(deliveries []qualificationDeliverySample,
return maximum
}
func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile, rawPath string) (qualificationProcessingSummary, error) {
func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile, rawPath string) (summary qualificationProcessingSummary, err error) {
t.Helper()
payload := qualificationFramePayload(profile, 0)
if len(payload) < 4 {
return qualificationProcessingSummary{}, errors.New("qualification payload too small")
}
path := newQualificationProcessingPath(t, profile, qualificationFramePacerKbps(profile))
defer path.Close()
var processed int64
defer func() {
if err != nil {
diagnostics := path.processingDiagnostics(processed)
err = fmt.Errorf("%w; diagnostics=%#v", err, diagnostics)
}
path.Close()
}()
if err := runQualificationProcessWarmup(t, path, profile, payload); err != nil {
return qualificationProcessingSummary{}, err
}
@@ -1773,15 +1860,13 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
}
receivedDone <- nil
}()
var processed int64
var nextRelease time.Time
payloadDigest := sha256.New()
for processed < targetFrames {
select {
case receiveErr := <-receivedDone:
if receiveErr != nil {
snapshot, _ := path.process.snapshot()
return qualificationProcessingSummary{}, fmt.Errorf("%w; gateway snapshot=%#v", receiveErr, snapshot)
return qualificationProcessingSummary{}, receiveErr
}
return qualificationProcessingSummary{}, errors.New("qualification processing receiver ended early")
default:
@@ -1799,8 +1884,7 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
processed++
}
if err := <-receivedDone; err != nil {
snapshot, _ := path.process.snapshot()
return qualificationProcessingSummary{}, fmt.Errorf("%w; gateway snapshot=%#v", err, snapshot)
return qualificationProcessingSummary{}, err
}
qualificationWaitUntil(started.Add(profile.Duration))
actualDuration := time.Since(started)
@@ -1827,7 +1911,7 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
if err != nil {
return qualificationProcessingSummary{}, err
}
summary, err := summarizeQualificationSamples(samples)
summary, err = summarizeQualificationSamples(samples)
if err != nil {
return qualificationProcessingSummary{}, err
}
@@ -1950,12 +2034,7 @@ func runQualificationWarmup(t *testing.T, path *qualificationPath, profile quali
}
func qualificationRuntimeSample(started time.Time) qualificationResourceSample {
var usage syscall.Rusage
cpuSeconds := -1.0
if syscall.Getrusage(syscall.RUSAGE_SELF, &usage) == nil {
cpuSeconds = float64(usage.Utime.Sec+usage.Stime.Sec) +
float64(usage.Utime.Usec+usage.Stime.Usec)/1_000_000
}
cpuSeconds := qualificationProcessCPUSeconds()
samples := []runtimemetrics.Sample{
{Name: "/memory/classes/heap/objects:bytes"},
{Name: "/sched/goroutines:goroutines"},