test(gateway): bind qualification to immutable Protocol
This commit is contained in:
@@ -39,6 +39,21 @@ func TestQualificationCatalogMatchesSection7(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestQualificationProtocolVersionIsExplicitAndImmutable(t *testing.T) {
|
||||||
|
const version = "v1.0.0-phase3c-gateway-rc.6"
|
||||||
|
t.Setenv("VERSEVDI_QUALIFICATION_PROTOCOL_VERSION", version)
|
||||||
|
got, err := qualificationProtocolVersion()
|
||||||
|
if err != nil || got != version {
|
||||||
|
t.Fatalf("qualificationProtocolVersion() = %q, want %q", got, version)
|
||||||
|
}
|
||||||
|
for _, invalid := range []string{"", "unknown", "v1", " v1.0.0", "v1.0.0+mutable"} {
|
||||||
|
t.Setenv("VERSEVDI_QUALIFICATION_PROTOCOL_VERSION", invalid)
|
||||||
|
if _, err := qualificationProtocolVersion(); err == nil {
|
||||||
|
t.Fatalf("qualificationProtocolVersion() accepted %q", invalid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestQualificationOutputAndStatisticsFailClosed(t *testing.T) {
|
func TestQualificationOutputAndStatisticsFailClosed(t *testing.T) {
|
||||||
if err := validateQualificationOutputDir("relative/evidence"); err == nil {
|
if err := validateQualificationOutputDir("relative/evidence"); err == nil {
|
||||||
t.Fatal("relative evidence directory was accepted")
|
t.Fatal("relative evidence directory was accepted")
|
||||||
|
|||||||
@@ -14,8 +14,8 @@ import (
|
|||||||
"math"
|
"math"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"regexp"
|
||||||
"runtime"
|
"runtime"
|
||||||
"runtime/debug"
|
|
||||||
"sort"
|
"sort"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -585,17 +585,13 @@ func qualificationMaximumFiveSecondBytes(deliveries []syntheticPacerDelivery) in
|
|||||||
return maximum
|
return maximum
|
||||||
}
|
}
|
||||||
|
|
||||||
func qualificationProtocolVersion() string {
|
func qualificationProtocolVersion() (string, error) {
|
||||||
info, ok := debug.ReadBuildInfo()
|
version := os.Getenv("VERSEVDI_QUALIFICATION_PROTOCOL_VERSION")
|
||||||
if !ok {
|
valid, err := regexp.MatchString(`^v[0-9]+\.[0-9]+\.[0-9]+-[0-9A-Za-z]+(?:[.-][0-9A-Za-z]+)*$`, version)
|
||||||
return "unknown"
|
if err != nil || !valid {
|
||||||
|
return "", errors.New("VERSEVDI_QUALIFICATION_PROTOCOL_VERSION must be an immutable prerelease tag")
|
||||||
}
|
}
|
||||||
for _, dependency := range info.Deps {
|
return version, nil
|
||||||
if dependency.Path == "git.sechmachine.io.vn/sechmachine/VerseVDI-Protocol" {
|
|
||||||
return dependency.Version
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return "unknown"
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func qualificationCandidateCommit() (string, error) {
|
func qualificationCandidateCommit() (string, error) {
|
||||||
@@ -633,6 +629,10 @@ func TestSection7Qualification(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal(err)
|
t.Fatal(err)
|
||||||
}
|
}
|
||||||
|
protocolVersion, err := qualificationProtocolVersion()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
if err := os.Mkdir(output, 0o750); err != nil {
|
if err := os.Mkdir(output, 0o750); err != nil {
|
||||||
t.Fatalf("create new qualification evidence directory: %v", err)
|
t.Fatalf("create new qualification evidence directory: %v", err)
|
||||||
}
|
}
|
||||||
@@ -641,8 +641,11 @@ func TestSection7Qualification(t *testing.T) {
|
|||||||
qualificationTraverseProfiles(t, media)
|
qualificationTraverseProfiles(t, media)
|
||||||
manifest := qualificationManifest{
|
manifest := qualificationManifest{
|
||||||
Status: "running", ToolVersion: qualificationToolVersion,
|
Status: "running", ToolVersion: qualificationToolVersion,
|
||||||
Command: fmt.Sprintf("VERSEVDI_QUALIFICATION_DIR=%s VERSEVDI_QUALIFICATION_COMMIT=%s GOWORK=off go test ./gateway -run '^TestSection7Qualification$' -count=1 -timeout 45m -v", output, commit),
|
Command: fmt.Sprintf(
|
||||||
CandidateCommit: commit, ProtocolVersion: qualificationProtocolVersion(),
|
"VERSEVDI_QUALIFICATION_DIR=%s VERSEVDI_QUALIFICATION_COMMIT=%s VERSEVDI_QUALIFICATION_PROTOCOL_VERSION=%s GOWORK=off go test ./gateway -run '^TestSection7Qualification$' -count=1 -timeout 45m -v",
|
||||||
|
output, commit, protocolVersion,
|
||||||
|
),
|
||||||
|
CandidateCommit: commit, ProtocolVersion: protocolVersion,
|
||||||
StartedAt: started.Format(time.RFC3339Nano), GoVersion: runtime.Version(),
|
StartedAt: started.Format(time.RFC3339Nano), GoVersion: runtime.Version(),
|
||||||
OS: runtime.GOOS, Architecture: runtime.GOARCH,
|
OS: runtime.GOOS, Architecture: runtime.GOARCH,
|
||||||
Topology: "bounded fixture provider -> gateway framing and mTLS/QUIC transport -> fixture client",
|
Topology: "bounded fixture provider -> gateway framing and mTLS/QUIC transport -> fixture client",
|
||||||
|
|||||||
+2
-2
@@ -18,7 +18,7 @@
|
|||||||
|
|
||||||
- [x] 3.1 Run focused red/green checks, strict OpenSpec validation, `make
|
- [x] 3.1 Run focused red/green checks, strict OpenSpec validation, `make
|
||||||
verify`, race/fuzz/resource checks, and freeze the harness commit.
|
verify`, race/fuzz/resource checks, and freeze the harness commit.
|
||||||
- [ ] 3.2 Run the opt-in ten-minute and six-profile command exactly once
|
- [x] 3.2 Run the opt-in ten-minute and six-profile command exactly once
|
||||||
against the frozen candidate and archive raw artifacts and hashes.
|
against the frozen candidate and archive raw artifacts and hashes.
|
||||||
- [ ] 3.3 Sync the canonical specification, archive the completed change, and
|
- [x] 3.3 Sync the canonical specification, archive the completed change, and
|
||||||
revalidate strictly without claiming live Apollo/macOS/firewall evidence.
|
revalidate strictly without claiming live Apollo/macOS/firewall evidence.
|
||||||
@@ -0,0 +1,67 @@
|
|||||||
|
# gateway-qualification Specification
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
Define the deterministic processing, impairment, pacing, and evidence boundaries
|
||||||
|
for qualifying a frozen Phase 3C gateway candidate.
|
||||||
|
## Requirements
|
||||||
|
### Requirement: Fixed media processing qualification
|
||||||
|
The qualification harness SHALL run 1080p60 H.264 at 20 Mbps, 1440p120 HEVC
|
||||||
|
at 50 Mbps, and 4K60 HEVC at 80 Mbps for ten wall-clock minutes each after a
|
||||||
|
recorded warm-up. It SHALL preserve encoded payload bytes, record every
|
||||||
|
monotonic processing sample, report count, min, median, p90, p95, p99, max,
|
||||||
|
mean, standard deviation, timing overhead, and observed bitrate, and fail when
|
||||||
|
any p95 exceeds 5 ms.
|
||||||
|
|
||||||
|
#### Scenario: Healthy fixed profile
|
||||||
|
- **WHEN** a frozen candidate runs one fixed profile for the normative duration
|
||||||
|
- **THEN** the harness emits compressed raw samples and a summary tied to the
|
||||||
|
exact source commit, Protocol version, environment, and payload hash.
|
||||||
|
|
||||||
|
#### Scenario: Processing gate failure
|
||||||
|
- **WHEN** payload integrity fails or measured p95 exceeds 5 ms
|
||||||
|
- **THEN** the qualification command exits unsuccessfully without recording a
|
||||||
|
passing candidate.
|
||||||
|
|
||||||
|
### Requirement: Bounded impairment qualification
|
||||||
|
The harness SHALL run exactly the baseline, latency, jitter, loss, reorder,
|
||||||
|
and constrained Section 7.2 profiles once. Baseline SHALL cover all three
|
||||||
|
media profiles and the other profiles SHALL cover 1080p60. Each artifact SHALL
|
||||||
|
record tool version, exact command/configuration, direction, queue discipline,
|
||||||
|
topology, fixed seed, and observed RTT, jitter, loss, reorder, throughput,
|
||||||
|
drops, and capacity-step statistics.
|
||||||
|
|
||||||
|
#### Scenario: Complete six-profile run
|
||||||
|
- **WHEN** the frozen candidate runs impairment qualification
|
||||||
|
- **THEN** one result exists for each named profile, with no Cartesian
|
||||||
|
expansion and with observed rather than configured statistics.
|
||||||
|
|
||||||
|
#### Scenario: Unsupported or unbounded configuration
|
||||||
|
- **WHEN** a profile name, packet count, queue bound, loss, reorder, or
|
||||||
|
bandwidth step falls outside the fixed catalog
|
||||||
|
- **THEN** the harness rejects it before allocating or running the simulation.
|
||||||
|
|
||||||
|
### Requirement: Fairness and cap qualification
|
||||||
|
The harness SHALL exercise the production fair pacer with eight equal-tier
|
||||||
|
synthetic sessions for the required 60-second virtual interval, report every
|
||||||
|
share error and Jain's fairness index, and fail above 10% share error. It SHALL
|
||||||
|
apply 25% and 50% capacity steps, fail convergence beyond ten virtual seconds,
|
||||||
|
and fail aggregate egress above 105% of the cap over any rolling five-second
|
||||||
|
window.
|
||||||
|
|
||||||
|
#### Scenario: Equal-tier and capacity-step evidence
|
||||||
|
- **WHEN** the frozen candidate runs scheduler qualification
|
||||||
|
- **THEN** the artifact contains per-flow bytes, share errors, Jain's index,
|
||||||
|
step convergence, and rolling cap observations derived from the production
|
||||||
|
pacer.
|
||||||
|
|
||||||
|
### Requirement: Honest qualification boundary
|
||||||
|
Qualification artifacts SHALL contain no provider endpoint, credential,
|
||||||
|
clipboard text, input payload, secret, raw media content, or claim of live
|
||||||
|
Apollo/macOS/firewall interoperability. The harness SHALL add no codec
|
||||||
|
operation, production dependency, cgo, sidecar, or direct provider route.
|
||||||
|
|
||||||
|
#### Scenario: Deterministic evidence publication
|
||||||
|
- **WHEN** qualification completes
|
||||||
|
- **THEN** the manifest labels fake-provider, virtual impairment, and local
|
||||||
|
processing evidence separately and leaves live interoperability
|
||||||
|
deferred-owner-e2e.
|
||||||
Reference in New Issue
Block a user