fix(gateway): close Phase 3C audit gaps

2026-07-30 01:46:00 +07:00
parent 040ca73ce9
commit d3852d15f3
23 changed files with 1619 additions and 219 deletions
@@ -90,6 +90,8 @@ func heartbeatLoop(ctx context.Context, client *gateway.ControlPlaneClient, serv
 	ticker := time.NewTicker(2 * time.Second)
 	defer ticker.Stop()
 	var sequence int64
+	var sampler heartbeatSampler
+	_, _ = sampler.sample(time.Now(), server.Metrics())
 	for {
 		select {
 		case <-ctx.Done():
@@ -104,11 +106,82 @@ func heartbeatLoop(ctx context.Context, client *gateway.ControlPlaneClient, serv
 				state = "draining"
 			}
 			metrics := server.Metrics()
-			_ = client.Heartbeat(ctx, protocol.GatewayHeartbeat{Version: "1", GatewayID: registration.GatewayID, Sequence: sequence, ObservedAt: time.Now().UTC().Format(time.RFC3339Nano), ActiveConnections: metrics.ActiveSessions, EgressKbps: registration.BandwidthCapacityKbps, State: state})
+			observedAt := time.Now()
+			egressKbps, telemetry := sampler.sample(observedAt, metrics)
+			_ = client.Heartbeat(ctx, protocol.GatewayHeartbeat{
+				Version: "1", GatewayID: registration.GatewayID, Sequence: sequence,
+				ObservedAt: observedAt.UTC().Format(time.RFC3339Nano), ActiveConnections: metrics.ActiveSessions,
+				EgressKbps: egressKbps, State: state, Telemetry: telemetry,
+			})
 		}
 	}
 }

+type heartbeatSampler struct {
+	observedAt time.Time
+	mediaBytes uint64
+}
+
+func (s *heartbeatSampler) sample(observedAt time.Time, metrics gateway.MetricsSnapshot) (int64, protocol.GatewayTelemetry) {
+	egressKbps := int64(0)
+	elapsedMillis := observedAt.Sub(s.observedAt).Milliseconds()
+	if !s.observedAt.IsZero() && elapsedMillis > 0 && metrics.MediaBytes >= s.mediaBytes {
+		delta := metrics.MediaBytes - s.mediaBytes
+		milliseconds := uint64(elapsedMillis)
+		whole, remainder := delta/milliseconds, delta%milliseconds
+		if whole > 125_000_000 {
+			egressKbps = 1_000_000_000
+		} else {
+			rate := whole*8 + remainder*8/milliseconds
+			if rate > 1_000_000_000 {
+				rate = 1_000_000_000
+			}
+			egressKbps = int64(rate)
+		}
+	}
+	s.observedAt, s.mediaBytes = observedAt, metrics.MediaBytes
+	return egressKbps, protocol.GatewayTelemetry{
+		AdmittedSessions: boundedMetric(metrics.AdmittedSessions), AdmissionRejects: boundedMetric(metrics.AdmissionRejects),
+		Reconnects: boundedMetric(metrics.Reconnects), DrainTransitions: boundedMetric(metrics.DrainTransitions),
+		MediaDrops: boundedMetric(metrics.MediaDrops), MediaPackets: boundedMetric(metrics.MediaPackets),
+		MediaBytes: boundedMetric(metrics.MediaBytes), QueueDelayMicros: boundedMetric(metrics.QueueDelayNanos / 1000),
+		ProcessingDelayMicros: boundedMetric(metrics.ProcessingDelayNanos / 1000), ProcessingSamples: boundedMetric(metrics.ProcessingSamples),
+		PacingDelayMicros: boundedMetric(metrics.PacingDelayNanos / 1000), ProviderErrors: boundedMetric(metrics.ProviderErrors),
+		InputRejected: boundedMetric(metrics.InputRejected), ControlRttMicros: boundedMetric(metrics.ControlRTTNanos / 1000),
+		ControlJitterMicros: boundedMetric(metrics.ControlJitterNanos / 1000), ControlLossPpm: boundedMetric(metrics.ControlLossPPM),
+		PendingReliable: boundedMetric(metrics.PendingReliable), ProviderState: providerStateName(metrics.ProviderState),
+	}
+}
+
+func boundedMetric(value uint64) int64 {
+	const maximum = uint64(^uint64(0) >> 1)
+	if value > maximum {
+		return int64(maximum)
+	}
+	return int64(value)
+}
+
+func providerStateName(value uint64) string {
+	switch value {
+	case 1:
+		return gateway.ProviderStateStarting
+	case 2:
+		return gateway.ProviderStateReady
+	case 3:
+		return gateway.ProviderStateDisconnected
+	case 4:
+		return gateway.ProviderStateTerminating
+	case 5:
+		return gateway.ProviderStateTerminated
+	case 6:
+		return gateway.ProviderStateCleanup
+	case 7:
+		return gateway.ProviderStateFailed
+	default:
+		return "unknown"
+	}
+}
+
 func loadTLS(certFile, keyFile, clientCAFile string) (*tls.Config, *tls.Config, error) {
 	certificate, err := tls.LoadX509KeyPair(certFile, keyFile)
 	if err != nil {
@@ -0,0 +1,116 @@
+package main
+
+import (
+	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"crypto/tls"
+	"crypto/x509"
+	"crypto/x509/pkix"
+	"io"
+	"math/big"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+
+	"git.sechmachine.io.vn/sechmachine/VerseVDI-Data-Plane/gateway"
+	protocol "git.sechmachine.io.vn/sechmachine/VerseVDI-Protocol/gen/go/protocol"
+)
+
+func TestHeartbeatReportsMeasuredEgressInsteadOfConfiguredCapacity(t *testing.T) {
+	heartbeats := make(chan protocol.GatewayHeartbeat, 1)
+	control := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
+		if request.URL.Path == "/api/v1/gateway/heartbeat" {
+			heartbeat, err := protocol.DecodeGatewayHeartbeat(mustReadBody(t, request))
+			if err != nil {
+				t.Errorf("decode heartbeat: %v", err)
+			} else {
+				heartbeats <- heartbeat
+			}
+		}
+		response.WriteHeader(http.StatusNoContent)
+	}))
+	defer control.Close()
+
+	server, err := gateway.NewServer(gateway.ServerConfig{
+		ListenAddress: "127.0.0.1:0", TLSConfig: heartbeatTestTLS(t), GatewayID: "gateway-1",
+		Admission: gateway.AdmissionFunc(func(context.Context, protocol.TunnelAdmissionRequest) (protocol.SessionAuthority, error) {
+			return protocol.SessionAuthority{}, gateway.ErrAdmissionRejected
+		}),
+		Provider: gateway.NewFakeApollo(gateway.FakeApolloConfig{}),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer server.Close()
+	registration := protocol.GatewayRegistration{GatewayID: "gateway-1", BandwidthCapacityKbps: 100000}
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	go heartbeatLoop(ctx, gateway.NewControlPlaneClient(control.URL, control.Client()), server, registration)
+
+	select {
+	case heartbeat := <-heartbeats:
+		cancel()
+		if heartbeat.EgressKbps != 0 {
+			t.Fatalf("idle measured egress = %d Kbps, want 0; configured capacity is not traffic", heartbeat.EgressKbps)
+		}
+	case <-time.After(3 * time.Second):
+		t.Fatal("heartbeat was not published")
+	}
+}
+
+func TestHeartbeatSamplerUsesByteDeltaAndMonotonicElapsed(t *testing.T) {
+	var sampler heartbeatSampler
+	start := time.Now()
+	if egress, _ := sampler.sample(start, gateway.MetricsSnapshot{MediaBytes: 1000}); egress != 0 {
+		t.Fatalf("first sample egress = %d, want baseline 0", egress)
+	}
+	egress, telemetry := sampler.sample(start.Add(2*time.Second), gateway.MetricsSnapshot{
+		AdmittedSessions: 2, AdmissionRejects: 3, Reconnects: 4, DrainTransitions: 5,
+		MediaDrops: 6, MediaPackets: 7, MediaBytes: 17000, QueueDelayNanos: 9000,
+		ProcessingDelayNanos: 10000, ProcessingSamples: 11, PacingDelayNanos: 12000,
+		ProviderErrors: 13, InputRejected: 14, ControlRTTNanos: 15000,
+		ControlJitterNanos: 16000, ControlLossPPM: 17, PendingReliable: 18, ProviderState: 2,
+	})
+	if egress != 64 || telemetry.MediaBytes != 17000 || telemetry.MediaPackets != 7 ||
+		telemetry.QueueDelayMicros != 9 || telemetry.ProviderState != gateway.ProviderStateReady {
+		t.Fatalf("sample = egress:%d telemetry:%#v", egress, telemetry)
+	}
+}
+
+func mustReadBody(t *testing.T, request *http.Request) []byte {
+	t.Helper()
+	defer request.Body.Close()
+	data, err := io.ReadAll(request.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return data
+}
+
+func heartbeatTestTLS(t *testing.T) *tls.Config {
+	t.Helper()
+	key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatal(err)
+	}
+	template := &x509.Certificate{
+		SerialNumber: big.NewInt(1), Subject: pkix.Name{CommonName: "gateway.test"},
+		NotBefore: time.Now().Add(-time.Hour), NotAfter: time.Now().Add(time.Hour),
+		IsCA: true, BasicConstraintsValid: true, KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature,
+		ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
+	}
+	der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key)
+	if err != nil {
+		t.Fatal(err)
+	}
+	certificate := tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}
+	pool := x509.NewCertPool()
+	pool.AddCert(template)
+	return &tls.Config{
+		MinVersion: tls.VersionTLS13, Certificates: []tls.Certificate{certificate},
+		ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: pool,
+	}
+}