test(gateway): qualify production path artifacts
This commit is contained in:
sechmachine
@@ -25,12 +25,12 @@ import (
|
||||
"regexp"
|
||||
"runtime"
|
||||
"runtime/debug"
|
||||
runtimemetrics "runtime/metrics"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@@ -38,7 +38,7 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
qualificationToolVersion = "versevdi-gateway-qualification/v3"
|
||||
qualificationToolVersion = "versevdi-gateway-qualification/v4"
|
||||
qualificationImpairmentQueuePackets = 64
|
||||
qualificationImpairmentMaxPackets = 100_000
|
||||
qualificationImpairmentPacketCount = 10_000
|
||||
@@ -116,6 +116,7 @@ type qualificationProcessingSummary struct {
|
||||
RawSamplesBytes int64 `json:"raw_samples_bytes"`
|
||||
ResourceSamples int `json:"resource_samples"`
|
||||
CPUSeconds float64 `json:"cpu_seconds"`
|
||||
CPUScope string `json:"cpu_scope"`
|
||||
PeakHeapBytes uint64 `json:"peak_heap_bytes"`
|
||||
PeakGoroutines int `json:"peak_goroutines"`
|
||||
Mallocs uint64 `json:"mallocs"`
|
||||
@@ -132,9 +133,12 @@ type qualificationImpairmentObservation struct {
|
||||
Sent int `json:"sent"`
|
||||
Delivered int `json:"delivered"`
|
||||
Dropped int `json:"dropped"`
|
||||
InjectedDropped int `json:"injected_dropped"`
|
||||
InjectedReordered int `json:"injected_reordered"`
|
||||
ObservedOutOfOrder int `json:"observed_out_of_order"`
|
||||
ObservedLatency time.Duration `json:"observed_one_way_latency_ns"`
|
||||
ObservedRTT time.Duration `json:"observed_rtt_ns"`
|
||||
RTTSource string `json:"rtt_source"`
|
||||
ObservedJitter time.Duration `json:"observed_jitter_ns"`
|
||||
ObservedLossPercent float64 `json:"observed_loss_percent"`
|
||||
ObservedReorderPercent float64 `json:"observed_reorder_percent"`
|
||||
@@ -304,6 +308,11 @@ type qualificationApolloFixture struct {
|
||||
closeOnce sync.Once
|
||||
sentPackets atomic.Uint64
|
||||
work protocol.ProviderSessionWork
|
||||
|
||||
controlImpairmentMu sync.Mutex
|
||||
controlRTT time.Duration
|
||||
controlJitter time.Duration
|
||||
controlRandom uint64
|
||||
}
|
||||
|
||||
func newQualificationApolloFixture(t *testing.T, serverTLS, clientTLS *tls.Config, sessionID string, profile qualificationMediaProfile) *qualificationApolloFixture {
|
||||
@@ -551,10 +560,7 @@ func (f *qualificationApolloFixture) serveControl() {
|
||||
switch command {
|
||||
case 1:
|
||||
case apolloENetSendReliable, apolloENetPing, apolloENetDisconnect:
|
||||
if _, err = f.control.WriteToUDP(sourceShapedENetAcknowledgePacket(7, 2, channel, sequence), remote); err != nil {
|
||||
f.fail(err)
|
||||
return
|
||||
}
|
||||
f.sendControlAcknowledge(sourceShapedENetAcknowledgePacket(7, 2, channel, sequence), remote)
|
||||
if command == apolloENetDisconnect {
|
||||
return
|
||||
}
|
||||
@@ -566,6 +572,45 @@ func (f *qualificationApolloFixture) serveControl() {
|
||||
}
|
||||
}
|
||||
|
||||
func (f *qualificationApolloFixture) setControlImpairment(profile qualificationImpairmentProfile) {
|
||||
f.controlImpairmentMu.Lock()
|
||||
f.controlRTT = profile.RTT
|
||||
f.controlJitter = profile.Jitter
|
||||
f.controlRandom = qualificationImpairmentSeed
|
||||
f.controlImpairmentMu.Unlock()
|
||||
}
|
||||
|
||||
func (f *qualificationApolloFixture) controlResponseDelay() time.Duration {
|
||||
f.controlImpairmentMu.Lock()
|
||||
defer f.controlImpairmentMu.Unlock()
|
||||
delay := f.controlRTT
|
||||
if f.controlJitter > 0 {
|
||||
f.controlRandom ^= f.controlRandom << 13
|
||||
f.controlRandom ^= f.controlRandom >> 7
|
||||
f.controlRandom ^= f.controlRandom << 17
|
||||
width := uint64(f.controlJitter*2 + 1)
|
||||
delay += time.Duration(f.controlRandom%width) - f.controlJitter
|
||||
}
|
||||
return max(delay, 0)
|
||||
}
|
||||
|
||||
func (f *qualificationApolloFixture) sendControlAcknowledge(packet []byte, remote *net.UDPAddr) {
|
||||
delay := f.controlResponseDelay()
|
||||
copyPacket := append([]byte(nil), packet...)
|
||||
copyRemote := *remote
|
||||
go func() {
|
||||
timer := time.NewTimer(delay)
|
||||
defer timer.Stop()
|
||||
<-timer.C
|
||||
if f.closed.Load() {
|
||||
return
|
||||
}
|
||||
if _, err := f.control.WriteToUDP(copyPacket, ©Remote); err != nil {
|
||||
f.fail(err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
func (f *qualificationApolloFixture) serveMedia(socket *net.UDPConn, video bool) {
|
||||
buffer := make([]byte, apolloMediaMaximumPacket)
|
||||
for {
|
||||
@@ -751,9 +796,20 @@ func (f *qualificationFleet) Close() {
|
||||
}
|
||||
|
||||
func newQualificationPath(t *testing.T, profile qualificationMediaProfile, pacerKbps int64) *qualificationPath {
|
||||
return newQualificationPathWithImpairment(t, profile, pacerKbps, nil)
|
||||
}
|
||||
|
||||
func newQualificationImpairedPath(t *testing.T, profile qualificationMediaProfile, pacerKbps int64, impairment qualificationImpairmentProfile) *qualificationPath {
|
||||
return newQualificationPathWithImpairment(t, profile, pacerKbps, &impairment)
|
||||
}
|
||||
|
||||
func newQualificationPathWithImpairment(t *testing.T, profile qualificationMediaProfile, pacerKbps int64, impairment *qualificationImpairmentProfile) *qualificationPath {
|
||||
t.Helper()
|
||||
serverTLS, clientTLS := testTLS(t)
|
||||
fixture := newQualificationApolloFixture(t, serverTLS, clientTLS, "qualification-session", profile)
|
||||
if impairment != nil {
|
||||
fixture.setControlImpairment(*impairment)
|
||||
}
|
||||
backend := &qualificationTracingBackend{native: NewNativeApolloBackend()}
|
||||
provider := NewApolloAdapter(backend, ProviderIdentity{})
|
||||
authority := protocol.SessionAuthority{
|
||||
@@ -1049,6 +1105,23 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
|
||||
if _, err := buffered.WriteString("source_sequence,sent_ns,delivered_ns,processing_ns,outcome,delivery_order,bytes,queue_packets\n"); err != nil {
|
||||
return qualificationImpairmentObservation{}, err
|
||||
}
|
||||
type scheduledPacket struct {
|
||||
index int
|
||||
target time.Duration
|
||||
}
|
||||
type rawSample struct {
|
||||
sent, delivered time.Duration
|
||||
processing time.Duration
|
||||
outcome string
|
||||
deliveryOrder int
|
||||
bytes int
|
||||
queuePackets int
|
||||
}
|
||||
type receivedPacket struct {
|
||||
index, deliveryOrder, queuePackets int
|
||||
deliveredAt time.Time
|
||||
processing time.Duration
|
||||
}
|
||||
state := qualificationImpairmentSeed
|
||||
random := func() uint64 {
|
||||
state ^= state << 13
|
||||
@@ -1065,22 +1138,126 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
|
||||
Sent: packetCount, ConfiguredRTT: profile.RTT, ConfiguredJitter: profile.Jitter,
|
||||
ConfiguredLossPercent: profile.LossPercent, ConfiguredReorder: profile.Reorder,
|
||||
ConfiguredCapacitySteps: append([]int(nil), profile.CapacitySteps...),
|
||||
RTTSource: "apollo_enet_acknowledge",
|
||||
}
|
||||
path := newQualificationPath(t, media, media.BitrateKbps)
|
||||
path := newQualificationImpairedPath(t, media, media.BitrateKbps, profile)
|
||||
defer path.Close()
|
||||
started := time.Now()
|
||||
payload := qualificationPayload(media)
|
||||
var deliveries []qualificationDeliverySample
|
||||
var totalRTT, totalJitter, previousRTT time.Duration
|
||||
previousDelivered := -1
|
||||
deliveryOrder := 0
|
||||
type pendingPacket struct {
|
||||
index int
|
||||
jitter time.Duration
|
||||
rawSamples := make([]rawSample, packetCount)
|
||||
jobs := make([]scheduledPacket, 0, packetCount)
|
||||
jobByIndex := make([]int, packetCount)
|
||||
for index := range jobByIndex {
|
||||
jobByIndex[index] = -1
|
||||
}
|
||||
pending := pendingPacket{index: -1}
|
||||
for index := 0; index < packetCount; index++ {
|
||||
jitter := time.Duration(0)
|
||||
if profile.Jitter > 0 {
|
||||
width := uint64(profile.Jitter*2 + 1)
|
||||
jitter = time.Duration(random()%width) - profile.Jitter
|
||||
}
|
||||
rawSamples[index].sent = time.Duration(index) * spacing
|
||||
if float64(random()%10_000) < profile.LossPercent*100 {
|
||||
rawSamples[index].outcome = "injected_dropped"
|
||||
observation.InjectedDropped++
|
||||
continue
|
||||
}
|
||||
delay := max(profile.RTT/2+jitter, 0)
|
||||
jobByIndex[index] = len(jobs)
|
||||
jobs = append(jobs, scheduledPacket{index: index, target: time.Duration(index)*spacing + delay})
|
||||
rawSamples[index].outcome = "traversal_dropped"
|
||||
}
|
||||
if profile.Reorder {
|
||||
for index := 18; index+1 < packetCount; index += 20 {
|
||||
first, second := jobByIndex[index], jobByIndex[index+1]
|
||||
if first < 0 || second < 0 {
|
||||
continue
|
||||
}
|
||||
earlier := min(jobs[first].target, jobs[second].target)
|
||||
later := max(jobs[first].target, jobs[second].target)
|
||||
jobs[second].target = earlier
|
||||
jobs[first].target = later + time.Nanosecond
|
||||
observation.InjectedReordered++
|
||||
}
|
||||
}
|
||||
sort.SliceStable(jobs, func(first, second int) bool {
|
||||
if jobs[first].target == jobs[second].target {
|
||||
return jobs[first].index < jobs[second].index
|
||||
}
|
||||
return jobs[first].target < jobs[second].target
|
||||
})
|
||||
|
||||
started := time.Now()
|
||||
beforeMetrics := path.server.Metrics()
|
||||
beforeIngress := path.session.mediaIngress.Load()
|
||||
beforeRecovered := path.session.mediaRecovered.Load()
|
||||
beforeEnqueued := path.session.mediaEnqueued.Load()
|
||||
beforePacer := path.server.pacer.reservations.Load()
|
||||
grace := max(2*profile.RTT+2*profile.Jitter, 2*time.Second)
|
||||
lastTarget := time.Duration(packetCount) * spacing
|
||||
if len(jobs) > 0 {
|
||||
lastTarget = jobs[len(jobs)-1].target
|
||||
}
|
||||
receiveCtx, receiveCancel := context.WithDeadline(context.Background(), started.Add(lastTarget+grace))
|
||||
defer receiveCancel()
|
||||
receivedDone := make(chan struct {
|
||||
packets []receivedPacket
|
||||
err error
|
||||
}, 1)
|
||||
go func() {
|
||||
result := struct {
|
||||
packets []receivedPacket
|
||||
err error
|
||||
}{packets: make([]receivedPacket, 0, len(jobs))}
|
||||
metrics := beforeMetrics
|
||||
seen := make([]bool, packetCount)
|
||||
for len(result.packets) < len(jobs) {
|
||||
recovered, receiveErr := path.receivePayload(receiveCtx)
|
||||
if receiveErr != nil {
|
||||
if errors.Is(receiveErr, context.DeadlineExceeded) || errors.Is(receiveErr, context.Canceled) {
|
||||
break
|
||||
}
|
||||
var timeout net.Error
|
||||
if errors.As(receiveErr, &timeout) && timeout.Timeout() {
|
||||
break
|
||||
}
|
||||
result.err = receiveErr
|
||||
break
|
||||
}
|
||||
if len(recovered) != len(payload) {
|
||||
result.err = errors.New("impaired payload length changed")
|
||||
break
|
||||
}
|
||||
index := int(binary.BigEndian.Uint32(recovered[len(recovered)-4:]))
|
||||
if index < 0 || index >= packetCount || jobByIndex[index] < 0 || seen[index] {
|
||||
result.err = errors.New("impaired payload sequence invalid")
|
||||
break
|
||||
}
|
||||
expected := append([]byte(nil), payload...)
|
||||
binary.BigEndian.PutUint32(expected[len(expected)-4:], uint32(index))
|
||||
if !bytes.Equal(recovered, expected) {
|
||||
result.err = errors.New("impaired payload bytes changed")
|
||||
break
|
||||
}
|
||||
seen[index] = true
|
||||
current := path.server.Metrics()
|
||||
processing := time.Duration(0)
|
||||
if samples := current.ProcessingSamples - metrics.ProcessingSamples; samples > 0 {
|
||||
processing = time.Duration((current.ProcessingDelayNanos - metrics.ProcessingDelayNanos) / samples)
|
||||
}
|
||||
metrics = current
|
||||
result.packets = append(result.packets, receivedPacket{
|
||||
index: index, deliveryOrder: len(result.packets) + 1,
|
||||
deliveredAt: time.Now(), processing: processing, queuePackets: len(path.session.video),
|
||||
})
|
||||
}
|
||||
receivedDone <- result
|
||||
}()
|
||||
|
||||
stepAt := make(map[int]time.Time, len(profile.CapacitySteps))
|
||||
deliver := func(packet pendingPacket) error {
|
||||
for _, packet := range jobs {
|
||||
if delay := time.Until(started.Add(packet.target)); delay > 0 {
|
||||
time.Sleep(delay)
|
||||
}
|
||||
if len(profile.CapacitySteps) == 2 {
|
||||
switch {
|
||||
case packet.index >= packetCount*2/3 && stepAt[profile.CapacitySteps[1]].IsZero():
|
||||
@@ -1091,79 +1268,67 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
|
||||
stepAt[profile.CapacitySteps[0]] = time.Now()
|
||||
}
|
||||
}
|
||||
sentAt := started.Add(time.Duration(packet.index) * spacing)
|
||||
target := sentAt.Add(profile.RTT/2 + packet.jitter)
|
||||
if delay := time.Until(target); delay > 0 {
|
||||
time.Sleep(delay)
|
||||
}
|
||||
current := append([]byte(nil), payload...)
|
||||
binary.BigEndian.PutUint32(current[len(current)-4:], uint32(packet.index))
|
||||
trace, processing, err := path.traverse(t, current)
|
||||
if err != nil || !trace.PayloadPreserved || !trace.ProductionPacer {
|
||||
if err == nil {
|
||||
err = errors.New("impaired packet bypassed production gateway path")
|
||||
}
|
||||
return err
|
||||
if _, err := path.emit(t, current); err != nil {
|
||||
receiveCancel()
|
||||
return qualificationImpairmentObservation{}, err
|
||||
}
|
||||
deliveredAt := time.Now()
|
||||
rtt := 2 * deliveredAt.Sub(sentAt)
|
||||
totalRTT += rtt
|
||||
if previousRTT != 0 {
|
||||
delta := rtt - previousRTT
|
||||
}
|
||||
received := <-receivedDone
|
||||
receiveCancel()
|
||||
if received.err != nil {
|
||||
return qualificationImpairmentObservation{}, received.err
|
||||
}
|
||||
|
||||
var deliveries []qualificationDeliverySample
|
||||
var totalLatency, totalJitter, previousLatency time.Duration
|
||||
previousDelivered := -1
|
||||
for _, packet := range received.packets {
|
||||
sample := &rawSamples[packet.index]
|
||||
sample.delivered = packet.deliveredAt.Sub(started)
|
||||
sample.processing = packet.processing
|
||||
sample.outcome = "delivered"
|
||||
sample.deliveryOrder = packet.deliveryOrder
|
||||
sample.bytes = media.PacketBytes
|
||||
sample.queuePackets = packet.queuePackets
|
||||
latency := sample.delivered - sample.sent
|
||||
totalLatency += latency
|
||||
if previousLatency != 0 {
|
||||
delta := latency - previousLatency
|
||||
if delta < 0 {
|
||||
delta = -delta
|
||||
}
|
||||
totalJitter += delta
|
||||
}
|
||||
previousRTT = rtt
|
||||
previousLatency = latency
|
||||
if previousDelivered >= 0 && packet.index < previousDelivered {
|
||||
observation.ObservedOutOfOrder++
|
||||
}
|
||||
previousDelivered = packet.index
|
||||
observation.Delivered++
|
||||
deliveryOrder++
|
||||
wireBytes := int64(len(current) + frameHeaderSize)
|
||||
deliveries = append(deliveries, qualificationDeliverySample{At: deliveredAt, Bytes: wireBytes})
|
||||
queuePackets := int(path.session.mediaQueueMaximum.Load())
|
||||
if _, err := fmt.Fprintf(buffered, "%d,%d,%d,%d,delivered,%d,%d,%d\n", packet.index,
|
||||
sentAt.Sub(started).Nanoseconds(), deliveredAt.Sub(started).Nanoseconds(),
|
||||
processing.Nanoseconds(), deliveryOrder, len(current), queuePackets); err != nil {
|
||||
return err
|
||||
}
|
||||
observation.MaxQueuePackets = max(observation.MaxQueuePackets, queuePackets)
|
||||
return nil
|
||||
deliveries = append(deliveries, qualificationDeliverySample{At: packet.deliveredAt, Bytes: int64(media.PacketBytes + frameHeaderSize)})
|
||||
observation.MaxQueuePackets = max(observation.MaxQueuePackets, packet.queuePackets)
|
||||
}
|
||||
for index := 0; index < packetCount; index++ {
|
||||
jitter := time.Duration(0)
|
||||
if profile.Jitter > 0 {
|
||||
width := uint64(profile.Jitter*2 + 1)
|
||||
jitter = time.Duration(random()%width) - profile.Jitter
|
||||
}
|
||||
if float64(random()%10_000) < profile.LossPercent*100 {
|
||||
observation.Dropped++
|
||||
if _, err := fmt.Fprintf(buffered, "%d,%d,0,0,dropped,0,0,%d\n", index, time.Duration(index)*spacing, path.session.mediaQueueMaximum.Load()); err != nil {
|
||||
return qualificationImpairmentObservation{}, err
|
||||
}
|
||||
continue
|
||||
}
|
||||
packet := pendingPacket{index: index, jitter: jitter}
|
||||
if profile.Reorder && index%20 == 18 {
|
||||
pending = packet
|
||||
continue
|
||||
}
|
||||
if err := deliver(packet); err != nil {
|
||||
return qualificationImpairmentObservation{}, err
|
||||
}
|
||||
if pending.index >= 0 {
|
||||
if err := deliver(pending); err != nil {
|
||||
return qualificationImpairmentObservation{}, err
|
||||
}
|
||||
pending.index = -1
|
||||
observation.InjectedReordered++
|
||||
}
|
||||
observation.Delivered = len(received.packets)
|
||||
observation.Dropped = packetCount - observation.Delivered
|
||||
completedAt := started
|
||||
if observation.Delivered > 0 {
|
||||
completedAt = received.packets[len(received.packets)-1].deliveredAt
|
||||
}
|
||||
if pending.index >= 0 {
|
||||
if err := deliver(pending); err != nil {
|
||||
afterMetrics := path.server.Metrics()
|
||||
if observation.Delivered > 0 && (path.session.mediaIngress.Load() <= beforeIngress ||
|
||||
path.session.mediaRecovered.Load() <= beforeRecovered ||
|
||||
path.session.mediaEnqueued.Load() <= beforeEnqueued ||
|
||||
afterMetrics.ProcessingSamples <= beforeMetrics.ProcessingSamples ||
|
||||
path.server.pacer.reservations.Load() <= beforePacer ||
|
||||
afterMetrics.MediaPackets <= beforeMetrics.MediaPackets) {
|
||||
return qualificationImpairmentObservation{}, errors.New("impaired traffic bypassed a production gateway stage")
|
||||
}
|
||||
observation.ObservedRTT = path.session.Telemetry().ControlRTT
|
||||
for index, sample := range rawSamples {
|
||||
if _, err := fmt.Fprintf(buffered, "%d,%d,%d,%d,%s,%d,%d,%d\n", index,
|
||||
sample.sent.Nanoseconds(), sample.delivered.Nanoseconds(), sample.processing.Nanoseconds(),
|
||||
sample.outcome, sample.deliveryOrder, sample.bytes, sample.queuePackets); err != nil {
|
||||
return qualificationImpairmentObservation{}, err
|
||||
}
|
||||
}
|
||||
@@ -1178,28 +1343,32 @@ func runQualificationImpairment(t *testing.T, profile qualificationImpairmentPro
|
||||
}
|
||||
closed = true
|
||||
if observation.Delivered > 0 {
|
||||
observation.ObservedRTT = totalRTT / time.Duration(observation.Delivered)
|
||||
observation.ObservedLatency = totalLatency / time.Duration(observation.Delivered)
|
||||
if observation.Delivered > 1 {
|
||||
observation.ObservedJitter = totalJitter / time.Duration(observation.Delivered-1)
|
||||
}
|
||||
observation.ObservedThroughputKbps = float64(observation.Delivered*media.PacketBytes*8) / time.Since(started).Seconds() / 1000
|
||||
observation.ObservedThroughputKbps = float64(observation.Delivered*media.PacketBytes*8) / completedAt.Sub(started).Seconds() / 1000
|
||||
}
|
||||
observation.ObservedLossPercent = float64(observation.Dropped) * 100 / float64(packetCount)
|
||||
observation.ObservedReorderPercent = float64(observation.ObservedOutOfOrder) * 100 / float64(packetCount)
|
||||
if packetCount >= qualificationImpairmentPacketCount {
|
||||
capacityFactor := 1.0
|
||||
if len(profile.CapacitySteps) > 0 {
|
||||
inverseRates := 1.0
|
||||
rates := 1.0
|
||||
for _, reduction := range profile.CapacitySteps {
|
||||
inverseRates += 100 / float64(100-reduction)
|
||||
rates += float64(100-reduction) / 100
|
||||
}
|
||||
capacityFactor = float64(len(profile.CapacitySteps)+1) / inverseRates
|
||||
capacityFactor = rates / float64(len(profile.CapacitySteps)+1)
|
||||
}
|
||||
expectedThroughput := float64(media.BitrateKbps) * (1 - profile.LossPercent/100) * capacityFactor
|
||||
lowerThroughput := expectedThroughput * 0.90
|
||||
upperThroughput := expectedThroughput * 1.05
|
||||
if observation.ObservedThroughputKbps < lowerThroughput || observation.ObservedThroughputKbps > upperThroughput {
|
||||
return qualificationImpairmentObservation{}, fmt.Errorf("observed throughput %.2f outside [%.2f,%.2f]", observation.ObservedThroughputKbps, lowerThroughput, upperThroughput)
|
||||
return qualificationImpairmentObservation{}, fmt.Errorf(
|
||||
"observed throughput %.2f outside [%.2f,%.2f] over %s with %d delivered/%d injected drops and steps %v",
|
||||
observation.ObservedThroughputKbps, lowerThroughput, upperThroughput,
|
||||
completedAt.Sub(started), observation.Delivered, observation.InjectedDropped, stepAt,
|
||||
)
|
||||
}
|
||||
}
|
||||
observation.RawSamples = filepath.Base(rawPath)
|
||||
@@ -1401,7 +1570,11 @@ func runQualificationProcessing(t *testing.T, profile qualificationMediaProfile,
|
||||
summary.RawResourcesSHA256 = resourceSum
|
||||
summary.RawResourcesBytes = resourceSize
|
||||
summary.ResourceSamples = len(resources)
|
||||
summary.CPUScope = "isolated gateway qualification process (gateway plus bounded fixture/client driver)"
|
||||
firstResource, lastResource := resources[0], resources[len(resources)-1]
|
||||
if firstResource.CPUSeconds < 0 || lastResource.CPUSeconds < 0 {
|
||||
return qualificationProcessingSummary{}, errors.New("process CPU usage unavailable")
|
||||
}
|
||||
summary.CPUSeconds = math.Max(0, lastResource.CPUSeconds-firstResource.CPUSeconds)
|
||||
summary.Mallocs = lastResource.Mallocs - firstResource.Mallocs
|
||||
summary.AllocatedBytes = lastResource.Allocated - firstResource.Allocated
|
||||
@@ -1441,12 +1614,16 @@ func runQualificationWarmup(t *testing.T, path *qualificationPath, profile quali
|
||||
}
|
||||
|
||||
func qualificationRuntimeSample(started time.Time) qualificationResourceSample {
|
||||
cpu := []runtimemetrics.Sample{{Name: "/cpu/classes/total:cpu-seconds"}}
|
||||
runtimemetrics.Read(cpu)
|
||||
var usage syscall.Rusage
|
||||
cpuSeconds := -1.0
|
||||
if syscall.Getrusage(syscall.RUSAGE_SELF, &usage) == nil {
|
||||
cpuSeconds = float64(usage.Utime.Sec+usage.Stime.Sec) +
|
||||
float64(usage.Utime.Usec+usage.Stime.Usec)/1_000_000
|
||||
}
|
||||
var memory runtime.MemStats
|
||||
runtime.ReadMemStats(&memory)
|
||||
return qualificationResourceSample{
|
||||
Elapsed: time.Since(started), CPUSeconds: cpu[0].Value.Float64(),
|
||||
Elapsed: time.Since(started), CPUSeconds: cpuSeconds,
|
||||
HeapBytes: memory.HeapAlloc, Goroutines: runtime.NumGoroutine(),
|
||||
Mallocs: memory.Mallocs, Allocated: memory.TotalAlloc,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user