Remove staged receive stage timing metrics

This commit is contained in:
JackDoan committed 2026-10-15 16:38:30 -05:00
1 parent 4d6f757d1f
commit 8f34981764
2 files changed
+6 -46

No files matched your search

+1 -15
View File
@@ -1,13 +1,10 @@
package nebula package nebula
import ( import (
"fmt"
"log/slog" "log/slog"
"runtime" "runtime"
"sync" "sync"
"time"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
) )
@@ -19,10 +16,6 @@ import (
// //
// Metrics are registered only for active pipelines. flow.rx.batches counts batches handed to the writer and // Metrics are registered only for active pipelines. flow.rx.batches counts batches handed to the writer and
// flow.rx.inline those an idle listener handled itself; their rates show whether staging is used. // flow.rx.inline those an idle listener handled itself; their rates show whether staging is used.
// flow.stage.<stage>.q<queue>.busy_ns locates the limiting stage: rx_read, rx_decrypt or rx_write. Take rates; 1e9
// ns/s fills a serial stage. These are elapsed times that exclude read syscalls and downstream waits; descheduling
// inflates them and worker times add up across the pool. Small batches sent to the writer include decryption in
// writer time.
// stagedConfig is tun.staged. workers is per routine; zero keeps receive inline. // stagedConfig is tun.staged. workers is per routine; zero keeps receive inline.
type stagedConfig struct { type stagedConfig struct {
@@ -57,15 +50,13 @@ type flowPool struct {
wg sync.WaitGroup wg sync.WaitGroup
} }
func newFlowPool(workers, queueLen int, busy metrics.Counter) *flowPool { func newFlowPool(workers, queueLen int) *flowPool {
p := &flowPool{jobs: make(chan flowJob, queueLen)} p := &flowPool{jobs: make(chan flowJob, queueLen)}
for range workers { for range workers {
p.wg.Go(func() { p.wg.Go(func() {
nb := make([]byte, 12) nb := make([]byte, 12)
for job := range p.jobs { for job := range p.jobs {
t := time.Now()
job.run(nb) job.run(nb)
busy.Inc(int64(time.Since(t)))
} }
}) })
} }
@@ -73,8 +64,3 @@ func newFlowPool(workers, queueLen int, busy metrics.Counter) *flowPool {
} }
func (p *flowPool) close() { close(p.jobs); p.wg.Wait() } func (p *flowPool) close() { close(p.jobs); p.wg.Wait() }
// stageBusy is the busy time counter of one stage of routine q.
func stageBusy(stage string, q int) metrics.Counter {
return metrics.GetOrRegisterCounter(fmt.Sprintf("flow.stage.%s.q%d.busy_ns", stage, q), nil)
}
+5 -31
View File
@@ -3,7 +3,6 @@ package nebula
import ( import (
"encoding/binary" "encoding/binary"
"net/netip" "net/netip"
"time"
"github.com/rcrowley/go-metrics" "github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
@@ -88,7 +87,6 @@ type stagedRx struct {
hostmapCache map[uint32]*HostInfo hostmapCache map[uint32]*HostInfo
wrxc *rxContext wrxc *rxContext
started time.Time
cur *rxBatch cur *rxBatch
free []*rxBatch free []*rxBatch
ordered chan *rxBatch ordered chan *rxBatch
@@ -96,7 +94,6 @@ type stagedRx struct {
inFlight int inFlight int
writerDone chan struct{} writerDone chan struct{}
readBusy, decryptBusy, writeBusy metrics.Counter
metricBatches metrics.Counter metricBatches metrics.Counter
metricInline metrics.Counter metricInline metrics.Counter
} }
@@ -111,14 +108,11 @@ func newStagedRx(f *Interface, q int, wrxc *rxContext) *stagedRx {
ordered: make(chan *rxBatch, stagedRxSlots), ordered: make(chan *rxBatch, stagedRxSlots),
returned: make(chan *rxBatch, stagedRxSlots), returned: make(chan *rxBatch, stagedRxSlots),
writerDone: make(chan struct{}), writerDone: make(chan struct{}),
readBusy: stageBusy("rx_read", q),
decryptBusy: stageBusy("rx_decrypt", q),
writeBusy: stageBusy("rx_write", q),
metricBatches: metrics.GetOrRegisterCounter("flow.rx.batches", nil), metricBatches: metrics.GetOrRegisterCounter("flow.rx.batches", nil),
metricInline: metrics.GetOrRegisterCounter("flow.rx.inline", nil), metricInline: metrics.GetOrRegisterCounter("flow.rx.inline", nil),
} }
workers := f.staged.workers workers := f.staged.workers
p.pool = newFlowPool(workers, stagedRxSlots*2*workers, p.decryptBusy) p.pool = newFlowPool(workers, stagedRxSlots*2*workers)
for range stagedRxSlots { for range stagedRxSlots {
// Two chunks per worker, so the writer can start committing while the workers are still on the rest // Two chunks per worker, so the writer can start committing while the workers are still on the rest
b := &rxBatch{items: make([]rxItem, 0, 256), chunks: make([]rxChunk, 2*workers)} b := &rxBatch{items: make([]rxItem, 0, 256), chunks: make([]rxChunk, 2*workers)}
@@ -143,9 +137,6 @@ func (p *stagedRx) listen(li handoffConn) error {
// deliver is the listener's per-packet work: classify the packet and add it to the current batch. Nothing here // deliver is the listener's per-packet work: classify the packet and add it to the current batch. Nothing here
// decrypts or touches the tun. // decrypts or touches the tun.
func (p *stagedRx) deliver(via netip.AddrPort, packet []byte) { func (p *stagedRx) deliver(via netip.AddrPort, packet []byte) {
if p.started.IsZero() {
p.started = time.Now()
}
b := p.cur b := p.cur
it := rxItem{packet: packet, via: via} it := rxItem{packet: packet, via: via}
p.classify(&it) p.classify(&it)
@@ -210,10 +201,6 @@ func (p *stagedRx) precheck(b *rxBatch) {
func (p *stagedRx) handoff(bufs [][]byte) [][]byte { func (p *stagedRx) handoff(bufs [][]byte) [][]byte {
p.cur.bufs = bufs p.cur.bufs = bufs
p.endBatch() p.endBatch()
if !p.started.IsZero() {
p.readBusy.Inc(int64(time.Since(p.started)))
p.started = time.Time{}
}
b := p.cur b := p.cur
if len(b.bufs) != len(bufs) || len(b.bufs[0]) < len(bufs[0]) { if len(b.bufs) != len(bufs) || len(b.bufs[0]) < len(bufs[0]) {
b.bufs = make([][]byte, len(bufs)) b.bufs = make([][]byte, len(bufs))
@@ -262,11 +249,8 @@ func (p *stagedRx) endBatch() {
p.ordered <- b p.ordered <- b
if len(p.free) == 0 { if len(p.free) == 0 {
// Every batch is in flight, wait for the writer: the socket buffer absorbs the backlog meanwhile. The wait is // Every batch is in flight, wait for the writer: the socket buffer absorbs the backlog meanwhile
// downstream backpressure, not listener busy time.
t := time.Now()
p.release(<-p.returned) p.release(<-p.returned)
p.started = p.started.Add(time.Since(t))
} }
p.cur = p.free[len(p.free)-1] p.cur = p.free[len(p.free)-1]
p.free = p.free[:len(p.free)-1] p.free = p.free[:len(p.free)-1]
@@ -290,9 +274,8 @@ func (p *stagedRx) release(b *rxBatch) {
} }
// process is the ordered stage for one batch: decrypt whatever no worker is decrypting, commit every item in arrival // process is the ordered stage for one batch: decrypt whatever no worker is decrypting, commit every item in arrival
// order, and flush the tun batcher. It reports how long it waited on workers. // order, and flush the tun batcher.
func (p *stagedRx) process(b *rxBatch, rxc *rxContext) time.Duration { func (p *stagedRx) process(b *rxBatch, rxc *rxContext) {
var waited time.Duration
if b.nChunks == 0 { if b.nChunks == 0 {
decryptRxItems(b.items, rxc.nb) decryptRxItems(b.items, rxc.nb)
p.commit(b.items, rxc) p.commit(b.items, rxc)
@@ -300,18 +283,11 @@ func (p *stagedRx) process(b *rxBatch, rxc *rxContext) time.Duration {
// Commit chunk by chunk, so the commit overlaps the workers still on later chunks // Commit chunk by chunk, so the commit overlaps the workers still on later chunks
for i := range b.nChunks { for i := range b.nChunks {
c := &b.chunks[i] c := &b.chunks[i]
select {
case <-c.done:
default:
t := time.Now()
<-c.done <-c.done
waited += time.Since(t)
}
p.commit(c.items, rxc) p.commit(c.items, rxc)
} }
} }
p.f.flushOutside(rxc) p.f.flushOutside(rxc)
return waited
} }
// commit runs the ordered stage over items in arrival order. A non-data packet goes through readOutsidePackets in // commit runs the ordered stage over items in arrival order. A non-data packet goes through readOutsidePackets in
@@ -378,9 +354,7 @@ func (p *stagedRx) writer() {
<-b.chunks[i].done <-b.chunks[i].done
} }
} else { } else {
t := time.Now() p.process(b, p.wrxc)
waited := p.process(b, p.wrxc)
p.writeBusy.Inc(int64(time.Since(t) - waited))
} }
b.reset() b.reset()
p.returned <- b p.returned <- b