mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-15 17:16:52 +01:00
Ryan Huber
f3f98e1690
Set tun.staged.workers to a positive count on Linux, with tun.use_offloads
enabled, to move encryption off each tun reader. The reader filters and
reserves counters for a whole read, hands the batch to that many encrypt
workers, and an ordered sender flushes the batches in read order. Eight slots
per routine bound the batches in flight; a full pipeline blocks the reader.
Zero workers, the default, keeps today's inline path, as do tun queues that
cannot hand off their read buffer and builds whose AES-GCM needs nonce-ordered
encryption (BoringCrypto, FIPS).
IMPORTANT: staged senders need peers with the 8192-entry replay window of
master. Released v1.11.x peers use 1024 and will drop reordered batches. The
window is not negotiated.
Readers keep today's pinning. The sender is only pinned with an explicit
tun.cpu_affinity: sender i uses entry i+1, unless that is its reader's CPU,
so a one-entry list leaves the sender unpinned rather than sharing it: two
busy transmit threads on one CPU serialise (eight TCP streams over 100 GbE
fell from 25.9 to 22.8 Gbit/s). Without a list the sender is not pinned. Placing it by queue index on a
hybrid CPU put it on an efficiency core, and one TCP stream over 100 GbE
reached 18.6-19.6 Gbit/s pinned against 20.2-20.6 unpinned.
New API, each needed across a package boundary:
- tio.Offload.SwapRxBuf hands the buffer of the last Read to the reader and
reads into the one given. It is a method on the concrete queue, found through
an unexported interface in nebula, so tio.Queue and its other
implementations are unchanged.
- batch.SendBatch.Reset drops queued packets without sending, for a sender
whose sockets are already closed; Flush now uses it.
Metrics, registered only for active pipelines: flow.tx.batches counts reads
handed to the sender, flow.stage.{tx_read,tx_encrypt,tx_send}.q<N>.busy_ns the
elapsed busy time of each stage, to find the limiting one.
Tests cover ordering and counters under backpressure with both ciphers (AES
skipped where nonce order is required), read errors with batches in flight,
shutdown, exhaustion, relays, fallback, firewall parity, and the actual reader
and send thread affinity for every placement, with pinning on and off: the
sender is pinned only to an explicit entry other than its reader's CPU.
82 lines
2.9 KiB
Go
82 lines
2.9 KiB
Go
package nebula
|
|
|
|
import (
|
|
"fmt"
|
|
"log/slog"
|
|
"runtime"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/rcrowley/go-metrics"
|
|
"github.com/slackhq/nebula/config"
|
|
"github.com/slackhq/nebula/noiseutil"
|
|
)
|
|
|
|
// Staged transmit (tun.staged.workers) moves encryption off each tun reader onto worker goroutines, behind which an
|
|
// ordered sender flushes every batch in read order. It reuses the inline filter and encryption: staging changes
|
|
// ownership and scheduling, not policy. Each routine has its own workers and sender, so order is per routine. Full
|
|
// queues block the reader. A read error drains the batches in flight, while shutdown waits for the crypto and then
|
|
// drops packets whose sockets have closed. Eight slots take about 9 MiB per routine; small-MSS arenas can grow.
|
|
//
|
|
// Metrics are registered only for active pipelines. flow.tx.batches counts reads handed to the sender.
|
|
// flow.stage.<stage>.q<queue>.busy_ns locates the limiting stage: tx_read, tx_encrypt or tx_send. Take rates; 1e9
|
|
// ns/s fills a serial stage. These are elapsed times that exclude read syscalls and downstream waits; descheduling
|
|
// inflates them and worker times add up across the pool.
|
|
|
|
// stagedConfig is tun.staged. workers is per routine and direction; zero keeps processing inline.
|
|
type stagedConfig struct {
|
|
workers int
|
|
tx bool
|
|
}
|
|
|
|
// parseStagedConfig enables staging on Linux only; queues and conns that cannot hand off batches also stay inline.
|
|
func parseStagedConfig(c *config.C, l *slog.Logger) stagedConfig {
|
|
sc := stagedConfig{workers: c.GetInt("tun.staged.workers", 0)}
|
|
if sc.workers <= 0 {
|
|
return stagedConfig{}
|
|
}
|
|
if runtime.GOOS != "linux" {
|
|
l.Warn("tun.staged is only supported on Linux; packet processing stays inline")
|
|
return stagedConfig{}
|
|
}
|
|
sc.tx = true
|
|
if noiseutil.EncryptLockNeeded {
|
|
l.Warn("tun.staged cannot encrypt in parallel with boringcrypto or fips140; transmit stays inline")
|
|
sc.tx = false
|
|
}
|
|
l.Info("Staged packet processing enabled", "workers", sc.workers, "tx", sc.tx)
|
|
return sc
|
|
}
|
|
|
|
// flowJob is one batch of crypto work. It owns its packet buffers until run returns; jobs and queues are reused.
|
|
type flowJob interface{ run(nb []byte) }
|
|
|
|
// flowPool runs one reader's crypto jobs. The reader is the only producer and closes the pool once its ordered stage
|
|
// has drained.
|
|
type flowPool struct {
|
|
jobs chan flowJob
|
|
wg sync.WaitGroup
|
|
}
|
|
|
|
func newFlowPool(workers, queueLen int, busy metrics.Counter) *flowPool {
|
|
p := &flowPool{jobs: make(chan flowJob, queueLen)}
|
|
for range workers {
|
|
p.wg.Go(func() {
|
|
nb := make([]byte, 12)
|
|
for job := range p.jobs {
|
|
t := time.Now()
|
|
job.run(nb)
|
|
busy.Inc(int64(time.Since(t)))
|
|
}
|
|
})
|
|
}
|
|
return p
|
|
}
|
|
|
|
func (p *flowPool) close() { close(p.jobs); p.wg.Wait() }
|
|
|
|
// stageBusy is the busy time counter of one stage of routine q.
|
|
func stageBusy(stage string, q int) metrics.Counter {
|
|
return metrics.GetOrRegisterCounter(fmt.Sprintf("flow.stage.%s.q%d.busy_ns", stage, q), nil)
|
|
}
|