mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-15 09:46:57 +01:00
Ryan Huber
e4f659a207
Charge staged reservations to each tunnel instead of dividing a static share of the window by routines. Several readers sending on one tunnel then share its credit: the limit is half the peer replay window, assumed from ReplayWindow, which must be a power of two of at least 8192 (checked at compile time). The other half allows finite unreserved traffic (inline and control sends, relay forwarding through SendVia) and network reordering; unlimited bypass traffic or delay can still exceed it. An outstanding-count cap alone would let a fast reader repeatedly send and reacquire credit, overtaking a paused reader without bound. So once readers overlap, spent credit is only recycled when every reservation has drained. A sole ordered reader gets each completion refunded, since its later batches cannot overtake its earlier ones. A reader that fails to acquire ends sole ownership, so a busy owner cannot keep a larger waiter blocked forever. This restores progress, not FIFO fairness. All transitions are a single CAS. Waiting readers flush their own oldest batch before retrying and never hold a partial acquisition: a failed multi-tunnel (relay) acquisition rolls back. Credit is released only after the flush or an explicit drop. A read that alone exceeds a tunnel's credit, such as a tiny-MSS superpacket, is sent segment by segment, synchronously. Acquisition is per read, so pressure on one tunnel also delays the other tunnels in the same read. The combined staged transmit, receive and shared credit measured 15 to 25 Gbit/s over one tunnel with eight TCP streams between two 100 GbE hosts. Tests model overtaking, ordered reuse, a blocked reader behind a busy owner, and relay rollback with both window sizes, and run eight competing readers plus unreserved control traffic on one tunnel, checking every packet is accepted by the receiver's replay window in wire order.
85 lines
3.2 KiB
Go
85 lines
3.2 KiB
Go
package nebula
|
|
|
|
import (
|
|
"fmt"
|
|
"log/slog"
|
|
"runtime"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/rcrowley/go-metrics"
|
|
"github.com/slackhq/nebula/config"
|
|
"github.com/slackhq/nebula/noiseutil"
|
|
)
|
|
|
|
// Staged processing (tun.staged.workers) moves crypto off each tun reader and udp listener onto worker goroutines,
|
|
// behind which an ordered stage sends or writes every batch in read order. It reuses the inline packet handlers:
|
|
// staging changes ownership and scheduling, not policy. Each routine has its own workers and ordered stages, so order
|
|
// is per routine. Full queues block the reader. A read error drains the batches in flight, while shutdown waits for
|
|
// the crypto and then drops packets whose destination has closed. TX uses eight slots and RX six; at listen.batch=64
|
|
// with GRO their buffers take about 9 MiB and 24 MiB per routine. Small-MSS TX arenas can grow.
|
|
//
|
|
// Metrics are registered only for active pipelines. flow.{tx,rx}.batches counts reads handed to the ordered stage and
|
|
// flow.tx.inline oversized reads the reader sent itself. flow.stage.<stage>.q<queue>.busy_ns locates the limiting
|
|
// stage: tx_read, tx_encrypt, tx_send, rx_read, rx_decrypt or rx_write. Take rates; 1e9 ns/s fills a serial stage.
|
|
// These are elapsed times that exclude read syscalls and downstream waits; descheduling inflates them and worker
|
|
// times add up across the pool. Small RX batches sent to the writer include decryption in writer time.
|
|
|
|
// stagedConfig is tun.staged. workers is per routine and direction; zero keeps processing inline.
|
|
type stagedConfig struct {
|
|
workers int
|
|
tx bool
|
|
rx bool
|
|
}
|
|
|
|
// parseStagedConfig enables staging on Linux only; queues and conns that cannot hand off batches also stay inline.
|
|
func parseStagedConfig(c *config.C, l *slog.Logger) stagedConfig {
|
|
sc := stagedConfig{workers: c.GetInt("tun.staged.workers", 0)}
|
|
if sc.workers <= 0 {
|
|
return stagedConfig{}
|
|
}
|
|
if runtime.GOOS != "linux" {
|
|
l.Warn("tun.staged is only supported on Linux; packet processing stays inline")
|
|
return stagedConfig{}
|
|
}
|
|
sc.tx, sc.rx = true, true
|
|
if noiseutil.EncryptLockNeeded {
|
|
l.Warn("tun.staged cannot encrypt in parallel with boringcrypto or fips140; transmit stays inline")
|
|
sc.tx = false
|
|
}
|
|
l.Info("Staged packet processing enabled", "workers", sc.workers, "tx", sc.tx, "rx", sc.rx)
|
|
return sc
|
|
}
|
|
|
|
// flowJob is one batch of crypto work. It owns its packet buffers until run returns; jobs and queues are reused.
|
|
type flowJob interface{ run(nb []byte) }
|
|
|
|
// flowPool runs one reader's crypto jobs. The reader is the only producer and closes the pool once its ordered stage
|
|
// has drained.
|
|
type flowPool struct {
|
|
jobs chan flowJob
|
|
wg sync.WaitGroup
|
|
}
|
|
|
|
func newFlowPool(workers, queueLen int, busy metrics.Counter) *flowPool {
|
|
p := &flowPool{jobs: make(chan flowJob, queueLen)}
|
|
for range workers {
|
|
p.wg.Go(func() {
|
|
nb := make([]byte, 12)
|
|
for job := range p.jobs {
|
|
t := time.Now()
|
|
job.run(nb)
|
|
busy.Inc(int64(time.Since(t)))
|
|
}
|
|
})
|
|
}
|
|
return p
|
|
}
|
|
|
|
func (p *flowPool) close() { close(p.jobs); p.wg.Wait() }
|
|
|
|
// stageBusy is the busy time counter of one stage of routine q.
|
|
func stageBusy(stage string, q int) metrics.Counter {
|
|
return metrics.GetOrRegisterCounter(fmt.Sprintf("flow.stage.%s.q%d.busy_ns", stage, q), nil)
|
|
}
|