tun: Add tun.staged.tx and tun.staged.inline_below

tx: false keeps transmit inline and stages only receive. inline_below
sets the idle-reader shortcut threshold, 0 hands every batch to the
workers and the ordered sender or writer.
This commit is contained in:
Ryan Huber committed 2026-10-16 21:24:28 +00:00
1 parent 82488caa32
commit d16a52f921
4 files changed
+93 -8

No files matched your search

+6 -2
View File
@@ -284,14 +284,18 @@ tun:
# Staged reservations may use half the window, the other half absorbs inline and control traffic, relay forwarding # Staged reservations may use half the window, the other half absorbs inline and control traffic, relay forwarding
# and network reordering; unlimited forwarding or delay can still exceed it. Waiting for credit on a busy tunnel # and network reordering; unlimited forwarding or delay can still exceed it. Waiting for credit on a busy tunnel
# also delays the other tunnels of the same read, and readers are not guaranteed fairness. # also delays the other tunnels of the same read, and readers are not guaranteed fairness.
# Idle batches below 256 TX segments or RX packets stay on the reader. With multiple routines, those sends still # Idle batches below inline_below TX segments or RX packets stay on the reader, default 256. With multiple routines,
# take tunnel reservation credit. # those sends still take tunnel reservation credit. 0 hands every batch to the workers and the ordered sender or
# writer; only TX reads too large for a tunnel's whole credit are still sent by the reader, after it drains.
# tx: false keeps transmit inline and stages only receive.
# tx_slots and rx_slots bound the batches each routine has in flight per direction, minimum 2. A TX slot holds one # tx_slots and rx_slots bound the batches each routine has in flight per direction, minimum 2. A TX slot holds one
# tun read and its ciphertext, about 1.1 MiB; the default is 8, or workers + 2 if more. An RX slot holds one # tun read and its ciphertext, about 1.1 MiB; the default is 8, or workers + 2 if more. An RX slot holds one
# recvmmsg batch, listen.batch * 64 KiB with udp_offloads or * 9 KiB without; the default keeps up to 384 receive # recvmmsg batch, listen.batch * 64 KiB with udp_offloads or * 9 KiB without; the default keeps up to 384 receive
# entries in flight in 4 to 6 slots: 6 slots or 24 MiB at the default listen.batch of 64. Lower them to save memory. # entries in flight in 4 to 6 slots: 6 slots or 24 MiB at the default listen.batch of 64. Lower them to save memory.
#staged: #staged:
# workers: 0 # workers: 0
# tx: true
# inline_below: 256
# tx_slots: 8 # tx_slots: 8
# rx_slots: 6 # rx_slots: 6
+6 -6
View File
@@ -34,7 +34,7 @@ type stagedConfig struct {
workers int workers int
tx bool tx bool
rx bool rx bool
// inlineBelow is the TX segment or RX packet count below which an idle reader skips the worker handoff // inlineBelow is the TX segment or RX packet count below which an idle reader skips the worker handoff, 0 never
inlineBelow int inlineBelow int
// txSlots and rxSlots are how many batches each routine can have in flight per direction // txSlots and rxSlots are how many batches each routine can have in flight per direction
txSlots, rxSlots int txSlots, rxSlots int
@@ -78,17 +78,17 @@ func parseStagedConfig(c *config.C, l *slog.Logger) stagedConfig {
l.Warn("tun.staged is only supported on Linux; packet processing stays inline") l.Warn("tun.staged is only supported on Linux; packet processing stays inline")
return stagedConfig{} return stagedConfig{}
} }
sc.tx, sc.rx = true, true sc.tx, sc.rx = c.GetBool("tun.staged.tx", true), true
// 256 is the measured crossover // 256 is the measured crossover, 0 hands every batch to the workers
sc.inlineBelow = 256 sc.inlineBelow = max(c.GetInt("tun.staged.inline_below", 256), 0)
if noiseutil.EncryptLockNeeded { if sc.tx && noiseutil.EncryptLockNeeded {
l.Warn("tun.staged cannot encrypt in parallel with boringcrypto or fips140; transmit stays inline") l.Warn("tun.staged cannot encrypt in parallel with boringcrypto or fips140; transmit stays inline")
sc.tx = false sc.tx = false
} }
sc.txSlots, sc.rxSlots = stagedSlots(sc.workers, c.GetInt("listen.batch", 64), sc.txSlots, sc.rxSlots = stagedSlots(sc.workers, c.GetInt("listen.batch", 64),
c.GetInt("tun.staged.tx_slots", 0), c.GetInt("tun.staged.rx_slots", 0)) c.GetInt("tun.staged.tx_slots", 0), c.GetInt("tun.staged.rx_slots", 0))
l.Info("Staged packet processing enabled", "workers", sc.workers, "tx", sc.tx, "rx", sc.rx, l.Info("Staged packet processing enabled", "workers", sc.workers, "tx", sc.tx, "rx", sc.rx,
"txSlots", sc.txSlots, "rxSlots", sc.rxSlots) "inlineBelow", sc.inlineBelow, "txSlots", sc.txSlots, "rxSlots", sc.rxSlots)
return sc return sc
} }
+68
View File
@@ -14,12 +14,14 @@ import (
"github.com/flynn/noise" "github.com/flynn/noise"
"github.com/rcrowley/go-metrics" "github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/cpupick" "github.com/slackhq/nebula/cpupick"
"github.com/slackhq/nebula/firewall" "github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil" "github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch" "github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio" "github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
"github.com/slackhq/nebula/util" "github.com/slackhq/nebula/util"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@@ -433,6 +435,72 @@ func TestFlowStagedTXInlineOnlyWhenEmpty(t *testing.T) {
require.Equal(t, want, decryptWire(t, receiver, sender.conn.captured())) require.Equal(t, want, decryptWire(t, receiver, sender.conn.captured()))
} }
// TestFlowStagedTXInlineBelowZero: with inline_below 0 an idle reader never sends a batch itself, in either
// direction: every read goes through the workers and the ordered sender or writer.
func TestFlowStagedTXInlineBelowZero(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
l := test.NewLogger()
c := config.NewC(l)
c.Settings = map[string]any{"tun": map[string]any{"staged": map[string]any{"workers": 2, "inline_below": 0}}}
sc := parseStagedConfig(c, l)
if !sc.tx {
t.Skip("transmit stays inline in this build")
}
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
batches, want := flowTestBatches(20, 2, 1, 4)
txInline := metrics.GetOrRegisterCounter("flow.tx.inline", nil)
tx := metrics.GetOrRegisterCounter("flow.tx.batches", nil)
rxInline := metrics.GetOrRegisterCounter("flow.rx.inline", nil)
txInline0, tx0, rxInline0, rx0 := txInline.Count(), tx.Count(), rxInline.Count(), rxBatches.Count()
sender.f.listenIn(newFlowBatchQueue(batches), 0)
wire := sender.conn.captured()
require.Equal(t, want, decryptWire(t, receiver, wire))
require.Zero(t, txInline.Count()-txInline0)
require.Equal(t, int64(len(batches)), tx.Count()-tx0)
receiver.receive(&rxConn{wire: wire, sizes: []int{1, 2, 4}})
require.Equal(t, want, receiver.tun.segments(t))
require.Zero(t, rxInline.Count()-rxInline0)
require.Positive(t, rxBatches.Count()-rx0)
}
// TestFlowStagedTXDisabled: with tx false the reader never builds a staged pipeline, even on a queue that could hand
// off its batches. Receive stays staged.
func TestFlowStagedTXDisabled(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
l := test.NewLogger()
c := config.NewC(l)
c.Settings = map[string]any{"tun": map[string]any{"staged": map[string]any{"workers": 2, "tx": false}}}
sc := parseStagedConfig(c, l)
require.False(t, sc.tx)
require.True(t, sc.rx)
sender, receiver := newFlowTestPair(t, noise.CipherChaChaPoly, sc)
batches, want := flowTestBatches(4, 8, 1, 45)
tx := metrics.GetOrRegisterCounter("flow.tx.batches", nil)
txInline := metrics.GetOrRegisterCounter("flow.tx.inline", nil)
tx0, txInline0, rx0 := tx.Count(), txInline.Count(), rxBatches.Count()
q := &swapCountQueue{flowBatchQueue: newFlowBatchQueue(batches)}
sender.f.listenIn(q, 0)
wire := sender.conn.captured()
require.Equal(t, want, decryptWire(t, receiver, wire))
require.Zero(t, q.swaps, "newStagedTx always takes the queue's buffer, so it never ran")
require.Equal(t, tx0, tx.Count())
require.Equal(t, txInline0, txInline.Count())
receiver.receive(&rxConn{wire: wire, batchLen: 512})
require.Equal(t, want, receiver.tun.segments(t))
require.Greater(t, rxBatches.Count(), rx0)
}
// swapCountQueue counts SwapRxBuf calls.
type swapCountQueue struct {
*flowBatchQueue
swaps int
}
func (q *swapCountQueue) SwapRxBuf(buf []byte) []byte {
q.swaps++
return q.flowBatchQueue.SwapRxBuf(buf)
}
// TestFlowStagedTXAdaptiveBudget: a fast reader's adaptive sends must stop spending credit until another reader's // TestFlowStagedTXAdaptiveBudget: a fast reader's adaptive sends must stop spending credit until another reader's
// earlier reservation reaches the wire. // earlier reservation reaches the wire.
func TestFlowStagedTXAdaptiveBudget(t *testing.T) { func TestFlowStagedTXAdaptiveBudget(t *testing.T) {
+13
View File
@@ -150,6 +150,19 @@ func TestParseStagedConfig(t *testing.T) {
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": 2, "tx_slots": 3, "rx_slots": 1}} c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": 2, "tx_slots": 3, "rx_slots": 1}}
got := parseStagedConfig(c, l) got := parseStagedConfig(c, l)
require.Equal(t, []int{3, 2}, []int{got.txSlots, got.rxSlots}, "overrides apply, down to the minimum") require.Equal(t, []int{3, 2}, []int{got.txSlots, got.rxSlots}, "overrides apply, down to the minimum")
c = config.NewC(l)
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": 2, "tx": false, "inline_below": 0}}
got = parseStagedConfig(c, l)
require.False(t, got.tx, "tx: false keeps transmit inline")
require.True(t, got.rx)
require.Zero(t, got.inlineBelow, "inline_below: 0 hands every batch off")
c = config.NewC(l)
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": 2, "inline_below": 32}}
require.Equal(t, 32, parseStagedConfig(c, l).inlineBelow)
c.Settings["tun"] = map[string]any{"staged": map[string]any{"workers": 2, "inline_below": -1}}
require.Zero(t, parseStagedConfig(c, l).inlineBelow, "a negative threshold is 0")
} }
func TestStagedSlots(t *testing.T) { func TestStagedSlots(t *testing.T) {