tun: Add opt-in staged transmit processing

Set tun.staged.workers to a positive count on Linux, with tun.use_offloads
enabled, to move encryption off each tun reader. The reader filters and
reserves counters for a whole read, hands the batch to that many encrypt
workers, and an ordered sender flushes the batches in read order. Eight slots
per routine bound the batches in flight; a full pipeline blocks the reader.
Zero workers, the default, keeps today's inline path, as do tun queues that
cannot hand off their read buffer and builds whose AES-GCM needs nonce-ordered
encryption (BoringCrypto, FIPS).

IMPORTANT: staged senders need peers with the 8192-entry replay window of
master. Released v1.11.x peers use 1024 and will drop reordered batches. The
window is not negotiated.

Readers keep today's pinning. The sender is only pinned with an explicit
tun.cpu_affinity: sender i uses entry i+1, unless that is its reader's CPU,
so a one-entry list leaves the sender unpinned rather than sharing it: two
busy transmit threads on one CPU serialise (eight TCP streams over 100 GbE
fell from 25.9 to 22.8 Gbit/s). Without a list the sender is not pinned. Placing it by queue index on a
hybrid CPU put it on an efficiency core, and one TCP stream over 100 GbE
reached 18.6-19.6 Gbit/s pinned against 20.2-20.6 unpinned.

New API, each needed across a package boundary:
- tio.Offload.SwapRxBuf hands the buffer of the last Read to the reader and
  reads into the one given. It is a method on the concrete queue, found through
  an unexported interface in nebula, so tio.Queue and its other
  implementations are unchanged.
- batch.SendBatch.Reset drops queued packets without sending, for a sender
  whose sockets are already closed; Flush now uses it.

Metrics, registered only for active pipelines: flow.tx.batches counts reads
handed to the sender, flow.stage.{tx_read,tx_encrypt,tx_send}.q<N>.busy_ns the
elapsed busy time of each stage, to find the limiting one.

Tests cover ordering and counters under backpressure with both ciphers (AES
skipped where nonce order is required), read errors with batches in flight,
shutdown, exhaustion, relays, fallback, firewall parity, and the actual reader
and send thread affinity for every placement, with pinning on and off: the
sender is pinned only to an explicit entry other than its reader's CPU.

interface.go

@@ -57,14 +57,19 @@ type InterfaceConfig struct {
 
 	// CpuAffinity, when non-empty, names the CPUs each TUN reader goroutine
 	// should pin to. Queue i pins to CpuAffinity[i % len(CpuAffinity)] —
-	// shorter lists than `routines` cycle. Empty list keeps the default
-	// pin-to-(i % NumCPU) behavior. Only consulted when PinThreads is true.
+	// shorter lists than `routines` cycle. Staged sender i uses entry i+1,
+	// unless that is its reader's CPU. Empty uses the automatic order. Only
+	// consulted when PinThreads is true.
 	CpuAffinity []int
+	// defaultCpuAffinity is the automatic pin order for readers, kept apart
+	// from CpuAffinity because staged senders are only pinned to an explicit list.
+	defaultCpuAffinity []int
 	// PinThreads controls whether each TUN reader OS thread is pinned to a
 	// single CPU (via tun.pin_threads, default true). Pinning keeps each
 	// goroutine's sendmmsg on one XPS-selected NIC TX ring so per-flow
 	// packets stay ordered on the wire.
 	PinThreads bool
+	staged     stagedConfig
 
 	l *slog.Logger
 }
@@ -94,11 +99,13 @@ type Interface struct {
 	// should pin to. Queue i pins to cpuAffinity[i % len(cpuAffinity)].
 	// Empty falls back to the default pin-to-(allowed CPU) behavior.
 	// Only consulted when pinThreads is true.
-	cpuAffinity []int
+	cpuAffinity        []int
+	defaultCpuAffinity []int
 	// pinThreads controls whether listenIn pins each TUN reader OS thread to
 	// a CPU at all (tun.pin_threads, default true). When false, threads are
 	// left free to migrate as on stock nebula.
 	pinThreads   bool
+	staged       stagedConfig
 	relayManager *relayManager
 
 	tryPromoteEvery atomic.Uint32
@@ -247,7 +254,9 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 		connectionManager:     c.connectionManager,
 		conntrackCacheTimeout: c.ConntrackCacheTimeout,
 		cpuAffinity:           c.CpuAffinity,
+		defaultCpuAffinity:    c.defaultCpuAffinity,
 		pinThreads:            c.PinThreads,
+		staged:                c.staged,
 		udpTxErrLimiter:       udpTxErrLimiter,
 		udpRelayTxErrLimiter:  udpRelayTxErrLimiter,
 		txOversizeLimiter:     txOversizeLimiter,
@@ -433,32 +442,30 @@ func (f *Interface) listenOut(i int) {
 	f.l.Debug("underlay reader is done", "reader", i)
 }
 
-func (f *Interface) pinThisThread(i int) {
-	var cpu int
+// threadCPU is the CPU tun reader i pins to.
+func (f *Interface) threadCPU(i int) int {
 	if n := len(f.cpuAffinity); n > 0 {
 		// Explicit tun.cpu_affinity list wins; parseCpuAffinity already
 		// validated the entries against the allowed CPU set.
-		cpu = f.cpuAffinity[i%n]
+		return f.cpuAffinity[i%n]
+	} else if n := len(f.defaultCpuAffinity); n > 0 {
+		return f.defaultCpuAffinity[i%n]
 	} else if allowed, err := util.AllowedCPUs(); err == nil && len(allowed) > 0 {
 		// Default: spread queues across the CPUs we're actually allowed to
 		// run on. Under a cpuset/taskset mask these aren't 0..NumCPU-1, so
 		// i % NumCPU would pick unrunnable IDs and every pin would fail.
-		cpu = allowed[i%len(allowed)]
-	} else {
-		cpu = i % runtime.NumCPU()
+		return allowed[i%len(allowed)]
 	}
+	return i % runtime.NumCPU()
+}
+
+func (f *Interface) pinThreadToCPU(i, cpu int) {
 	if err := util.PinThreadToCPU(cpu); err != nil {
-		f.l.Warn("failed to pin tun reader to CPU", "queue", i, "cpu", cpu, "err", err)
+		f.l.Warn("failed to pin tun processing thread to CPU", "queue", i, "cpu", cpu, "err", err)
 	}
 }
 
 func (f *Interface) listenIn(queue tio.Queue, i int) {
-	// Pinning this thread (and goroutine) to a single CPU keeps every sendmmsg from this goroutine going through the
-	// same TX ring on the nic, so the wire sees per-flow order. Skip entirely when tun.pin_threads is false.
-	if f.pinThreads {
-		f.pinThisThread(i)
-	}
-
 	rejectBuf := make([]byte, mtu)
 	arenaSize := batch.SendBatchCap * (udp.MTU + 32)
 	sb := batch.NewSendBatch(f.writers[i], batch.SendBatchCap, arenaSize)
@@ -467,6 +474,26 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
 
 	conntrackCache := firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout)
 
+	if sq, ok := queue.(handoffQueue); ok && f.staged.tx {
+		// Every stage on its own goroutine, see flow_tx.go
+		err := newStagedTx(f, sq, i).run(fwPacket, rejectBuf, conntrackCache)
+		if !f.closed.Load() && f.ctx.Err() == nil {
+			f.l.Error("Error while reading outbound packet, closing", "error", err, "reader", i)
+			f.onFatal(err)
+		}
+		f.l.Debug("overlay reader is done", "reader", i)
+		return
+	}
+	if f.staged.tx {
+		f.l.Info("tun queue cannot hand off read batches, transmit stays inline", "reader", i)
+	}
+
+	// Pinning this thread (and goroutine) to a single CPU keeps every sendmmsg from this goroutine going through the
+	// same TX ring on the nic, so the wire sees per-flow order. Skip entirely when tun.pin_threads is false.
+	if f.pinThreads {
+		f.pinThreadToCPU(i, f.threadCPU(i))
+	}
+
 	for {
 		pkts, err := queue.Read()
 		if err != nil {