tun: Keep small idle batches on the reader

With staging enabled, a reader whose pipeline is empty handles a batch below
256 TX segments or RX packets itself, without a worker handoff or buffer swap.
The threshold is the measured crossover, not another configuration knob.
Inline work only starts once that reader's ordered work has completed, so
ownership and order hold across switches.

With several readers, transmit still takes shared tunnel credit and the same
reservations as staged batches, releasing it after the synchronous flush; a
sole reader with an empty pipeline has nothing to overtake and uses the
original inline path. Small batches a reader sends itself run on the reader's
CPU, so they can use a different XPS TX ring than its sender.

Metrics: flow.tx.inline and flow.rx.inline count reads handled on the reader
(TX inline also counts oversized reads).

With receive IRQ CPUs kept free of Nebula threads, this improved one TCP
stream over 100 GbE from 16.7 to 21.1 Gbit/s; that measurement does not isolate
the threshold. Tests cover the switch in both directions, ordering behind an
in-flight batch, a paused reader against repeated small sends on another,
RecvError handling on the listener, and send affinity across mode switches.

flow_tx.go

@@ -270,7 +270,7 @@ func (s *txSlot) run(nb []byte) {
 type stagedTx struct {
 	txReserver
 	q         int
-	cpu       int // reader CPU
+	cpu       int // reader CPU, inline sends stay here
 	senderCPU int // -1 leaves the sender within its inherited affinity
 	queue     handoffQueue
 	pool      *flowPool
@@ -359,6 +359,13 @@ func (p *stagedTx) run(fwPacket *firewall.ParsedPacket, rejectBuf []byte, conntr
 // it itself, with how long it waited for the sender.
 func (p *stagedTx) handle(pkts []tio.Packet, fwPacket *firewall.ParsedPacket, rejectBuf []byte, localCache firewall.ConntrackCache) time.Duration {
 	p.reap()
+	inline := p.inFlight == 0 && p.small(pkts)
+	if inline && p.f.routines == 1 {
+		// With only one reader and an empty pipeline, no other staged reservation can be overtaken. Keep this common
+		// case inexpensive.
+		p.inlineSoleReader(pkts, fwPacket, rejectBuf, localCache)
+		return 0
+	}
 
 	slot, stall := p.acquire()
 	items := p.collect(slot.items[:0], pkts, fwPacket, rejectBuf, p.q, localCache)
@@ -392,6 +399,12 @@ func (p *stagedTx) handle(pkts []tio.Packet, fwPacket *firewall.ParsedPacket, re
 		return stall
 	}
 
+	if inline {
+		p.inline(slot)
+		p.free = append(p.free, slot)
+		return stall
+	}
+
 	// The packets now live in the slot, and the queue reads into the slot's old buffer
 	slot.buf = p.queue.SwapRxBuf(slot.buf)
 
@@ -474,3 +487,47 @@ func (p *stagedTx) close() {
 	<-p.senderDone
 	p.pool.close()
 }
+
+// small counts segments against the inline threshold.
+func (p *stagedTx) small(pkts []tio.Packet) bool {
+	below := uint64(p.f.staged.inlineBelow)
+	if below == 0 {
+		return false
+	}
+	var n uint64
+	for _, pkt := range pkts {
+		if n += segmentCount(pkt); n >= below {
+			return false
+		}
+	}
+	return true
+}
+
+// inline encrypts a budgeted batch on the reader, with no worker handoff or buffer swap. The local sender is idle,
+// but other readers may hold reservations on these tunnels, so keep our credit until the flush completes.
+func (p *stagedTx) inline(slot *txSlot) {
+	p.metricInline.Inc(1)
+	for i := range slot.items {
+		p.f.encryptTxItem(&slot.items[i], p.nb, p.sb)
+		if p.sb.Len() >= batch.SendBatchCap {
+			p.f.flushSendBatch(p.sb, p.q)
+		}
+	}
+	p.f.flushSendBatch(p.sb, p.q)
+	slot.releaseBudget()
+	clear(slot.items)
+	slot.items = slot.items[:0]
+}
+
+// inlineSoleReader needs no credit: the interface has no other tun reader and this reader has drained every
+// reservation before taking the inline path.
+func (p *stagedTx) inlineSoleReader(pkts []tio.Packet, fwPacket *firewall.ParsedPacket, rejectBuf []byte, localCache firewall.ConntrackCache) {
+	p.metricInline.Inc(1)
+	for _, pkt := range pkts {
+		p.f.consumeInsidePacket(pkt, fwPacket, p.nb, p.sb, rejectBuf, p.q, localCache)
+		if p.sb.Len() >= batch.SendBatchCap {
+			p.f.flushSendBatch(p.sb, p.q)
+		}
+	}
+	p.f.flushSendBatch(p.sb, p.q)
+}