fix after rebase

grr heap usage!

e2e/helpers_test.go

@@ -4,13 +4,15 @@
 package e2e
 
 import (
-	"log/slog"
+	"io"
 	"net/netip"
 	"os"
 	"strings"
 	"testing"
 	"time"
 
+	"log/slog"
+
 	"dario.cat/mergo"
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
@@ -380,7 +382,7 @@ func getAddrs(ns []netip.Prefix) []netip.Addr {
 func NewTestLogger() *slog.Logger {
 	v := os.Getenv("TEST_LOGS")
 	if v == "" {
-		return slog.New(slog.DiscardHandler)
+		return slog.New(slog.NewTextHandler(io.Discard, nil))
 	}
 
 	level := slog.LevelInfo

ecn_inner_test.go

@@ -1,6 +1,7 @@
 package nebula
 
 import (
+	"io"
 	"log/slog"
 	"testing"
 )
@@ -49,7 +50,7 @@ func v6WithTC(tc byte) []byte {
 }
 
 func TestApplyOuterECN(t *testing.T) {
-	silent := slog.New(slog.DiscardHandler)
+	silent := slog.New(slog.NewTextHandler(io.Discard, nil))
 	hi := &HostInfo{}
 
 	// Build a v4 packet helper with a given inner ECN field.

firewall.go

@@ -80,8 +80,8 @@ type firewallMetrics struct {
 type FirewallConntrack struct {
 	sync.Mutex
 
-	Conns      map[firewall.Packet]*conn
-	TimerWheel *TimerWheel[firewall.Packet]
+	Conns      map[firewall.PacketKey]*conn
+	TimerWheel *TimerWheel[firewall.PacketKey]
 }
 
 // FirewallTable is the entry point for a rule, the evaluation order is:
@@ -166,8 +166,8 @@ func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Dur
 
 	return &Firewall{
 		Conntrack: &FirewallConntrack{
-			Conns:      make(map[firewall.Packet]*conn),
-			TimerWheel: NewTimerWheel[firewall.Packet](tmin, tmax),
+			Conns:      make(map[firewall.PacketKey]*conn),
+			TimerWheel: NewTimerWheel[firewall.PacketKey](tmin, tmax),
 		},
 		InRules:           newFirewallTable(),
 		OutRules:          newFirewallTable(),
@@ -422,12 +422,27 @@ var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
 
 // Drop returns an error if the packet should be dropped, explaining why. It
 // returns nil if the packet should not be dropped.
-func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error {
-	// Check if we spoke to this tuple, if we did then allow this packet
-	if f.inConns(fp, h, caPool, localCache) {
+//
+// key is the dense conntrack key — used as-is for the inConns fast path
+// without touching fp at all. fp is the rich Packet form rule matching
+// needs (CIDR lookups, family checks); on the conntrack-miss slow path
+// Drop ensures fp is hydrated from key (idempotent if the caller already
+// filled fp). On accept-via-conntrack the caller's fp is left untouched.
+func (f *Firewall) Drop(key firewall.PacketKey, fp *firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error {
+	// Check if we spoke to this tuple, if we did then allow this packet.
+	// Hot path: only the dense key is touched.
+	if f.inConns(key, h, caPool, localCache) {
 		return nil
 	}
 
+	// Conntrack miss → rule matching needs the rich Packet form. Hydrate
+	// from the key if the caller passed a zero-valued fp (the inbound path
+	// after batch.ParsePacket). Outbound callers Hydrate themselves and
+	// skip this hop.
+	if !fp.LocalAddr.IsValid() {
+		key.Hydrate(fp)
+	}
+
 	// Make sure remote address matches nebula certificate, and determine how to treat it
 	if h.networks == nil {
 		// Simple case: Certificate has one address and no unsafe networks
@@ -467,13 +482,13 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
 	}
 
 	// We now know which firewall table to check against
-	if !table.match(fp, incoming, h.ConnectionState.peerCert, caPool) {
+	if !table.match(*fp, incoming, h.ConnectionState.peerCert, caPool) {
 		f.metrics(incoming).droppedNoRule.Inc(1)
 		return ErrNoMatchingRule
 	}
 
 	// We always want to conntrack since it is a faster operation
-	f.addConn(fp, incoming)
+	f.addConn(key, fp.Protocol, incoming)
 
 	return nil
 }
@@ -502,9 +517,9 @@ func (f *Firewall) EmitStats() {
 	metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV()))
 }
 
-func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool {
+func (f *Firewall) inConns(key firewall.PacketKey, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool {
 	if localCache != nil {
-		if _, ok := localCache[fp]; ok {
+		if _, ok := localCache[key]; ok {
 			return true
 		}
 	}
@@ -517,7 +532,7 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 		f.evict(ep)
 	}
 
-	c, ok := conntrack.Conns[fp]
+	c, ok := conntrack.Conns[key]
 
 	if !ok {
 		conntrack.Unlock()
@@ -526,7 +541,11 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 
 	if c.rulesVersion != f.rulesVersion {
 		// This conntrack entry was for an older rule set, validate
-		// it still passes with the current rule set
+		// it still passes with the current rule set. Rule matching needs
+		// the rich Packet form, so hydrate from key.
+		var fp firewall.Packet
+		key.Hydrate(&fp)
+
 		table := f.OutRules
 		if c.incoming {
 			table = f.InRules
@@ -542,7 +561,7 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 					"oldRulesVersion", c.rulesVersion,
 				)
 			}
-			delete(conntrack.Conns, fp)
+			delete(conntrack.Conns, key)
 			conntrack.Unlock()
 			return false
 		}
@@ -559,7 +578,7 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 		c.rulesVersion = f.rulesVersion
 	}
 
-	switch fp.Protocol {
+	switch key.Protocol {
 	case firewall.ProtoTCP:
 		c.Expires = time.Now().Add(f.TCPTimeout)
 	case firewall.ProtoUDP:
@@ -571,17 +590,17 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 	conntrack.Unlock()
 
 	if localCache != nil {
-		localCache[fp] = struct{}{}
+		localCache[key] = struct{}{}
 	}
 
 	return true
 }
 
-func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
+func (f *Firewall) addConn(key firewall.PacketKey, protocol uint8, incoming bool) {
 	var timeout time.Duration
 	c := &conn{}
 
-	switch fp.Protocol {
+	switch protocol {
 	case firewall.ProtoTCP:
 		timeout = f.TCPTimeout
 	case firewall.ProtoUDP:
@@ -592,9 +611,9 @@ func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
 
 	conntrack := f.Conntrack
 	conntrack.Lock()
-	if _, ok := conntrack.Conns[fp]; !ok {
+	if _, ok := conntrack.Conns[key]; !ok {
 		conntrack.TimerWheel.Advance(time.Now())
-		conntrack.TimerWheel.Add(fp, timeout)
+		conntrack.TimerWheel.Add(key, timeout)
 	}
 
 	// Record which rulesVersion allowed this connection, so we can retest after
@@ -602,16 +621,16 @@ func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
 	c.incoming = incoming
 	c.rulesVersion = f.rulesVersion
 	c.Expires = time.Now().Add(timeout)
-	conntrack.Conns[fp] = c
+	conntrack.Conns[key] = c
 	conntrack.Unlock()
 }
 
 // Evict checks if a conntrack entry has expired, if so it is removed, if not it is re-added to the wheel
 // Caller must own the connMutex lock!
-func (f *Firewall) evict(p firewall.Packet) {
+func (f *Firewall) evict(key firewall.PacketKey) {
 	// Are we still tracking this conn?
 	conntrack := f.Conntrack
-	t, ok := conntrack.Conns[p]
+	t, ok := conntrack.Conns[key]
 	if !ok {
 		return
 	}
@@ -621,12 +640,12 @@ func (f *Firewall) evict(p firewall.Packet) {
 	// Timeout is in the future, re-add the timer
 	if newT > 0 {
 		conntrack.TimerWheel.Advance(time.Now())
-		conntrack.TimerWheel.Add(p, newT)
+		conntrack.TimerWheel.Add(key, newT)
 		return
 	}
 
 	// This conn is done
-	delete(conntrack.Conns, p)
+	delete(conntrack.Conns, key)
 }
 
 func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {

firewall/cache.go

@@ -10,8 +10,10 @@ import (
 )
 
 // ConntrackCache is used as a local routine cache to know if a given flow
-// has been seen in the conntrack table.
-type ConntrackCache map[Packet]struct{}
+// has been seen in the conntrack table. Keyed on PacketKey (dense form)
+// rather than Packet so the lookup hashes raw bytes instead of the
+// unique.Handle each netip.Addr in Packet carries.
+type ConntrackCache map[PacketKey]struct{}
 
 type ConntrackCacheTicker struct {
 	cacheV    uint64

firewall/cache_test.go

@@ -23,7 +23,7 @@ func newFixedTicker(t *testing.T, l *slog.Logger, cacheLen int) *ConntrackCacheT
 		cache: make(ConntrackCache, cacheLen),
 	}
 	for i := 0; i < cacheLen; i++ {
-		c.cache[Packet{LocalPort: uint16(i) + 1}] = struct{}{}
+		c.cache[PacketKey{LocalPort: uint16(i) + 1}] = struct{}{}
 	}
 	c.cacheTick.Store(1) // cacheV starts at 0, so Get() takes the reset path
 	return c

firewall/packet.go

@@ -19,6 +19,25 @@ const (
 	PortFragment = -1 // Special value for matching `port: fragment`
 )
 
+// PacketKey is the firewall's conntrack and ConntrackCache map key — the
+// dense form of the 5-tuple plus the protocol and fragment flag the
+// firewall actually discriminates flows on. Kept separate from Packet so
+// the conntrack-hit fast path doesn't pay for hashing the unique.Handle
+// each netip.Addr carries, and so the inbound parser can skip the
+// AddrFrom4/AddrFrom16 calls until rule matching actually needs them.
+//
+// Superset of the coalescer's flowKey shape (same 5-tuple, just in
+// Local/Remote orientation rather than wire src/dst).
+type PacketKey struct {
+	LocalAddr  [16]byte
+	RemoteAddr [16]byte
+	LocalPort  uint16
+	RemotePort uint16
+	IsV6       bool
+	Protocol   uint8
+	Fragment   bool
+}
+
 type Packet struct {
 	LocalAddr  netip.Addr
 	RemoteAddr netip.Addr
@@ -31,6 +50,61 @@ type Packet struct {
 	Fragment   bool
 }
 
+// Key derives a PacketKey from a populated Packet. Used by the few code
+// paths that have a Packet but no Key in hand (e.g. tests). Both inbound
+// and outbound production parsers write straight into a PacketKey via
+// batch.ParsePacket, so this function is rarely on the hot path.
+func (fp *Packet) Key() PacketKey {
+	k := PacketKey{
+		Protocol: fp.Protocol,
+		Fragment: fp.Fragment,
+	}
+	k.LocalPort = fp.LocalPort
+	k.RemotePort = fp.RemotePort
+	k.IsV6 = !fp.LocalAddr.Is4()
+	if k.IsV6 {
+		k.LocalAddr = fp.LocalAddr.As16()
+		k.RemoteAddr = fp.RemoteAddr.As16()
+	} else {
+		v4 := fp.LocalAddr.As4()
+		copy(k.LocalAddr[:4], v4[:])
+		v4 = fp.RemoteAddr.As4()
+		copy(k.RemoteAddr[:4], v4[:])
+	}
+	return k
+}
+
+// Hydrate fills fp's netip.Addr fields and copies the rest from k. Called
+// by the firewall slow path when conntrack misses and rule matching needs
+// the rich Packet form (CIDR lookups, family checks). The fast path skips
+// this entirely.
+func (k *PacketKey) Hydrate(fp *Packet) {
+	fp.LocalPort = k.LocalPort
+	fp.RemotePort = k.RemotePort
+	fp.Protocol = k.Protocol
+	fp.Fragment = k.Fragment
+	if k.IsV6 {
+		fp.LocalAddr = netip.AddrFrom16(k.LocalAddr)
+		fp.RemoteAddr = netip.AddrFrom16(k.RemoteAddr)
+	} else {
+		var v4 [4]byte
+		copy(v4[:], k.LocalAddr[:4])
+		fp.LocalAddr = netip.AddrFrom4(v4)
+		copy(v4[:], k.RemoteAddr[:4])
+		fp.RemoteAddr = netip.AddrFrom4(v4)
+	}
+}
+
+func (k *PacketKey) GetRemoteAddr() netip.Addr {
+	if k.IsV6 {
+		return netip.AddrFrom16(k.RemoteAddr)
+	} else {
+		var v4 [4]byte
+		copy(v4[:], k.RemoteAddr[:4])
+		return netip.AddrFrom4(v4)
+	}
+}
+
 func (fp *Packet) Copy() *Packet {
 	return &Packet{
 		LocalAddr:  fp.LocalAddr,

firewall_test.go

@@ -211,44 +211,44 @@ func TestFirewall_Drop(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// Drop outbound
-	assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil))
+	assert.Equal(t, ErrNoMatchingRule, fw.Drop(p.Key(), &p, false, &h, cp, nil))
 	// Allow inbound
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p.Key(), &p, true, &h, cp, nil))
 	// Allow outbound because conntrack
-	require.NoError(t, fw.Drop(p, false, &h, cp, nil))
+	require.NoError(t, fw.Drop(p.Key(), &p, false, &h, cp, nil))
 
 	// test remote mismatch
 	oldRemote := p.RemoteAddr
 	p.RemoteAddr = netip.MustParseAddr("1.2.3.10")
-	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP)
+	assert.Equal(t, fw.Drop(p.Key(), &p, false, &h, cp, nil), ErrInvalidRemoteIP)
 	p.RemoteAddr = oldRemote
 
 	// ensure signer doesn't get in the way of group checks
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum"))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad"))
-	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p.Key(), &p, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caSha doesn't drop on match
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad"))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum"))
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p.Key(), &p, true, &h, cp, nil))
 
 	// ensure ca name doesn't get in the way of group checks
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", ""))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", ""))
-	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p.Key(), &p, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caName doesn't drop on match
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", ""))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", ""))
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p.Key(), &p, true, &h, cp, nil))
 }
 
 func TestFirewall_DropV6(t *testing.T) {
@@ -289,44 +289,44 @@ func TestFirewall_DropV6(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// Drop outbound
-	assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil))
+	assert.Equal(t, ErrNoMatchingRule, fw.Drop(p.Key(), &p, false, &h, cp, nil))
 	// Allow inbound
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p.Key(), &p, true, &h, cp, nil))
 	// Allow outbound because conntrack
-	require.NoError(t, fw.Drop(p, false, &h, cp, nil))
+	require.NoError(t, fw.Drop(p.Key(), &p, false, &h, cp, nil))
 
 	// test remote mismatch
 	oldRemote := p.RemoteAddr
 	p.RemoteAddr = netip.MustParseAddr("fd12::56")
-	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP)
+	assert.Equal(t, fw.Drop(p.Key(), &p, false, &h, cp, nil), ErrInvalidRemoteIP)
 	p.RemoteAddr = oldRemote
 
 	// ensure signer doesn't get in the way of group checks
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum"))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad"))
-	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p.Key(), &p, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caSha doesn't drop on match
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad"))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum"))
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p.Key(), &p, true, &h, cp, nil))
 
 	// ensure ca name doesn't get in the way of group checks
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", ""))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", ""))
-	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p.Key(), &p, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caName doesn't drop on match
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", ""))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", ""))
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p.Key(), &p, true, &h, cp, nil))
 }
 
 func BenchmarkFirewallTable_match(b *testing.B) {
@@ -533,10 +533,10 @@ func TestFirewall_Drop2(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// h1/c1 lacks the proper groups
-	require.ErrorIs(t, fw.Drop(p, true, &h1, cp, nil), ErrNoMatchingRule)
+	require.ErrorIs(t, fw.Drop(p.Key(), &p, true, &h1, cp, nil), ErrNoMatchingRule)
 	// c has the proper groups
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p.Key(), &p, true, &h, cp, nil))
 }
 
 func TestFirewall_Drop3(t *testing.T) {
@@ -613,18 +613,18 @@ func TestFirewall_Drop3(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// c1 should pass because host match
-	require.NoError(t, fw.Drop(p, true, &h1, cp, nil))
+	require.NoError(t, fw.Drop(p.Key(), &p, true, &h1, cp, nil))
 	// c2 should pass because ca sha match
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, true, &h2, cp, nil))
+	require.NoError(t, fw.Drop(p.Key(), &p, true, &h2, cp, nil))
 	// c3 should fail because no match
 	resetConntrack(fw)
-	assert.Equal(t, fw.Drop(p, true, &h3, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p.Key(), &p, true, &h3, cp, nil), ErrNoMatchingRule)
 
 	// Test a remote address match
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "1.2.3.4/24", "", "", ""))
-	require.NoError(t, fw.Drop(p, true, &h1, cp, nil))
+	require.NoError(t, fw.Drop(p.Key(), &p, true, &h1, cp, nil))
 }
 
 func TestFirewall_Drop3V6(t *testing.T) {
@@ -661,7 +661,7 @@ func TestFirewall_Drop3V6(t *testing.T) {
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
 	cp := cert.NewCAPool()
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "fd12::34/120", "", "", ""))
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p.Key(), &p, true, &h, cp, nil))
 }
 
 func TestFirewall_DropConntrackReload(t *testing.T) {
@@ -702,12 +702,12 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// Drop outbound
-	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p.Key(), &p, false, &h, cp, nil), ErrNoMatchingRule)
 	// Allow inbound
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p.Key(), &p, true, &h, cp, nil))
 	// Allow outbound because conntrack
-	require.NoError(t, fw.Drop(p, false, &h, cp, nil))
+	require.NoError(t, fw.Drop(p.Key(), &p, false, &h, cp, nil))
 
 	oldFw := fw
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
@@ -716,7 +716,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	fw.rulesVersion = oldFw.rulesVersion + 1
 
 	// Allow outbound because conntrack and new rules allow port 10
-	require.NoError(t, fw.Drop(p, false, &h, cp, nil))
+	require.NoError(t, fw.Drop(p.Key(), &p, false, &h, cp, nil))
 
 	oldFw = fw
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
@@ -725,7 +725,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	fw.rulesVersion = oldFw.rulesVersion + 1
 
 	// Drop outbound because conntrack doesn't match new ruleset
-	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p.Key(), &p, false, &h, cp, nil), ErrNoMatchingRule)
 }
 
 func TestFirewall_ICMPPortBehavior(t *testing.T) {
@@ -770,12 +770,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0
 			p.RemotePort = 0
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(p.Key(), p, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			require.NoError(t, fw.Drop(*p, true, &h, cp, nil))
+			require.NoError(t, fw.Drop(p.Key(), p, true, &h, cp, nil))
 			//now also allow outbound
-			require.NoError(t, fw.Drop(*p, false, &h, cp, nil))
+			require.NoError(t, fw.Drop(p.Key(), p, false, &h, cp, nil))
 		})
 
 		t.Run("nonzero ports", func(t *testing.T) {
@@ -783,12 +783,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0xabcd
 			p.RemotePort = 0x1234
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(p.Key(), p, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			require.NoError(t, fw.Drop(*p, true, &h, cp, nil))
+			require.NoError(t, fw.Drop(p.Key(), p, true, &h, cp, nil))
 			//now also allow outbound
-			require.NoError(t, fw.Drop(*p, false, &h, cp, nil))
+			require.NoError(t, fw.Drop(p.Key(), p, false, &h, cp, nil))
 		})
 	})
 
@@ -800,12 +800,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0
 			p.RemotePort = 0
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(p.Key(), p, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(p.Key(), p, true, &h, cp, nil), ErrNoMatchingRule)
 			//now also allow outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(p.Key(), p, false, &h, cp, nil), ErrNoMatchingRule)
 		})
 
 		t.Run("nonzero ports, still blocked", func(t *testing.T) {
@@ -813,12 +813,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0xabcd
 			p.RemotePort = 0x1234
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(p.Key(), p, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(p.Key(), p, true, &h, cp, nil), ErrNoMatchingRule)
 			//now also allow outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(p.Key(), p, false, &h, cp, nil), ErrNoMatchingRule)
 		})
 
 		t.Run("nonzero, matching ports, still blocked", func(t *testing.T) {
@@ -826,12 +826,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 80
 			p.RemotePort = 80
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(p.Key(), p, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(p.Key(), p, true, &h, cp, nil), ErrNoMatchingRule)
 			//now also allow outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(p.Key(), p, false, &h, cp, nil), ErrNoMatchingRule)
 		})
 	})
 	t.Run("Any proto, any port", func(t *testing.T) {
@@ -843,12 +843,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0
 			p.RemotePort = 0
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(p.Key(), p, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			require.NoError(t, fw.Drop(*p, true, &h, cp, nil))
+			require.NoError(t, fw.Drop(p.Key(), p, true, &h, cp, nil))
 			//now also allow outbound
-			require.NoError(t, fw.Drop(*p, false, &h, cp, nil))
+			require.NoError(t, fw.Drop(p.Key(), p, false, &h, cp, nil))
 		})
 
 		t.Run("nonzero ports, allowed", func(t *testing.T) {
@@ -857,15 +857,15 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0xabcd
 			p.RemotePort = 0x1234
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(p.Key(), p, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			require.NoError(t, fw.Drop(*p, true, &h, cp, nil))
+			require.NoError(t, fw.Drop(p.Key(), p, true, &h, cp, nil))
 			//now also allow outbound
-			require.NoError(t, fw.Drop(*p, false, &h, cp, nil))
+			require.NoError(t, fw.Drop(p.Key(), p, false, &h, cp, nil))
 			//different ID is blocked
 			p.RemotePort++
-			require.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			require.Equal(t, fw.Drop(p.Key(), p, false, &h, cp, nil), ErrNoMatchingRule)
 		})
 	})
 
@@ -913,7 +913,7 @@ func TestFirewall_DropIPSpoofing(t *testing.T) {
 		Protocol:   firewall.ProtoUDP,
 		Fragment:   false,
 	}
-	assert.Equal(t, fw.Drop(p, true, &h1, cp, nil), ErrInvalidRemoteIP)
+	assert.Equal(t, fw.Drop(p.Key(), &p, true, &h1, cp, nil), ErrInvalidRemoteIP)
 }
 
 func BenchmarkLookup(b *testing.B) {
@@ -1327,7 +1327,7 @@ func (c *testcase) Test(t *testing.T, fw *Firewall) {
 	t.Helper()
 	cp := cert.NewCAPool()
 	resetConntrack(fw)
-	err := fw.Drop(c.p, true, c.h, cp, nil)
+	err := fw.Drop(c.p.Key(), &c.p, true, c.h, cp, nil)
 	if c.err == nil {
 		require.NoError(t, err, "failed to not drop remote address %s", c.p.RemoteAddr)
 	} else {
@@ -1519,6 +1519,6 @@ func (mf *mockFirewall) AddRule(incoming bool, proto uint8, startPort int32, end
 
 func resetConntrack(fw *Firewall) {
 	fw.Conntrack.Lock()
-	fw.Conntrack.Conns = map[firewall.Packet]*conn{}
+	fw.Conntrack.Conns = map[firewall.PacketKey]*conn{}
 	fw.Conntrack.Unlock()
 }

go.mod

@@ -43,6 +43,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/google/btree v1.1.2 // indirect
+	github.com/guptarohit/asciigraph v0.9.0 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect

go.sum

@@ -60,6 +60,8 @@ github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
+github.com/guptarohit/asciigraph v0.9.0 h1:MvCSRRVkT2XvU1IO6n92o7l7zqx1DiFaoszOUZQztbY=
+github.com/guptarohit/asciigraph v0.9.0/go.mod h1:dYl5wwK4gNsnFf9Zp+l06rFiDZ5YtXM6x7SRWZ3KGag=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=

inside.go

@@ -11,18 +11,25 @@ import (
 	"github.com/slackhq/nebula/iputil"
 	"github.com/slackhq/nebula/noiseutil"
 	"github.com/slackhq/nebula/overlay/batch"
+	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
-	"github.com/slackhq/nebula/wire"
 )
 
-func (f *Interface) consumeInsidePacket(pkt wire.TunPacket, fwPacket *firewall.Packet, nb []byte, sendBatch *batch.SendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
+func (f *Interface) consumeInsidePacket(pkt tio.Packet, fwPacket *firewall.Packet, nb []byte, sendBatch batch.TxBatcher, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
+	// borrowed: pkt.Bytes is owned by the originating tio.Queue and is
+	// only valid until the next Read on that queue. Every consumer below
+	// (parse, self-forward, handshake cache, sendInsideMessage) reads it
+	// synchronously; do not retain pkt outside this call. If a future
+	// caller needs to keep the packet, use pkt.Clone() to detach it from
+	// the borrow.
+	//
 	// pkt.Bytes is either one IP datagram (GSO zero) or a TSO/USO
 	// superpacket. In both cases the L3+L4 headers at the start describe
-	// the same 5-tuple every segment will share, so a single newPacket /
+	// the same 5-tuple every segment will share, so a single parse +
 	// firewall check covers the whole superpacket.
 	packet := pkt.Bytes
-	err := newPacket(packet, false, fwPacket)
-	if err != nil {
+	var parsed batch.RxParsed
+	if err := batch.ParsePacket(packet, false, &parsed); err != nil {
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			f.l.Debug("Error while validating outbound packet",
 				"packet", packet,
@@ -32,6 +39,8 @@ func (f *Interface) consumeInsidePacket(pkt wire.TunPacket, fwPacket *firewall.P
 		return
 	}
 
+	parsed.Key.Hydrate(fwPacket)
+
 	// Ignore local broadcast packets
 	if f.dropLocalBroadcast {
 		if f.myBroadcastAddrsTable.Contains(fwPacket.RemoteAddr) {
@@ -45,7 +54,11 @@ func (f *Interface) consumeInsidePacket(pkt wire.TunPacket, fwPacket *firewall.P
 		// routes packets from the Nebula addr to the Nebula addr through the Nebula
 		// TUN device.
 		if immediatelyForwardToSelf {
-			err := pkt.PerSegment(func(seg []byte) error {
+			// Write copies into the kernel queue synchronously, so seg's lifetime ends at return.
+			// A self-forwarded superpacket would be re-handed to the
+			// kernel as one giant blob; segment first so the loopback
+			// path sees one IP datagram per Write.
+			err := tio.SegmentSuperpacket(pkt, func(seg []byte) error {
 				_, werr := f.readers[q].Write(seg)
 				return werr
 			})
@@ -64,10 +77,10 @@ func (f *Interface) consumeInsidePacket(pkt wire.TunPacket, fwPacket *firewall.P
 	}
 
 	hostinfo, ready := f.getOrHandshakeConsiderRouting(fwPacket, func(hh *HandshakeHostInfo) {
-		// borrowed: PerSegment builds each segment in the kernel-supplied pkt
+		// borrowed: SegmentSuperpacket builds each segment in the kernel-supplied pkt
 		// bytes underneath. cachePacket explicitly copies its argument (handshake_manager.go cachePacket),
 		// so retaining segments past the loop is safe.
-		err := pkt.PerSegment(func(seg []byte) error {
+		err := tio.SegmentSuperpacket(pkt, func(seg []byte) error {
 			hh.cachePacket(f.l, header.Message, 0, seg, f.sendMessageNow, f.cachedPacketMetrics)
 			return nil
 		})
@@ -94,9 +107,9 @@ func (f *Interface) consumeInsidePacket(pkt wire.TunPacket, fwPacket *firewall.P
 		return
 	}
 
-	dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
+	dropReason := f.firewall.Drop(parsed.Key, fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason == nil {
-		f.sendInsideMessage(hostinfo, pkt, nb, sendBatch)
+		f.sendInsideMessage(hostinfo, pkt, nb, sendBatch, rejectBuf, q)
 	} else {
 		f.rejectInside(packet, rejectBuf, q)
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
@@ -139,10 +152,10 @@ func (f *Interface) sendInsideEncrypt(hostinfo *HostInfo, ci *ConnectionState, s
 // segment of a TSO/USO superpacket) into the caller's batch slot for
 // later sendmmsg flush. Segmentation is fused with encryption here so the
 // kernel-supplied superpacket bytes never get written into a separate
-// scratch arena: PerSegment builds each segment's plaintext in
+// scratch arena: SegmentSuperpacket builds each segment's plaintext in
 // segScratch[:segLen] in turn, and we encrypt directly into a fresh
 // SendBatch slot.
-func (f *Interface) sendInsideMessage(hostinfo *HostInfo, pkt wire.TunPacket, nb []byte, sendBatch *batch.SendBatch) {
+func (f *Interface) sendInsideMessage(hostinfo *HostInfo, pkt tio.Packet, nb []byte, sendBatch batch.TxBatcher, rejectBuf []byte, q int) {
 	ci := hostinfo.ConnectionState
 	if ci.eKey == nil {
 		return
@@ -183,7 +196,7 @@ func (f *Interface) sendInsideMessage(hostinfo *HostInfo, pkt wire.TunPacket, nb
 			return
 		}
 
-		err = pkt.PerSegment(func(seg []byte) error {
+		err = tio.SegmentSuperpacket(pkt, func(seg []byte) error {
 			//relay header + header + plaintext + AEAD tag (16 bytes for both AES-GCM and ChaCha20-Poly1305) + relay tag
 			scratch := sendBatch.Reserve(header.Len + header.Len + len(seg) + 16 + 16)
 
@@ -212,7 +225,7 @@ func (f *Interface) sendInsideMessage(hostinfo *HostInfo, pkt wire.TunPacket, nb
 		return
 	}
 
-	err := pkt.PerSegment(func(seg []byte) error {
+	err := tio.SegmentSuperpacket(pkt, func(seg []byte) error {
 		// header + plaintext + AEAD tag (16 bytes for both AES-GCM and ChaCha20-Poly1305)
 		scratch := sendBatch.Reserve(header.Len + len(seg) + 16)
 
@@ -381,15 +394,16 @@ func (f *Interface) getOrHandshakeConsiderRouting(fwPacket *firewall.Packet, cac
 }
 
 func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte) {
-	fp := &firewall.Packet{}
-	err := newPacket(p, false, fp)
-	if err != nil {
+	var parsed batch.RxParsed
+	if err := batch.ParsePacket(p, false, &parsed); err != nil {
 		f.l.Warn("error while parsing outgoing packet for firewall check", "error", err)
 		return
 	}
+	fp := &firewall.Packet{}
+	parsed.Key.Hydrate(fp)
 
 	// check if packet is in outbound fw rules
-	dropReason := f.firewall.Drop(*fp, false, hostinfo, f.pki.GetCAPool(), nil)
+	dropReason := f.firewall.Drop(parsed.Key, fp, false, hostinfo, f.pki.GetCAPool(), nil)
 	if dropReason != nil {
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			f.l.Debug("dropping cached packet",
@@ -500,9 +514,8 @@ func (f *Interface) prepareSendVia(via *HostInfo,
 // via is the HostInfo through which the message is relayed.
 // ad is the plaintext data to authenticate, but not encrypt
 // nb is a buffer used to store the nonce value, re-used for performance reasons.
-// out is a buffer used to store the result of the Encrypt operation.
-// The write goes through writers[0] — SendVia is called from contexts
-// without a per-queue index (handshake, async control paths).
+// out is a buffer used to store the result of the Encrypt operation
+// q indicates which writer to use to send the packet.
 func (f *Interface) SendVia(via *HostInfo,
 	relay *Relay,
 	ad,
@@ -511,10 +524,6 @@ func (f *Interface) SendVia(via *HostInfo,
 	nocopy bool,
 ) {
 	toSend, err := f.prepareSendVia(via, relay, ad, nb, out, nocopy)
-	if err != nil {
-		via.logger(f.l).Info("Failed to prepareSendVia", "error", err)
-		return
-	}
 	err = f.writers[0].WriteTo(toSend, via.remote)
 	if err != nil {
 		via.logger(f.l).Info("Failed to WriteTo in sendVia", "error", err)

interface.go

@@ -14,7 +14,6 @@ import (
 	"github.com/gaissmai/bart"
 	"github.com/rcrowley/go-metrics"
 	"github.com/slackhq/nebula/util"
-	"github.com/slackhq/nebula/wire"
 
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/firewall"
@@ -41,7 +40,6 @@ type InterfaceConfig struct {
 	DropLocalBroadcast bool
 	DropMulticast      bool
 	routines           int
-	batchSize          int
 	MessageMetrics     *MessageMetrics
 	version            string
 	relayManager       *relayManager
@@ -81,7 +79,6 @@ type Interface struct {
 	dropLocalBroadcast    bool
 	dropMulticast         bool
 	routines              int
-	batchSize             int
 	disconnectInvalid     atomic.Bool
 	closed                atomic.Bool
 	// cpuAffinity, when non-empty, names the CPUs each TUN reader goroutine
@@ -211,7 +208,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 		dropLocalBroadcast:    c.DropLocalBroadcast,
 		dropMulticast:         c.DropMulticast,
 		routines:              c.routines,
-		batchSize:             c.batchSize,
 		version:               c.version,
 		writers:               make([]udp.Conn, c.routines),
 		readers:               make([]tio.Queue, c.routines),
@@ -283,17 +279,15 @@ func (f *Interface) activate() error {
 	}
 	f.readers = f.inside.Readers()
 	for i := range f.readers {
-		caps := f.readers[i].Capabilities()
+		caps := tio.QueueCapabilities(f.readers[i])
 		if caps.TSO || caps.USO {
 			// Multi-lane: TCP gets coalesced when TSO is on, UDP when USO
 			// is on, everything else (and either lane disabled) falls
 			// through to passthrough so non-IP / non-TCP-UDP traffic still
 			// reaches the TUN.
-			arena := util.NewArena(max(f.batchSize, 1) * 65535)
-			f.batchers[i] = batch.NewMultiCoalescer(f.readers[i], f.l, arena, caps.TSO, caps.USO)
+			f.batchers[i] = batch.NewMultiCoalescer(f.readers[i], caps.TSO, caps.USO)
 		} else {
-			arena := util.NewArena(max(f.batchSize, 1) * udp.MTU)
-			f.batchers[i] = batch.NewPassthrough(f.readers[i], f.batchSize, arena)
+			f.batchers[i] = batch.NewPassthrough(f.readers[i])
 		}
 	}
 
@@ -354,10 +348,12 @@ func (f *Interface) listenOut(i int) {
 	lhh := f.lightHouse.NewRequestHandler()
 	h := &header.H{}
 	fwPacket := &firewall.Packet{}
+	parsedRx := &batch.RxParsed{}
 	nb := make([]byte, 12, 12)
 
 	listener := func(fromUdpAddr netip.AddrPort, payload []byte, meta udp.RxMeta) {
-		f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, payload, h, fwPacket, lhh, nb, i, ctCache.Get(), meta)
+		plaintext := f.batchers[i].Reserve(len(payload))
+		f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, plaintext[:0], payload, h, fwPacket, parsedRx, lhh, nb, i, ctCache.Get(), meta)
 	}
 
 	flusher := func() {
@@ -376,57 +372,52 @@ func (f *Interface) listenOut(i int) {
 	f.l.Debug("underlay reader is done", "reader", i)
 }
 
-func (f *Interface) listenIn(reader tio.Queue, q int) {
-	// Pinning this thread (and goroutine) to a single CPU keeps every sendmmsg from this goroutine going through the
-	// same TX ring on the nic, so the wire sees per-flow order.
-	cpu := q % runtime.NumCPU()
+func (f *Interface) listenIn(reader tio.Queue, i int) {
+	// Pin this goroutine to one CPU. LockOSThread alone keeps the goroutine
+	// on a single OS thread but the kernel can still migrate that thread
+	// across CPUs — XPS reads smp_processor_id() at sendmmsg time and picks
+	// the TX ring from the current CPU's xps_cpus map, so an unpinned
+	// thread bouncing between CPUs spreads one nebula flow's packets across
+	// multiple TX rings, which the rings then drain at independent rates
+	// and the wire delivers reordered.
+	//
+	// Pinning keeps every sendmmsg from this goroutine going through the
+	// same TX ring, so the wire sees per-flow order. Cost: less scheduler
+	// flexibility — if i % NumCPU collides between two TUN reader
+	// goroutines they share a CPU.
+	cpu := i % runtime.NumCPU()
 	if n := len(f.cpuAffinity); n > 0 {
-		cpu = f.cpuAffinity[q%n]
+		cpu = f.cpuAffinity[i%n]
 	}
 	if err := util.PinThreadToCPU(cpu); err != nil {
-		f.l.Warn("failed to pin tun reader to CPU", "queue", q, "cpu", cpu, "err", err)
+		f.l.Warn("failed to pin tun reader to CPU", "queue", i, "cpu", cpu, "err", err)
 	}
-
-	const bonusInfo = 16
-	bufferScale := udp.MTU + bonusInfo
-	numTunPackets := 1
-	caps := reader.Capabilities()
-	if caps.TSO || caps.USO {
-		bufferScale = 65535 + bonusInfo
-		numTunPackets = f.batchSize
-	}
-
 	rejectBuf := make([]byte, mtu)
-	tunPackets := make([]wire.TunPacket, numTunPackets)
-	packetMem := make([]byte, bufferScale*numTunPackets)
-
-	arenaSize := batch.SendBatchCap * (udp.MTU + 32)
-	sb := batch.NewSendBatch(f.writers[q], batch.SendBatchCap, util.NewArena(arenaSize))
+	sb := batch.NewSendBatch(f.writers[i], batch.SendBatchCap, udp.MTU+32)
 	fwPacket := &firewall.Packet{}
 	nb := make([]byte, 12, 12)
 
 	conntrackCache := firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout)
 
 	for {
-		n, err := reader.Read(tunPackets, packetMem)
+		pkts, err := reader.Read()
 		if err != nil {
 			if !f.closed.Load() {
-				f.l.Error("Error while reading outbound packet, closing", "error", err, "reader", q)
+				f.l.Error("Error while reading outbound packet, closing", "error", err, "reader", i)
 				f.onFatal(err)
 			}
 			break
 		}
 
-		ctCache := conntrackCache.Get()
-		for i := range n {
-			f.consumeInsidePacket(tunPackets[i], fwPacket, nb, sb, rejectBuf, q, ctCache)
+		for _, pkt := range pkts {
+			f.consumeInsidePacket(pkt, fwPacket, nb, sb, rejectBuf, i, conntrackCache.Get())
 		}
 		if err := sb.Flush(); err != nil {
-			f.l.Error("Failed to write outgoing batch", "error", err, "writer", q)
+			f.l.Error("Failed to write outgoing batch", "error", err, "writer", i)
 		}
 	}
 
-	f.l.Debug("overlay reader is done", "reader", q)
+	f.l.Debug("overlay reader is done", "reader", i)
 }
 
 func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {

main.go

@@ -221,7 +221,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		DropLocalBroadcast:    c.GetBool("tun.drop_local_broadcast", false),
 		DropMulticast:         c.GetBool("tun.drop_multicast", false),
 		routines:              routines,
-		batchSize:             c.GetInt("listen.batch", 64),
 		MessageMetrics:        messageMetrics,
 		version:               buildVersion,
 		relayManager:          NewRelayManager(ctx, l, hostMap, c),

noiseutil/cipher_state_test.go

@@ -147,62 +147,6 @@ func buildCipherStatesB(b *testing.B, c noise.CipherFunc) (*noise.CipherState, *
 	return eI, dR
 }
 
-// TestDecryptDangerRelayShapeNoAlloc covers the AD-only relay path used in
-// outside.go's handleOutsideRelayPacket: the body is AD, the trailing 16 bytes
-// are the AEAD tag, the plaintext is empty, and the caller passes nil as the
-// destination because it only needs the auth side-effect. The call must
-// succeed, return an empty plaintext, and not allocate on the hot path.
-func TestDecryptDangerRelayShapeNoAlloc(t *testing.T) {
-	cases := []struct {
-		name string
-		c    noise.CipherFunc
-		wrap func(*noise.CipherState) CipherState
-	}{
-		{"AESGCM", CipherAESGCM, func(cs *noise.CipherState) CipherState { return NewCipherStateAESGCM(cs) }},
-		{"ChaChaPoly", noise.CipherChaChaPoly, func(cs *noise.CipherState) CipherState { return NewCipherStateChaChaPoly(cs) }},
-	}
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			encCS, decCS := buildCipherStates(t, tc.c)
-			enc, dec := tc.wrap(encCS), tc.wrap(decCS)
-
-			ad := make([]byte, 1200) // typical relay packet body size
-			for i := range ad {
-				ad[i] = byte(i)
-			}
-			nb := make([]byte, 12)
-
-			// Build the "signature value" the way handleOutsideRelayPacket sees it:
-			// empty plaintext encrypted with the body as AD yields just the 16-byte tag.
-			tag, err := enc.EncryptDanger(nil, ad, nil, 1, nb)
-			require.NoError(t, err)
-			require.Len(t, tag, dec.Overhead())
-
-			// Sanity: the relay-shaped call returns empty plaintext, no error.
-			out, err := dec.DecryptDanger(nil, ad, tag, 1, nb)
-			require.NoError(t, err)
-			assert.Empty(t, out)
-
-			// Tampering with the AD must fail authentication.
-			ad[0] ^= 0xff
-			_, err = dec.DecryptDanger(nil, ad, tag, 1, nb)
-			require.Error(t, err)
-			ad[0] ^= 0xff
-
-			// The hot path must not allocate. AllocsPerRun does a warm-up run, so any
-			// one-time setup is excluded. Counter has to advance so the AEAD nonce is
-			// unique per call, but we don't care whether the auth succeeds — we only
-			// care about whether the call path allocates.
-			var counter uint64 = 2
-			allocs := testing.AllocsPerRun(100, func() {
-				_, _ = dec.DecryptDanger(nil, ad, tag, counter, nb)
-				counter++
-			})
-			assert.Equal(t, 0.0, allocs, "DecryptDanger(nil, ...) must not allocate")
-		})
-	}
-}
-
 func TestCipherStateNilSafety(t *testing.T) {
 	var aes *CipherStateAESGCM
 	_, err := aes.EncryptDanger(nil, nil, nil, 0, make([]byte, 12))

outside.go

@@ -2,28 +2,20 @@ package nebula
 
 import (
 	"context"
-	"encoding/binary"
 	"errors"
 	"log/slog"
 	"net/netip"
 	"time"
 
-	"github.com/google/gopacket/layers"
-	"golang.org/x/net/ipv6"
-
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/overlay/batch"
 	"github.com/slackhq/nebula/udp"
-	"golang.org/x/net/ipv4"
-)
-
-const (
-	minFwPacketLen = 4
 )
 
 var ErrOutOfWindow = errors.New("out of window packet")
 
-func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache, meta udp.RxMeta) {
+func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, parsedRx *batch.RxParsed, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache, meta udp.RxMeta) {
 	err := h.Parse(packet)
 	if err != nil {
 		// Hole punch packets are 0 or 1 byte big, so lets ignore printing those errors
@@ -111,11 +103,11 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, h *header.H
 
 	// Relay packets are special
 	if isMessageRelay {
-		f.handleOutsideRelayPacket(hostinfo, via, packet, h, fwPacket, lhf, nb, q, localCache, meta)
+		f.handleOutsideRelayPacket(hostinfo, via, out, packet, h, fwPacket, parsedRx, lhf, nb, q, localCache, meta)
+
 		return
 	}
 
-	out := f.batchers[q].Reserve(len(packet))[:0]
 	out, err = f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 	if err != nil {
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
@@ -136,7 +128,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, h *header.H
 	case header.Message:
 		switch h.Subtype {
 		case header.MessageNone:
-			f.handleOutsideMessagePacket(hostinfo, out, packet, fwPacket, nb, q, localCache, meta)
+			f.handleOutsideMessagePacket(hostinfo, out, packet, fwPacket, parsedRx, nb, q, localCache, meta)
 		default:
 			hostinfo.logger(f.l).Error("IsValidSubType was true, but unexpected message subtype seen", "from", via, "header", h)
 			return
@@ -169,7 +161,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, h *header.H
 	}
 }
 
-func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache, meta udp.RxMeta) {
+func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, parsedRx *batch.RxParsed, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache, meta udp.RxMeta) {
 	// The entire body is sent as AD, not encrypted.
 	// The packet consists of a 16-byte parsed Nebula header, Associated Data-protected payload, and a trailing 16-byte AEAD signature value.
 	// The packet is guaranteed to be at least 16 bytes at this point, b/c it got past the h.Parse() call above. If it's
@@ -177,10 +169,9 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
 	// which will gracefully fail in the DecryptDanger call.
 	signedPayload := packet[:len(packet)-hostinfo.ConnectionState.dKey.Overhead()]
 	signatureValue := packet[len(packet)-hostinfo.ConnectionState.dKey.Overhead():]
-	// The decrypted output is empty (relay packets carry their payload as AD) and unused.
-	// The recursive readOutsidePackets call below operates on signedPayload. Passing
-	// nil avoids reserving an arena slot.
-	if _, err := hostinfo.ConnectionState.dKey.DecryptDanger(nil, signedPayload, signatureValue, h.MessageCounter, nb); err != nil {
+	var err error
+	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, signedPayload, signatureValue, h.MessageCounter, nb)
+	if err != nil {
 		return
 	}
 	// Successfully validated the thing. Get rid of the Relay header.
@@ -213,7 +204,8 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
 			relay:     relay,
 			IsRelayed: true,
 		}
-		f.readOutsidePackets(via, signedPayload, h, fwPacket, lhf, nb, q, localCache, meta)
+		f.readOutsidePackets(via, out[:0], signedPayload, h, fwPacket, parsedRx, lhf, nb, q, localCache, meta)
+		return
 	case ForwardingType:
 		// Find the target HostInfo relay object
 		targetHI, targetRelay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relay.PeerAddr)
@@ -232,7 +224,6 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
 			case ForwardingType:
 				// Forward this packet through the relay tunnel
 				// Find the target HostInfo //todo it would potentially be nice to batch these
-				out := f.batchers[q].Reserve(len(packet) + header.Len + hostinfo.ConnectionState.dKey.Overhead())[:0]
 				f.SendVia(targetHI, targetRelay, signedPayload, nb, out, false)
 			case TerminalType:
 				hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
@@ -313,191 +304,16 @@ var (
 )
 
 // newPacket validates and parses the interesting bits for the firewall out of the ip and sub protocol headers
+// newPacket parses data into a fully-hydrated firewall.Packet — kept as a
+// thin wrapper around newPacketKey + Hydrate so there's one source of
+// parse logic. Callers that don't need the netip.Addr-rich form (e.g.
+// conntrack-only paths) should use newPacketKey directly.
 func newPacket(data []byte, incoming bool, fp *firewall.Packet) error {
-	if len(data) < 1 {
-		return ErrPacketTooShort
+	var parsed batch.RxParsed
+	if err := batch.ParsePacket(data, incoming, &parsed); err != nil {
+		return err
 	}
-
-	version := int((data[0] >> 4) & 0x0f)
-	switch version {
-	case ipv4.Version:
-		return parseV4(data, incoming, fp)
-	case ipv6.Version:
-		return parseV6(data, incoming, fp)
-	}
-	return ErrUnknownIPVersion
-}
-
-func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
-	dataLen := len(data)
-	if dataLen < ipv6.HeaderLen {
-		return ErrIPv6PacketTooShort
-	}
-
-	if incoming {
-		fp.RemoteAddr, _ = netip.AddrFromSlice(data[8:24])
-		fp.LocalAddr, _ = netip.AddrFromSlice(data[24:40])
-	} else {
-		fp.LocalAddr, _ = netip.AddrFromSlice(data[8:24])
-		fp.RemoteAddr, _ = netip.AddrFromSlice(data[24:40])
-	}
-
-	protoAt := 6             // NextHeader is at 6 bytes into the ipv6 header
-	offset := ipv6.HeaderLen // Start at the end of the ipv6 header
-	next := 0
-	for {
-		if protoAt >= dataLen {
-			break
-		}
-		proto := layers.IPProtocol(data[protoAt])
-
-		switch proto {
-		case layers.IPProtocolESP, layers.IPProtocolNoNextHeader:
-			fp.Protocol = uint8(proto)
-			fp.RemotePort = 0
-			fp.LocalPort = 0
-			fp.Fragment = false
-			return nil
-
-		case layers.IPProtocolICMPv6:
-			if dataLen < offset+6 {
-				return ErrIPv6PacketTooShort
-			}
-			fp.Protocol = uint8(proto)
-			fp.LocalPort = 0 //incoming vs outgoing doesn't matter for icmpv6
-			icmptype := data[offset+1]
-			switch icmptype {
-			case layers.ICMPv6TypeEchoRequest, layers.ICMPv6TypeEchoReply:
-				fp.RemotePort = binary.BigEndian.Uint16(data[offset+4 : offset+6]) //identifier
-			default:
-				fp.RemotePort = 0
-			}
-			fp.Fragment = false
-			return nil
-
-		case layers.IPProtocolTCP, layers.IPProtocolUDP:
-			if dataLen < offset+4 {
-				return ErrIPv6PacketTooShort
-			}
-
-			fp.Protocol = uint8(proto)
-			if incoming {
-				fp.RemotePort = binary.BigEndian.Uint16(data[offset : offset+2])
-				fp.LocalPort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
-			} else {
-				fp.LocalPort = binary.BigEndian.Uint16(data[offset : offset+2])
-				fp.RemotePort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
-			}
-
-			fp.Fragment = false
-			return nil
-
-		case layers.IPProtocolIPv6Fragment:
-			// Fragment header is 8 bytes, need at least offset+4 to read the offset field
-			if dataLen < offset+8 {
-				return ErrIPv6PacketTooShort
-			}
-
-			// Check if this is the first fragment
-			fragmentOffset := binary.BigEndian.Uint16(data[offset+2:offset+4]) &^ uint16(0x7) // Remove the reserved and M flag bits
-			if fragmentOffset != 0 {
-				// Non-first fragment, use what we have now and stop processing
-				fp.Protocol = data[offset]
-				fp.Fragment = true
-				fp.RemotePort = 0
-				fp.LocalPort = 0
-				return nil
-			}
-
-			// The next loop should be the transport layer since we are the first fragment
-			next = 8 // Fragment headers are always 8 bytes
-
-		case layers.IPProtocolAH:
-			// Auth headers, used by IPSec, have a different meaning for header length
-			if dataLen <= offset+1 {
-				break
-			}
-
-			next = int(data[offset+1]+2) << 2
-
-		default:
-			// Normal ipv6 header length processing
-			if dataLen <= offset+1 {
-				break
-			}
-
-			next = int(data[offset+1]+1) << 3
-		}
-
-		if next <= 0 {
-			// Safety check, each ipv6 header has to be at least 8 bytes
-			next = 8
-		}
-
-		protoAt = offset
-		offset = offset + next
-	}
-
-	return ErrIPv6CouldNotFindPayload
-}
-
-func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
-	// Do we at least have an ipv4 header worth of data?
-	if len(data) < ipv4.HeaderLen {
-		return ErrIPv4PacketTooShort
-	}
-
-	// Adjust our start position based on the advertised ip header length
-	ihl := int(data[0]&0x0f) << 2
-
-	// Well-formed ip header length?
-	if ihl < ipv4.HeaderLen {
-		return ErrIPv4InvalidHeaderLength
-	}
-
-	// Check if this is the second or further fragment of a fragmented packet.
-	flagsfrags := binary.BigEndian.Uint16(data[6:8])
-	fp.Fragment = (flagsfrags & 0x1FFF) != 0
-
-	// Firewall handles protocol checks
-	fp.Protocol = data[9]
-
-	// Accounting for a variable header length, do we have enough data for our src/dst tuples?
-	minLen := ihl
-	if !fp.Fragment {
-		if fp.Protocol == firewall.ProtoICMP {
-			minLen += minFwPacketLen + 2
-		} else {
-			minLen += minFwPacketLen
-		}
-	}
-
-	if len(data) < minLen {
-		return ErrIPv4InvalidHeaderLength
-	}
-
-	if incoming { // Firewall packets are locally oriented
-		fp.RemoteAddr, _ = netip.AddrFromSlice(data[12:16])
-		fp.LocalAddr, _ = netip.AddrFromSlice(data[16:20])
-	} else {
-		fp.LocalAddr, _ = netip.AddrFromSlice(data[12:16])
-		fp.RemoteAddr, _ = netip.AddrFromSlice(data[16:20])
-	}
-
-	if fp.Fragment {
-		fp.RemotePort = 0
-		fp.LocalPort = 0
-	} else if fp.Protocol == firewall.ProtoICMP { //note that orientation doesn't matter on ICMP
-		fp.RemotePort = binary.BigEndian.Uint16(data[ihl+4 : ihl+6]) //identifier
-		fp.LocalPort = 0                                             //code would be uint16(data[ihl+1])
-	} else if incoming {
-		fp.RemotePort = binary.BigEndian.Uint16(data[ihl : ihl+2])  //src port
-		fp.LocalPort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
-	} else {
-		fp.LocalPort = binary.BigEndian.Uint16(data[ihl : ihl+2])    //src port
-		fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
-	}
-
+	parsed.Key.Hydrate(fp)
 	return nil
 }
 
@@ -562,7 +378,7 @@ func applyOuterECN(pkt []byte, outerECN byte, hostinfo *HostInfo, l *slog.Logger
 	}
 }
 
-func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache, meta udp.RxMeta) {
+func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, out []byte, packet []byte, fwPacket *firewall.Packet, parsedRx *batch.RxParsed, nb []byte, q int, localCache firewall.ConntrackCache, meta udp.RxMeta) {
 	// RFC 6040 normal-mode combine: fold any outer CE mark stamped by the
 	// underlay into the inner header before firewall + TUN write. Other
 	// outer codepoints are advisory only — we keep the inner unchanged.
@@ -570,7 +386,13 @@ func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, out []byte, p
 		applyOuterECN(out, meta.OuterECN, hostinfo, f.l)
 	}
 
-	err := newPacket(out, true, fwPacket)
+	// Single IP+L4 walk feeds the firewall conntrack key (parsedRx.Key)
+	// and the batcher hint (parsedRx.tcp/udp). Replaces newPacket — and
+	// pointedly does NOT fill fwPacket.LocalAddr/RemoteAddr, since
+	// firewall.Drop's fast path uses Key alone and only hydrates fwPacket
+	// from Key on the slow path.
+	*fwPacket = firewall.Packet{}
+	err := batch.ParsePacket(out, true, parsedRx)
 	if err != nil {
 		hostinfo.logger(f.l).Warn("Error while validating inbound packet",
 			"error", err,
@@ -579,7 +401,7 @@ func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, out []byte, p
 		return
 	}
 
-	dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache)
+	dropReason := f.firewall.Drop(parsedRx.Key, fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason != nil {
 		// NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore
 		// This gives us a buffer to build the reject packet in
@@ -593,7 +415,7 @@ func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, out []byte, p
 		return
 	}
 
-	err = f.batchers[q].Commit(out)
+	err = f.batchers[q].CommitInbound(out, parsedRx)
 	if err != nil {
 		f.l.Error("Failed to write to tun", "error", err)
 	}

outside_test.go

@@ -11,6 +11,7 @@ import (
 	"github.com/google/gopacket/layers"
 
 	"github.com/slackhq/nebula/firewall"
+	"github.com/slackhq/nebula/overlay/batch"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/net/ipv4"
@@ -21,13 +22,13 @@ func Test_newPacket(t *testing.T) {
 
 	// length fails
 	err := newPacket([]byte{}, true, p)
-	require.ErrorIs(t, err, ErrPacketTooShort)
+	require.ErrorIs(t, err, batch.ErrPacketTooShort)
 
 	err = newPacket([]byte{0x40}, true, p)
-	require.ErrorIs(t, err, ErrIPv4PacketTooShort)
+	require.ErrorIs(t, err, batch.ErrIPv4PacketTooShort)
 
 	err = newPacket([]byte{0x60}, true, p)
-	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
+	require.ErrorIs(t, err, batch.ErrIPv6PacketTooShort)
 
 	// length fail with ip options
 	h := ipv4.Header{
@@ -40,15 +41,15 @@ func Test_newPacket(t *testing.T) {
 
 	b, _ := h.Marshal()
 	err = newPacket(b, true, p)
-	require.ErrorIs(t, err, ErrIPv4InvalidHeaderLength)
+	require.ErrorIs(t, err, batch.ErrIPv4InvalidHeaderLength)
 
 	// not an ipv4 packet
 	err = newPacket([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p)
-	require.ErrorIs(t, err, ErrUnknownIPVersion)
+	require.ErrorIs(t, err, batch.ErrUnknownIPVersion)
 
 	// invalid ihl
 	err = newPacket([]byte{4<<4 | (8 >> 2 & 0x0f), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p)
-	require.ErrorIs(t, err, ErrIPv4InvalidHeaderLength)
+	require.ErrorIs(t, err, batch.ErrIPv4InvalidHeaderLength)
 
 	// account for variable ip header length - incoming
 	h = ipv4.Header{
@@ -115,7 +116,7 @@ func Test_newPacket_v6(t *testing.T) {
 	require.NoError(t, err)
 
 	err = newPacket(buffer.Bytes(), true, p)
-	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
+	require.ErrorIs(t, err, batch.ErrIPv6CouldNotFindPayload)
 
 	// A v6 packet with a hop-by-hop extension
 	// ICMPv6 Payload (Echo Request)
@@ -149,12 +150,12 @@ func Test_newPacket_v6(t *testing.T) {
 	// A full IPv6 header and 1 byte in the first extension, but missing
 	// the length byte.
 	err = newPacket(buffer.Bytes()[:41], true, p)
-	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
+	require.ErrorIs(t, err, batch.ErrIPv6CouldNotFindPayload)
 
 	// A full IPv6 header plus 1 full extension, but only 1 byte of the
 	// next layer, missing length byte
 	err = newPacket(buffer.Bytes()[:49], true, p)
-	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
+	require.ErrorIs(t, err, batch.ErrIPv6CouldNotFindPayload)
 	err = nil
 
 	// A good ICMP packet
@@ -217,7 +218,7 @@ func Test_newPacket_v6(t *testing.T) {
 	b = buffer.Bytes()
 	b[6] = 255 // 255 is a reserved protocol number
 	err = newPacket(b, true, p)
-	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
+	require.ErrorIs(t, err, batch.ErrIPv6CouldNotFindPayload)
 
 	// A good UDP packet
 	ip = layers.IPv6{
@@ -264,7 +265,7 @@ func Test_newPacket_v6(t *testing.T) {
 
 	// Too short UDP packet
 	err = newPacket(b[:len(b)-10], false, p) // pull off the last 10 bytes
-	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
+	require.ErrorIs(t, err, batch.ErrIPv6PacketTooShort)
 
 	// A good TCP packet
 	b[6] = byte(layers.IPProtocolTCP)
@@ -291,7 +292,7 @@ func Test_newPacket_v6(t *testing.T) {
 
 	// Too short TCP packet
 	err = newPacket(b[:len(b)-10], false, p) // pull off the last 10 bytes
-	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
+	require.ErrorIs(t, err, batch.ErrIPv6PacketTooShort)
 
 	// A good UDP packet with an AH header
 	ip = layers.IPv6{
@@ -336,12 +337,12 @@ func Test_newPacket_v6(t *testing.T) {
 
 	// Ensure buffer bounds checking during processing
 	err = newPacket(b[:41], true, p)
-	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
+	require.ErrorIs(t, err, batch.ErrIPv6PacketTooShort)
 
 	// Invalid AH header
 	b = buffer.Bytes()
 	err = newPacket(b, true, p)
-	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
+	require.ErrorIs(t, err, batch.ErrIPv6CouldNotFindPayload)
 }
 
 func Test_newPacket_ipv6Fragment(t *testing.T) {
@@ -448,7 +449,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 
 	// Too short of a fragment packet
 	err = newPacket(secondFrag[:len(secondFrag)-10], false, p)
-	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
+	require.ErrorIs(t, err, batch.ErrIPv6PacketTooShort)
 }
 
 func BenchmarkParseV6(b *testing.B) {
@@ -529,7 +530,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("Normal", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(normalPacket, true, fp); err != nil {
+			if err = newPacket(normalPacket, true, fp); err != nil {
 				b.Fatal(err)
 			}
 		}
@@ -537,7 +538,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("FirstFragment", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(firstFrag, true, fp); err != nil {
+			if err = newPacket(firstFrag, true, fp); err != nil {
 				b.Fatal(err)
 			}
 		}
@@ -545,7 +546,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("SecondFragment", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(secondFrag, true, fp); err != nil {
+			if err = newPacket(secondFrag, true, fp); err != nil {
 				b.Fatal(err)
 			}
 		}
@@ -590,7 +591,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("200 HopByHop headers", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(evilBytes, false, fp); err != nil {
+			if err = newPacket(evilBytes, false, fp); err != nil {
 				b.Fatal(err)
 			}
 		}

overlay/batch/batch.go

@@ -0,0 +1,35 @@
+package batch
+
+import "net/netip"
+
+type RxBatcher interface {
+	// Reserve creates a pkt to borrow
+	Reserve(sz int) []byte
+	// Commit borrows pkt. The caller must keep pkt valid until the next Flush.
+	// Walks IP+L4 headers itself; prefer CommitInbound when the caller already
+	// has an RxParsed in hand from ParsePacket.
+	Commit(pkt []byte) error
+	// CommitInbound is Commit with a hint produced by ParsePacket, so the
+	// batcher can skip the IP+L4 re-parse. Borrowed slice contract is the
+	// same as Commit. Implementations that don't coalesce may delegate to
+	// Commit.
+	CommitInbound(pkt []byte, parsed *RxParsed) error
+	// Flush emits every queued packet in arrival order. Returns the
+	// first error observed; keeps draining so one bad packet doesn't hold up
+	// the rest. After Flush returns, borrowed payload slices may be recycled.
+	Flush() error
+}
+
+type TxBatcher interface {
+	// Reserve creates a pkt to borrow
+	Reserve(sz int) []byte
+	// Commit borrows pkt and records its destination plus the 2-bit
+	// IP-level ECN codepoint to set on the outer (carrier) header. The
+	// caller must keep pkt valid until the next Flush. Pass 0 (Not-ECT)
+	// to leave the outer ECN field unset.
+	Commit(pkt []byte, dst netip.AddrPort, outerECN byte)
+	// Flush emits every queued packet via the underlying batch writer in
+	// arrival order. Returns an errors.Join of one or more errors. After Flush returns,
+	// borrowed payload slices may be recycled.
+	Flush() error
+}

overlay/batch/coalesce_core.go

@@ -147,3 +147,17 @@ func mergeECNIntoSeed(seedHdr, pktHdr []byte, isV6 bool) {
 		seedHdr[1] |= pktHdr[1] & 0x03
 	}
 }
+
+// reserveFromBacking implements the Reserve half of the RxBatcher contract
+// shared by TCP and UDP coalescers. The backing slice grows on demand;
+// already-committed slices reference the old array and remain valid until
+// Flush resets backing.
+func reserveFromBacking(backing *[]byte, sz int) []byte {
+	if len(*backing)+sz > cap(*backing) {
+		newCap := max(cap(*backing)*2, sz)
+		*backing = make([]byte, 0, newCap)
+	}
+	start := len(*backing)
+	*backing = (*backing)[:start+sz]
+	return (*backing)[start : start+sz : start+sz]
+}

overlay/batch/inbound.go

@@ -0,0 +1,443 @@
+package batch
+
+import (
+	"encoding/binary"
+	"errors"
+
+	"github.com/slackhq/nebula/firewall"
+)
+
+// IANA protocol numbers we recognise during the inbound parse. Kept local
+// (rather than reaching for the firewall constants for every one of these)
+// so the byte-comparison hot path doesn't depend on cross-package values.
+const (
+	ipProtoICMP         = 1
+	ipProtoIPv6Fragment = 44
+	ipProtoESP          = 50
+	ipProtoAH           = 51
+	ipProtoICMPv6       = 58
+	ipProtoNoNextHdr    = 59
+
+	icmpv6TypeEchoRequest = 128
+	icmpv6TypeEchoReply   = 129
+)
+
+// Packet parse errors — the canonical sentinel set for IP+L4 parsing.
+// Both inbound and outbound callers share this surface, so any code path
+// that ends up at firewall.PacketKey reports drops with the same errors.
+var (
+	ErrPacketTooShort          = errors.New("packet is too short")
+	ErrUnknownIPVersion        = errors.New("packet is an unknown ip version")
+	ErrIPv4InvalidHeaderLength = errors.New("invalid ipv4 header length")
+	ErrIPv4PacketTooShort      = errors.New("ipv4 packet is too short")
+	ErrIPv6PacketTooShort      = errors.New("ipv6 packet is too short")
+	ErrIPv6CouldNotFindPayload = errors.New("could not find payload in ipv6 packet")
+)
+
+// RxKind discriminates how an inbound plaintext packet should be committed
+// after its firewall.Packet has been built. RxKindPassthrough means the
+// IP shape is valid (firewall could match on it) but the coalescer's
+// strict checks reject it — caller should still write it via the
+// passthrough lane.
+type RxKind uint8
+
+const (
+	RxKindPassthrough RxKind = iota
+	RxKindTCP
+	RxKindUDP
+)
+
+// RxParsed is the unified result of one IP+L4 walk:
+//   - Key: the firewall's conntrack/cache lookup key. The dense form lets
+//     firewall.Drop hit conntrack without ever filling the rich Packet's
+//     netip.Addr fields. On a conntrack miss, Drop hydrates the caller's
+//     Packet from Key.
+//   - tcp/udp: the coalescer hint so commitParsed doesn't re-walk the
+//     headers. Meaningful only when Kind is RxKindTCP / RxKindUDP.
+type RxParsed struct {
+	Kind RxKind
+	Key  firewall.PacketKey
+	tcp  parsedTCP
+	udp  parsedUDP
+}
+
+// ParsePacket walks an IP packet once and fills parsed.Key. When incoming
+// is true and the L4 shape is coalesce-eligible, also fills parsed.tcp /
+// parsed.udp so CommitInbound can dispatch into the coalescer without
+// re-walking the headers.
+//
+// Direction selects the Key orientation:
+//
+//	incoming=true  → wire src → Key.RemoteAddr/Port, wire dst → Key.LocalAddr/Port
+//	incoming=false → wire src → Key.LocalAddr/Port,  wire dst → Key.RemoteAddr/Port
+//
+// ICMP always lands the identifier in Key.RemotePort, regardless of direction.
+//
+// Eligibility rules for the coalescer hint match the coalescer's own
+// parseTCPBase/parseUDP:
+//   - IPv4 strict: IHL == 20, no fragmentation (MF or offset), proto TCP/UDP.
+//   - IPv6 strict: NextHeader is directly TCP or UDP (no extension headers).
+//
+// The hint is only filled for incoming packets, since the outbound path
+// does not feed an inbound coalescer. Outbound callers see Kind stay at
+// RxKindPassthrough and parsed.tcp/udp stay zero.
+func ParsePacket(pkt []byte, incoming bool, parsed *RxParsed) error {
+	parsed.Kind = RxKindPassthrough
+	// Reset Key in full: v4 only writes the low 4 bytes of each address
+	// field, so without this a v6 call followed by a v4 reusing the same
+	// RxParsed would inherit the high 12 bytes — breaking the conntrack
+	// map equality for v4 flows.
+	parsed.Key = firewall.PacketKey{}
+	if len(pkt) < 1 {
+		return ErrPacketTooShort
+	}
+	switch pkt[0] >> 4 {
+	case 4:
+		return parsePacketV4(pkt, incoming, parsed)
+	case 6:
+		return parsePacketV6(pkt, incoming, parsed)
+	}
+	return ErrUnknownIPVersion
+}
+
+// parsePacketV4 fills parsed.Key from an IPv4 packet. Direction selects
+// Local/Remote orientation. When incoming and the shape is strict, also
+// fills the coalescer hint.
+func parsePacketV4(pkt []byte, incoming bool, parsed *RxParsed) error {
+	if len(pkt) < 20 {
+		return ErrIPv4PacketTooShort
+	}
+	ihl := int(pkt[0]&0x0f) << 2
+	if ihl < 20 {
+		return ErrIPv4InvalidHeaderLength
+	}
+	flagsfrags := binary.BigEndian.Uint16(pkt[6:8])
+	parsed.Key.Fragment = (flagsfrags & 0x1FFF) != 0
+	parsed.Key.Protocol = pkt[9]
+	parsed.Key.IsV6 = false
+
+	// minFwPacketLen (4) is the L4-header prefix the firewall needs to pull
+	// ports; ICMP needs two extra bytes for the identifier.
+	minLen := ihl
+	if !parsed.Key.Fragment {
+		if parsed.Key.Protocol == firewall.ProtoICMP {
+			minLen += 4 + 2
+		} else {
+			minLen += 4
+		}
+	}
+	if len(pkt) < minLen {
+		return ErrIPv4InvalidHeaderLength
+	}
+
+	if incoming {
+		copy(parsed.Key.RemoteAddr[:4], pkt[12:16])
+		copy(parsed.Key.LocalAddr[:4], pkt[16:20])
+	} else {
+		copy(parsed.Key.LocalAddr[:4], pkt[12:16])
+		copy(parsed.Key.RemoteAddr[:4], pkt[16:20])
+	}
+
+	switch {
+	case parsed.Key.Fragment:
+		parsed.Key.RemotePort = 0
+		parsed.Key.LocalPort = 0
+	case parsed.Key.Protocol == firewall.ProtoICMP:
+		parsed.Key.RemotePort = binary.BigEndian.Uint16(pkt[ihl+4 : ihl+6])
+		parsed.Key.LocalPort = 0
+	case incoming:
+		parsed.Key.RemotePort = binary.BigEndian.Uint16(pkt[ihl : ihl+2])
+		parsed.Key.LocalPort = binary.BigEndian.Uint16(pkt[ihl+2 : ihl+4])
+	default:
+		parsed.Key.LocalPort = binary.BigEndian.Uint16(pkt[ihl : ihl+2])
+		parsed.Key.RemotePort = binary.BigEndian.Uint16(pkt[ihl+2 : ihl+4])
+	}
+
+	// Coalescer hint is inbound-only: no inbound coalescer fires on outgoing.
+	if !incoming {
+		return nil
+	}
+	// Coalescer-eligible? Strict shape: IHL==20, no MF/offset, TCP or UDP.
+	if ihl != 20 || (flagsfrags&0x3FFF) != 0 {
+		return nil
+	}
+	if parsed.Key.Protocol != ipProtoTCP && parsed.Key.Protocol != ipProtoUDP {
+		return nil
+	}
+	totalLen := int(binary.BigEndian.Uint16(pkt[2:4]))
+	if totalLen > len(pkt) || totalLen < 20 {
+		return nil
+	}
+	pktTrim := pkt[:totalLen]
+
+	switch parsed.Key.Protocol {
+	case ipProtoTCP:
+		fillParsedTCPv4(pktTrim, parsed)
+	case ipProtoUDP:
+		fillParsedUDPv4(pktTrim, parsed)
+	}
+	return nil
+}
+
+// fillParsedTCPv4 fills parsed.tcp from a strict-shape IPv4+TCP packet
+// already validated to have IHL==20 and to be totalLen-trimmed.
+func fillParsedTCPv4(pkt []byte, parsed *RxParsed) {
+	if len(pkt) < 40 { // IPv4(20) + min TCP(20)
+		return
+	}
+	tcpOff := int(pkt[32]>>4) * 4
+	if tcpOff < 20 || tcpOff > 60 {
+		return
+	}
+	if len(pkt) < 20+tcpOff {
+		return
+	}
+	p := &parsed.tcp
+	p.ipHdrLen = 20
+	p.tcpHdrLen = tcpOff
+	p.hdrLen = 20 + tcpOff
+	p.payLen = len(pkt) - p.hdrLen
+	p.seq = binary.BigEndian.Uint32(pkt[24:28])
+	p.flags = pkt[33]
+	p.fk.isV6 = false
+	p.fk.sport = parsed.Key.RemotePort
+	p.fk.dport = parsed.Key.LocalPort
+	copy(p.fk.src[:4], pkt[12:16])
+	copy(p.fk.dst[:4], pkt[16:20])
+	parsed.Kind = RxKindTCP
+}
+
+// fillParsedUDPv4 fills parsed.udp from a strict-shape IPv4+UDP packet.
+func fillParsedUDPv4(pkt []byte, parsed *RxParsed) {
+	if len(pkt) < 28 { // IPv4(20) + UDP(8)
+		return
+	}
+	udpLen := int(binary.BigEndian.Uint16(pkt[24:26]))
+	if udpLen < 8 || udpLen > len(pkt)-20 {
+		return
+	}
+	p := &parsed.udp
+	p.ipHdrLen = 20
+	p.hdrLen = 28
+	p.payLen = udpLen - 8
+	p.fk.isV6 = false
+	p.fk.sport = parsed.Key.RemotePort
+	p.fk.dport = parsed.Key.LocalPort
+	copy(p.fk.src[:4], pkt[12:16])
+	copy(p.fk.dst[:4], pkt[16:20])
+	parsed.Kind = RxKindUDP
+}
+
+// parsePacketV6 fills parsed.Key from an IPv6 packet. Direction selects
+// Local/Remote orientation. The coalescer hint fast path only triggers
+// when NextHeader is directly TCP or UDP — any extension header chain
+// falls into the lenient walk below, and the hint stays unfilled.
+func parsePacketV6(pkt []byte, incoming bool, parsed *RxParsed) error {
+	if len(pkt) < 40 {
+		return ErrIPv6PacketTooShort
+	}
+	parsed.Key.IsV6 = true
+	if incoming {
+		copy(parsed.Key.RemoteAddr[:], pkt[8:24])
+		copy(parsed.Key.LocalAddr[:], pkt[24:40])
+	} else {
+		copy(parsed.Key.LocalAddr[:], pkt[8:24])
+		copy(parsed.Key.RemoteAddr[:], pkt[24:40])
+	}
+
+	if proto := pkt[6]; proto == ipProtoTCP || proto == ipProtoUDP {
+		// Strict v6: ports are at the IP header end. Always fill key; only
+		// fill the coalescer hint if the L4 shape passes.
+		if len(pkt) < 44 {
+			return ErrIPv6PacketTooShort
+		}
+		parsed.Key.Protocol = proto
+		parsed.Key.Fragment = false
+		if incoming {
+			parsed.Key.RemotePort = binary.BigEndian.Uint16(pkt[40:42])
+			parsed.Key.LocalPort = binary.BigEndian.Uint16(pkt[42:44])
+		} else {
+			parsed.Key.LocalPort = binary.BigEndian.Uint16(pkt[40:42])
+			parsed.Key.RemotePort = binary.BigEndian.Uint16(pkt[42:44])
+		}
+
+		// Coalescer hint is inbound-only.
+		if !incoming {
+			return nil
+		}
+		payloadLen := int(binary.BigEndian.Uint16(pkt[4:6]))
+		if 40+payloadLen > len(pkt) {
+			return nil
+		}
+		pktTrim := pkt[:40+payloadLen]
+
+		switch proto {
+		case ipProtoTCP:
+			fillParsedTCPv6(pktTrim, parsed)
+		case ipProtoUDP:
+			fillParsedUDPv6(pktTrim, parsed)
+		}
+		return nil
+	}
+
+	// Slow path: walk extension header chain. Coalescer hint never fires
+	// here, so direction only matters for L4 port orientation.
+	return walkV6Headers(pkt, incoming, parsed)
+}
+
+func fillParsedTCPv6(pkt []byte, parsed *RxParsed) {
+	if len(pkt) < 60 { // IPv6(40) + min TCP(20)
+		return
+	}
+	tcpOff := int(pkt[52]>>4) * 4
+	if tcpOff < 20 || tcpOff > 60 {
+		return
+	}
+	if len(pkt) < 40+tcpOff {
+		return
+	}
+	p := &parsed.tcp
+	p.ipHdrLen = 40
+	p.tcpHdrLen = tcpOff
+	p.hdrLen = 40 + tcpOff
+	p.payLen = len(pkt) - p.hdrLen
+	p.seq = binary.BigEndian.Uint32(pkt[44:48])
+	p.flags = pkt[53]
+	p.fk.isV6 = true
+	p.fk.sport = parsed.Key.RemotePort
+	p.fk.dport = parsed.Key.LocalPort
+	copy(p.fk.src[:], pkt[8:24])
+	copy(p.fk.dst[:], pkt[24:40])
+	parsed.Kind = RxKindTCP
+}
+
+func fillParsedUDPv6(pkt []byte, parsed *RxParsed) {
+	if len(pkt) < 48 { // IPv6(40) + UDP(8)
+		return
+	}
+	udpLen := int(binary.BigEndian.Uint16(pkt[44:46]))
+	if udpLen < 8 || udpLen > len(pkt)-40 {
+		return
+	}
+	p := &parsed.udp
+	p.ipHdrLen = 40
+	p.hdrLen = 48
+	p.payLen = udpLen - 8
+	p.fk.isV6 = true
+	p.fk.sport = parsed.Key.RemotePort
+	p.fk.dport = parsed.Key.LocalPort
+	copy(p.fk.src[:], pkt[8:24])
+	copy(p.fk.dst[:], pkt[24:40])
+	parsed.Kind = RxKindUDP
+}
+
+// walkV6Headers handles every IPv6 case the strict "NextHeader == TCP/UDP"
+// fast path doesn't: ESP, NoNextHeader, ICMPv6, fragment headers (first vs
+// later), AH, generic extension headers. Coalescer eligibility is always
+// RxKindPassthrough on this path (parsed already initialised that way).
+// Direction matters only for the L4 port orientation when the chain
+// terminates at TCP/UDP.
+func walkV6Headers(pkt []byte, incoming bool, parsed *RxParsed) error {
+	dataLen := len(pkt)
+	protoAt := 6
+	offset := 40
+	next := 0
+	for {
+		if protoAt >= dataLen {
+			break
+		}
+		proto := pkt[protoAt]
+		switch proto {
+		case ipProtoESP, ipProtoNoNextHdr:
+			parsed.Key.Protocol = proto
+			parsed.Key.RemotePort = 0
+			parsed.Key.LocalPort = 0
+			parsed.Key.Fragment = false
+			return nil
+
+		case ipProtoICMPv6:
+			if dataLen < offset+6 {
+				return ErrIPv6PacketTooShort
+			}
+			parsed.Key.Protocol = proto
+			parsed.Key.LocalPort = 0
+			switch pkt[offset+1] {
+			case icmpv6TypeEchoRequest, icmpv6TypeEchoReply:
+				parsed.Key.RemotePort = binary.BigEndian.Uint16(pkt[offset+4 : offset+6])
+			default:
+				parsed.Key.RemotePort = 0
+			}
+			parsed.Key.Fragment = false
+			return nil
+
+		case ipProtoTCP, ipProtoUDP:
+			// Reachable when an extension-header chain ends at TCP/UDP. The
+			// strict-eligible fast path above already handled the no-extension
+			// case; here we only fill firewall ports and stay passthrough.
+			if dataLen < offset+4 {
+				return ErrIPv6PacketTooShort
+			}
+			parsed.Key.Protocol = proto
+			if incoming {
+				parsed.Key.RemotePort = binary.BigEndian.Uint16(pkt[offset : offset+2])
+				parsed.Key.LocalPort = binary.BigEndian.Uint16(pkt[offset+2 : offset+4])
+			} else {
+				parsed.Key.LocalPort = binary.BigEndian.Uint16(pkt[offset : offset+2])
+				parsed.Key.RemotePort = binary.BigEndian.Uint16(pkt[offset+2 : offset+4])
+			}
+			parsed.Key.Fragment = false
+			return nil
+
+		case ipProtoIPv6Fragment:
+			if dataLen < offset+8 {
+				return ErrIPv6PacketTooShort
+			}
+			fragmentOffset := binary.BigEndian.Uint16(pkt[offset+2:offset+4]) &^ uint16(0x7)
+			if fragmentOffset != 0 {
+				// Non-first fragment: report the fragment flag and stop.
+				parsed.Key.Protocol = pkt[offset]
+				parsed.Key.Fragment = true
+				parsed.Key.RemotePort = 0
+				parsed.Key.LocalPort = 0
+				return nil
+			}
+			next = 8
+
+		case ipProtoAH:
+			if dataLen <= offset+1 {
+				break
+			}
+			next = int(pkt[offset+1]+2) << 2
+
+		default:
+			if dataLen <= offset+1 {
+				break
+			}
+			next = int(pkt[offset+1]+1) << 3
+		}
+
+		if next <= 0 {
+			next = 8
+		}
+		protoAt = offset
+		offset = offset + next
+	}
+	return ErrIPv6CouldNotFindPayload
+}
+
+// CommitInbound dispatches pkt to the appropriate lane using parsed.Kind,
+// skipping the IP+L4 re-parse that MultiCoalescer.Commit would otherwise
+// do. Borrowed slice contract is identical to MultiCoalescer.Commit.
+func (m *MultiCoalescer) CommitInbound(pkt []byte, parsed *RxParsed) error {
+	switch parsed.Kind {
+	case RxKindTCP:
+		if m.tcp != nil {
+			return m.tcp.commitParsed(pkt, parsed.tcp)
+		}
+	case RxKindUDP:
+		if m.udp != nil {
+			return m.udp.commitParsed(pkt, parsed.udp)
+		}
+	}
+	return m.pt.Commit(pkt)
+}

overlay/batch/inbound_bench_test.go

@@ -0,0 +1,394 @@
+package batch
+
+import (
+	"encoding/binary"
+	"net/netip"
+	"testing"
+
+	"github.com/slackhq/nebula/firewall"
+)
+
+// parseV4InboundBaseline mirrors what outside.go's parseV4(incoming=true)
+// does, so the "split" bench measures the *current* state: firewall-side
+// parse, then m.Commit re-parses inside the coalescer. Two walks per
+// packet. Kept faithful in shape (one read per field, AddrFromSlice for
+// the addrs) so the CPU profile matches the production parseV4.
+func parseV4InboundBaseline(pkt []byte, fp *firewall.Packet) bool {
+	if len(pkt) < 20 {
+		return false
+	}
+	ihl := int(pkt[0]&0x0f) << 2
+	if ihl < 20 {
+		return false
+	}
+	flagsfrags := binary.BigEndian.Uint16(pkt[6:8])
+	fp.Fragment = (flagsfrags & 0x1FFF) != 0
+	fp.Protocol = pkt[9]
+	minLen := ihl
+	if !fp.Fragment {
+		if fp.Protocol == firewall.ProtoICMP {
+			minLen += 4 + 2
+		} else {
+			minLen += 4
+		}
+	}
+	if len(pkt) < minLen {
+		return false
+	}
+	fp.RemoteAddr, _ = netip.AddrFromSlice(pkt[12:16])
+	fp.LocalAddr, _ = netip.AddrFromSlice(pkt[16:20])
+	switch {
+	case fp.Fragment:
+		fp.RemotePort = 0
+		fp.LocalPort = 0
+	case fp.Protocol == firewall.ProtoICMP:
+		fp.RemotePort = binary.BigEndian.Uint16(pkt[ihl+4 : ihl+6])
+		fp.LocalPort = 0
+	default:
+		fp.RemotePort = binary.BigEndian.Uint16(pkt[ihl : ihl+2])
+		fp.LocalPort = binary.BigEndian.Uint16(pkt[ihl+2 : ihl+4])
+	}
+	return true
+}
+
+// parseV6InboundBaseline is the v6 analogue: replicates parseV6's
+// extension-header walk so the split bench captures its true cost.
+func parseV6InboundBaseline(pkt []byte, fp *firewall.Packet) bool {
+	dataLen := len(pkt)
+	if dataLen < 40 {
+		return false
+	}
+	fp.RemoteAddr, _ = netip.AddrFromSlice(pkt[8:24])
+	fp.LocalAddr, _ = netip.AddrFromSlice(pkt[24:40])
+
+	protoAt := 6
+	offset := 40
+	next := 0
+	for {
+		if protoAt >= dataLen {
+			return false
+		}
+		proto := pkt[protoAt]
+		switch proto {
+		case ipProtoESP, ipProtoNoNextHdr:
+			fp.Protocol = proto
+			fp.RemotePort = 0
+			fp.LocalPort = 0
+			fp.Fragment = false
+			return true
+		case ipProtoICMPv6:
+			if dataLen < offset+6 {
+				return false
+			}
+			fp.Protocol = proto
+			fp.LocalPort = 0
+			switch pkt[offset+1] {
+			case icmpv6TypeEchoRequest, icmpv6TypeEchoReply:
+				fp.RemotePort = binary.BigEndian.Uint16(pkt[offset+4 : offset+6])
+			default:
+				fp.RemotePort = 0
+			}
+			fp.Fragment = false
+			return true
+		case ipProtoTCP, ipProtoUDP:
+			if dataLen < offset+4 {
+				return false
+			}
+			fp.Protocol = proto
+			fp.RemotePort = binary.BigEndian.Uint16(pkt[offset : offset+2])
+			fp.LocalPort = binary.BigEndian.Uint16(pkt[offset+2 : offset+4])
+			fp.Fragment = false
+			return true
+		case ipProtoIPv6Fragment:
+			if dataLen < offset+8 {
+				return false
+			}
+			fragmentOffset := binary.BigEndian.Uint16(pkt[offset+2:offset+4]) &^ uint16(0x7)
+			if fragmentOffset != 0 {
+				fp.Protocol = pkt[offset]
+				fp.Fragment = true
+				fp.RemotePort = 0
+				fp.LocalPort = 0
+				return true
+			}
+			next = 8
+		case ipProtoAH:
+			if dataLen <= offset+1 {
+				return false
+			}
+			next = int(pkt[offset+1]+2) << 2
+		default:
+			if dataLen <= offset+1 {
+				return false
+			}
+			next = int(pkt[offset+1]+1) << 3
+		}
+		if next <= 0 {
+			next = 8
+		}
+		protoAt = offset
+		offset = offset + next
+	}
+}
+
+// runRxSplit drives the split path: faithful inbound parse for the firewall
+// side, then m.Commit re-parses to coalesce. v6 controls which baseline
+// parser we run.
+func runRxSplit(b *testing.B, pkts [][]byte, batchSize int, v6 bool) {
+	b.Helper()
+	m := NewMultiCoalescer(nopTunWriter{}, true, true)
+	var fp firewall.Packet
+	b.ReportAllocs()
+	b.SetBytes(int64(len(pkts[0])))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		pkt := pkts[i%len(pkts)]
+		var ok bool
+		if v6 {
+			ok = parseV6InboundBaseline(pkt, &fp)
+		} else {
+			ok = parseV4InboundBaseline(pkt, &fp)
+		}
+		if !ok {
+			b.Fatal("baseline parse failed")
+		}
+		if err := m.Commit(pkt); err != nil {
+			b.Fatal(err)
+		}
+		if (i+1)%batchSize == 0 {
+			if err := m.Flush(); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+	_ = m.Flush()
+}
+
+// runRxUnified drives the unified path: ParseInbound walks once, filling
+// the conntrack key + coalescer hint in parsed; CommitInbound dispatches
+// without re-parsing.
+func runRxUnified(b *testing.B, pkts [][]byte, batchSize int) {
+	b.Helper()
+	m := NewMultiCoalescer(nopTunWriter{}, true, true)
+	var parsed RxParsed
+	b.ReportAllocs()
+	b.SetBytes(int64(len(pkts[0])))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		pkt := pkts[i%len(pkts)]
+		if err := ParsePacket(pkt, true, &parsed); err != nil {
+			b.Fatal(err)
+		}
+		if err := m.CommitInbound(pkt, &parsed); err != nil {
+			b.Fatal(err)
+		}
+		if (i+1)%batchSize == 0 {
+			if err := m.Flush(); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+	_ = m.Flush()
+}
+
+// buildUDPv4Bulk returns N UDP packets on a single 5-tuple suitable for the
+// UDP coalescer's append path.
+func buildUDPv4Bulk(n, payloadLen int) [][]byte {
+	pkts := make([][]byte, n)
+	pay := make([]byte, payloadLen)
+	for i := range n {
+		pkts[i] = buildUDPv4(1000, 53, pay)
+	}
+	return pkts
+}
+
+func buildTCPv6Bulk(n, payloadLen int) [][]byte {
+	pkts := make([][]byte, n)
+	pay := make([]byte, payloadLen)
+	seq := uint32(1000)
+	for i := range n {
+		pkts[i] = buildTCPv6(0, seq, tcpAck, pay)
+		seq += uint32(payloadLen)
+	}
+	return pkts
+}
+
+func buildICMPv4Bulk(n int) [][]byte {
+	pkts := make([][]byte, n)
+	for i := range pkts {
+		pkts[i] = buildICMPv4()
+	}
+	return pkts
+}
+
+// === TCPv4 ===
+
+func BenchmarkRxSplitTCPv4(b *testing.B) {
+	pkts := buildTCPv4BulkFlow(tcpCoalesceMaxSegs, 1200)
+	runRxSplit(b, pkts, tcpCoalesceMaxSegs, false)
+}
+
+func BenchmarkRxUnifiedTCPv4(b *testing.B) {
+	pkts := buildTCPv4BulkFlow(tcpCoalesceMaxSegs, 1200)
+	runRxUnified(b, pkts, tcpCoalesceMaxSegs)
+}
+
+// === TCPv4 interleaved (4 flows) ===
+
+func BenchmarkRxSplitTCPv4Interleaved4(b *testing.B) {
+	pkts := buildTCPv4Interleaved(4, tcpCoalesceMaxSegs, 1200)
+	runRxSplit(b, pkts, len(pkts), false)
+}
+
+func BenchmarkRxUnifiedTCPv4Interleaved4(b *testing.B) {
+	pkts := buildTCPv4Interleaved(4, tcpCoalesceMaxSegs, 1200)
+	runRxUnified(b, pkts, len(pkts))
+}
+
+// === UDPv4 ===
+
+func BenchmarkRxSplitUDPv4(b *testing.B) {
+	pkts := buildUDPv4Bulk(udpCoalesceMaxSegs, 1200)
+	runRxSplit(b, pkts, udpCoalesceMaxSegs, false)
+}
+
+func BenchmarkRxUnifiedUDPv4(b *testing.B) {
+	pkts := buildUDPv4Bulk(udpCoalesceMaxSegs, 1200)
+	runRxUnified(b, pkts, udpCoalesceMaxSegs)
+}
+
+// === TCPv6 ===
+
+func BenchmarkRxSplitTCPv6(b *testing.B) {
+	pkts := buildTCPv6Bulk(tcpCoalesceMaxSegs, 1200)
+	runRxSplit(b, pkts, tcpCoalesceMaxSegs, true)
+}
+
+func BenchmarkRxUnifiedTCPv6(b *testing.B) {
+	pkts := buildTCPv6Bulk(tcpCoalesceMaxSegs, 1200)
+	runRxUnified(b, pkts, tcpCoalesceMaxSegs)
+}
+
+// === ICMPv4 (passthrough) — measures the unified parser on the coalescer-
+// rejected path, where both lenient and unified must still fill fp. ===
+
+func BenchmarkRxSplitICMPv4(b *testing.B) {
+	pkts := buildICMPv4Bulk(64)
+	runRxSplit(b, pkts, 64, false)
+}
+
+func BenchmarkRxUnifiedICMPv4(b *testing.B) {
+	pkts := buildICMPv4Bulk(64)
+	runRxUnified(b, pkts, 64)
+}
+
+// === Firewall fast-path (conntrack-hit) — exercises the savings from the
+// dense PacketKey: smaller hash key for the per-routine ConntrackCache,
+// and skipping the AddrFrom4 calls that the old path needed to fill the
+// netip.Addr-rich firewall.Packet up-front. ===
+//
+// The "split" baseline simulates the legacy path: parseV4InboundBaseline
+// fills a netip.Addr-rich Packet, then we probe a localCache keyed on
+// Packet. The "unified" path: ParseInbound fills only the dense PacketKey,
+// and we probe a localCache keyed on PacketKey. Both paths follow with
+// the coalescer Commit so the bench captures end-to-end RX-side cost.
+
+// runRxSplitWithCache mirrors runRxSplit but runs the legacy-style
+// firewall fast path (localCache keyed on firewall.Packet) on every
+// packet so we can compare against the unified path.
+func runRxSplitWithCache(b *testing.B, pkts [][]byte, batchSize int) {
+	b.Helper()
+	m := NewMultiCoalescer(nopTunWriter{}, true, true)
+	var fp firewall.Packet
+
+	// Pre-warm a per-packet cache keyed on the netip.Addr-rich Packet form.
+	cache := make(map[firewall.Packet]struct{}, len(pkts))
+	for _, pkt := range pkts {
+		var seedFp firewall.Packet
+		if !parseV4InboundBaseline(pkt, &seedFp) {
+			b.Fatal("seed parse failed")
+		}
+		cache[seedFp] = struct{}{}
+	}
+
+	b.ReportAllocs()
+	b.SetBytes(int64(len(pkts[0])))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		pkt := pkts[i%len(pkts)]
+		if !parseV4InboundBaseline(pkt, &fp) {
+			b.Fatal("baseline parse failed")
+		}
+		if _, ok := cache[fp]; !ok {
+			b.Fatal("cache miss")
+		}
+		if err := m.Commit(pkt); err != nil {
+			b.Fatal(err)
+		}
+		if (i+1)%batchSize == 0 {
+			if err := m.Flush(); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+	_ = m.Flush()
+}
+
+// runRxUnifiedWithCache: unified path with a PacketKey-keyed localCache.
+// Each iteration: ParseInbound → conntrack-cache hit → CommitInbound.
+func runRxUnifiedWithCache(b *testing.B, pkts [][]byte, batchSize int) {
+	b.Helper()
+	m := NewMultiCoalescer(nopTunWriter{}, true, true)
+	var parsed RxParsed
+
+	cache := make(firewall.ConntrackCache, len(pkts))
+	for _, pkt := range pkts {
+		var seed RxParsed
+		if err := ParsePacket(pkt, true, &seed); err != nil {
+			b.Fatal(err)
+		}
+		cache[seed.Key] = struct{}{}
+	}
+
+	b.ReportAllocs()
+	b.SetBytes(int64(len(pkts[0])))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		pkt := pkts[i%len(pkts)]
+		if err := ParsePacket(pkt, true, &parsed); err != nil {
+			b.Fatal(err)
+		}
+		if _, ok := cache[parsed.Key]; !ok {
+			b.Fatal("cache miss")
+		}
+		if err := m.CommitInbound(pkt, &parsed); err != nil {
+			b.Fatal(err)
+		}
+		if (i+1)%batchSize == 0 {
+			if err := m.Flush(); err != nil {
+				b.Fatal(err)
+			}
+		}
+	}
+	_ = m.Flush()
+}
+
+func BenchmarkRxSplitTCPv4WithCache(b *testing.B) {
+	pkts := buildTCPv4BulkFlow(tcpCoalesceMaxSegs, 1200)
+	runRxSplitWithCache(b, pkts, tcpCoalesceMaxSegs)
+}
+
+func BenchmarkRxUnifiedTCPv4WithCache(b *testing.B) {
+	pkts := buildTCPv4BulkFlow(tcpCoalesceMaxSegs, 1200)
+	runRxUnifiedWithCache(b, pkts, tcpCoalesceMaxSegs)
+}
+
+func BenchmarkRxSplitInterleaved4WithCache(b *testing.B) {
+	pkts := buildTCPv4Interleaved(4, tcpCoalesceMaxSegs, 1200)
+	runRxSplitWithCache(b, pkts, len(pkts))
+}
+
+func BenchmarkRxUnifiedInterleaved4WithCache(b *testing.B) {
+	pkts := buildTCPv4Interleaved(4, tcpCoalesceMaxSegs, 1200)
+	runRxUnifiedWithCache(b, pkts, len(pkts))
+}

overlay/batch/inbound_test.go

@@ -0,0 +1,174 @@
+package batch
+
+import (
+	"net/netip"
+	"testing"
+
+	"github.com/slackhq/nebula/firewall"
+)
+
+// TestParseInboundParity asserts that ParseInbound + Key.Hydrate produces
+// the same firewall.Packet that the lenient baseline parsers (which
+// mirror outside.go's parseV4/parseV6 with incoming=true) produce for
+// every shape we care about. Catches drift between the unified
+// parse-then-hydrate flow and the production newPacket behavior so
+// swapping one for the other is observably safe.
+func TestParseInboundParity(t *testing.T) {
+	cases := []struct {
+		name string
+		pkt  []byte
+		v6   bool
+	}{
+		{"tcp_v4", buildTCPv4Ports(1234, 443, 1000, tcpAck, []byte("payload")), false},
+		{"tcp_v4_psh", buildTCPv4Ports(1234, 443, 2000, tcpAckPsh, make([]byte, 1200)), false},
+		{"udp_v4", buildUDPv4(40000, 53, []byte("dnsquery")), false},
+		{"icmp_v4", buildICMPv4(), false},
+		{"tcp_v6", buildTCPv6(0, 5000, tcpAck, make([]byte, 800)), true},
+		{"udp_v6", buildUDPv6(40001, 53, []byte("v6dns")), true},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			var fpUnified, fpBaseline firewall.Packet
+			var parsed RxParsed
+
+			if err := ParsePacket(tc.pkt, true, &parsed); err != nil {
+				t.Fatalf("ParsePacket: %v", err)
+			}
+			parsed.Key.Hydrate(&fpUnified)
+			var ok bool
+			if tc.v6 {
+				ok = parseV6InboundBaseline(tc.pkt, &fpBaseline)
+			} else {
+				ok = parseV4InboundBaseline(tc.pkt, &fpBaseline)
+			}
+			if !ok {
+				t.Fatalf("baseline parse failed")
+			}
+
+			if fpUnified != fpBaseline {
+				t.Errorf("firewall.Packet mismatch:\n  unified:  %+v\n  baseline: %+v", fpUnified, fpBaseline)
+			}
+		})
+	}
+}
+
+// TestParseInboundFlowKey checks that the coalescer hint the unified parser
+// produces matches what parseTCPBase/parseUDP would produce on the same
+// packet — same flowKey, ipHdrLen, payLen, etc. The hint is only valid
+// when Kind is RxKindTCP/RxKindUDP.
+func TestParseInboundFlowKey(t *testing.T) {
+	t.Run("tcp_v4", func(t *testing.T) {
+		pkt := buildTCPv4Ports(1234, 443, 5000, tcpAck, make([]byte, 800))
+		var parsed RxParsed
+		if err := ParsePacket(pkt, true, &parsed); err != nil {
+			t.Fatal(err)
+		}
+		if parsed.Kind != RxKindTCP {
+			t.Fatalf("kind=%v want TCP", parsed.Kind)
+		}
+		ref, ok := parseTCPBase(pkt)
+		if !ok {
+			t.Fatal("parseTCPBase failed")
+		}
+		if parsed.tcp != ref {
+			t.Errorf("parsedTCP mismatch:\n  unified: %+v\n  ref:     %+v", parsed.tcp, ref)
+		}
+	})
+
+	t.Run("udp_v4", func(t *testing.T) {
+		pkt := buildUDPv4(40000, 53, []byte("dnsquery"))
+		var parsed RxParsed
+		if err := ParsePacket(pkt, true, &parsed); err != nil {
+			t.Fatal(err)
+		}
+		if parsed.Kind != RxKindUDP {
+			t.Fatalf("kind=%v want UDP", parsed.Kind)
+		}
+		ref, ok := parseUDP(pkt)
+		if !ok {
+			t.Fatal("parseUDP failed")
+		}
+		if parsed.udp != ref {
+			t.Errorf("parsedUDP mismatch:\n  unified: %+v\n  ref:     %+v", parsed.udp, ref)
+		}
+	})
+
+	t.Run("tcp_v6", func(t *testing.T) {
+		pkt := buildTCPv6(0, 9000, tcpAck, make([]byte, 800))
+		var parsed RxParsed
+		if err := ParsePacket(pkt, true, &parsed); err != nil {
+			t.Fatal(err)
+		}
+		if parsed.Kind != RxKindTCP {
+			t.Fatalf("kind=%v want TCP", parsed.Kind)
+		}
+		ref, ok := parseTCPBase(pkt)
+		if !ok {
+			t.Fatal("parseTCPBase failed")
+		}
+		if parsed.tcp != ref {
+			t.Errorf("parsedTCP mismatch:\n  unified: %+v\n  ref:     %+v", parsed.tcp, ref)
+		}
+	})
+}
+
+// TestParseInboundICMPPassthrough confirms ICMP packets populate the
+// conntrack key (including the ICMP identifier in RemotePort) but stay
+// RxKindPassthrough so the batcher writes them verbatim. After Hydrate
+// the firewall.Packet form should match what the legacy parseV4 produced.
+func TestParseInboundICMPPassthrough(t *testing.T) {
+	pkt := buildICMPv4()
+	// Stamp a non-zero identifier into the ICMP header so we can check
+	// RemotePort gets it.
+	pkt[20] = 8 // type=echo
+	pkt[24] = 0xab
+	pkt[25] = 0xcd
+
+	var parsed RxParsed
+	if err := ParsePacket(pkt, true, &parsed); err != nil {
+		t.Fatal(err)
+	}
+	if parsed.Kind != RxKindPassthrough {
+		t.Errorf("kind=%v want Passthrough", parsed.Kind)
+	}
+	var fp firewall.Packet
+	parsed.Key.Hydrate(&fp)
+	if fp.Protocol != firewall.ProtoICMP {
+		t.Errorf("Protocol=%d want %d", fp.Protocol, firewall.ProtoICMP)
+	}
+	if fp.RemotePort != 0xabcd {
+		t.Errorf("RemotePort=0x%x want 0xabcd", fp.RemotePort)
+	}
+	if fp.LocalPort != 0 {
+		t.Errorf("LocalPort=%d want 0", fp.LocalPort)
+	}
+	wantRemote := netip.MustParseAddr("10.0.0.1")
+	wantLocal := netip.MustParseAddr("10.0.0.2")
+	if fp.RemoteAddr != wantRemote || fp.LocalAddr != wantLocal {
+		t.Errorf("addrs: remote=%v local=%v want %v/%v", fp.RemoteAddr, fp.LocalAddr, wantRemote, wantLocal)
+	}
+}
+
+// TestParseInboundV4Fragment confirms a fragmented v4 packet fills the
+// conntrack key with Fragment=true and falls into Passthrough on the
+// coalescer side.
+func TestParseInboundV4Fragment(t *testing.T) {
+	// Build a TCP packet then twiddle the IP flags to make it look like a
+	// non-first fragment (offset != 0).
+	pkt := buildTCPv4Ports(1234, 443, 1000, tcpAck, []byte("payload"))
+	// Set a non-zero fragment offset (bytes 6-7, low 13 bits).
+	pkt[6] = 0x00
+	pkt[7] = 0x10 // offset = 16 (in 8-byte units)
+
+	var parsed RxParsed
+	if err := ParsePacket(pkt, true, &parsed); err != nil {
+		t.Fatal(err)
+	}
+	if !parsed.Key.Fragment {
+		t.Error("Fragment=false, want true")
+	}
+	if parsed.Kind != RxKindPassthrough {
+		t.Errorf("kind=%v want Passthrough", parsed.Kind)
+	}
+}

overlay/batch/multi_coalesce.go

@@ -2,10 +2,7 @@ package batch
 
 import (
 	"errors"
-	"log/slog"
-
-	"github.com/slackhq/nebula/overlay/tio"
-	"github.com/slackhq/nebula/util"
+	"io"
 )
 
 // MultiCoalescer fans plaintext packets out to lane-specific batchers based
@@ -30,38 +27,37 @@ type MultiCoalescer struct {
 	udp *UDPCoalescer
 	pt  *Passthrough
 
-	// arena is shared across every lane (constructor hands the same
-	// *Arena to TCP, UDP, and Passthrough), so there's exactly one
-	// backing slab per MultiCoalescer instance. Each lane's Flush calls
-	// Reset; the resets are idempotent because Multi.Flush drains lanes
-	// sequentially and never Reserves in between, so a later lane's
-	// slots stay readable across an earlier lane's Reset (the underlying
-	// bytes are still alive — Reset only re-slices len to 0).
-	arena *util.Arena
+	// arena shared across all lanes so a single Reserve grows one backing
+	// slice; lane Commit calls borrow into this same arena.
+	backing []byte
 }
 
 // NewMultiCoalescer builds a multi-lane batcher. tcpEnabled lets the caller
 // opt out of TCP coalescing (e.g. when the queue can't do TSO); udpEnabled
 // likewise gates UDP coalescing (only enable when USO was negotiated).
 // Either lane disabled redirects its traffic into the passthrough lane.
-// arena is the single backing slab shared across every lane; the caller
-// pre-sizes it via NewArena so the hot path never allocates.
-func NewMultiCoalescer(w tio.Queue, l *slog.Logger, arena *util.Arena, tcpEnabled, udpEnabled bool) *MultiCoalescer {
+func NewMultiCoalescer(w io.Writer, tcpEnabled, udpEnabled bool) *MultiCoalescer {
 	m := &MultiCoalescer{
-		pt:    NewPassthrough(w, initialSlots, arena),
-		arena: arena,
+		pt:      NewPassthrough(w),
+		backing: make([]byte, 0, initialSlots*65535),
 	}
 	if tcpEnabled {
-		m.tcp = NewTCPCoalescer(w, l, arena)
+		m.tcp = NewTCPCoalescer(w)
 	}
 	if udpEnabled {
-		m.udp = NewUDPCoalescer(w, arena)
+		m.udp = NewUDPCoalescer(w)
 	}
 	return m
 }
 
 func (m *MultiCoalescer) Reserve(sz int) []byte {
-	return m.arena.Reserve(sz)
+	if len(m.backing)+sz > cap(m.backing) {
+		newCap := max(cap(m.backing)*2, sz)
+		m.backing = make([]byte, 0, newCap)
+	}
+	start := len(m.backing)
+	m.backing = m.backing[:start+sz]
+	return m.backing[start : start+sz : start+sz]
 }
 
 // Commit dispatches pkt to the appropriate lane based on IP version + L4
@@ -132,5 +128,6 @@ func (m *MultiCoalescer) Flush() error {
 	if err := m.pt.Flush(); err != nil {
 		errs = append(errs, err)
 	}
+	m.backing = m.backing[:0]
 	return errors.Join(errs...)
 }

overlay/batch/multi_coalesce_test.go

@@ -2,9 +2,6 @@ package batch
 
 import (
 	"testing"
-
-	"github.com/slackhq/nebula/test"
-	"github.com/slackhq/nebula/util"
 )
 
 // TestMultiCoalescerRoutesByProto confirms TCP/UDP/other land in the right
@@ -12,7 +9,7 @@ import (
 // else (ICMP here) falls through to plain Write.
 func TestMultiCoalescerRoutesByProto(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	m := NewMultiCoalescer(w, test.NewLogger(), util.NewArena(0), true, true)
+	m := NewMultiCoalescer(w, true, true)
 
 	tcpPay := make([]byte, 1200)
 	udpPay := make([]byte, 1200)
@@ -54,7 +51,7 @@ func TestMultiCoalescerRoutesByProto(t *testing.T) {
 // the kernel via the passthrough lane rather than being lost.
 func TestMultiCoalescerDisabledUDPFallsThrough(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	m := NewMultiCoalescer(w, test.NewLogger(), util.NewArena(0), true, false) // TSO on, USO off
+	m := NewMultiCoalescer(w, true, false) // TSO on, USO off
 
 	if err := m.Commit(buildUDPv4(1000, 53, make([]byte, 800))); err != nil {
 		t.Fatal(err)
@@ -76,7 +73,7 @@ func TestMultiCoalescerDisabledUDPFallsThrough(t *testing.T) {
 // TestMultiCoalescerDisabledTCPFallsThrough mirrors the TSO=off case.
 func TestMultiCoalescerDisabledTCPFallsThrough(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	m := NewMultiCoalescer(w, test.NewLogger(), util.NewArena(0), false, true) // TSO off, USO on
+	m := NewMultiCoalescer(w, false, true) // TSO off, USO on
 
 	pay := make([]byte, 1200)
 	if err := m.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {

overlay/batch/passthrough.go

@@ -3,28 +3,36 @@ package batch
 import (
 	"io"
 
-	"github.com/slackhq/nebula/util"
+	"github.com/slackhq/nebula/udp"
 )
 
 // Passthrough is a RxBatcher that doesn't batch anything, it just accumulates and then sends packets.
 type Passthrough struct {
-	out   io.Writer
-	slots [][]byte
-	// arena is injected; see TCPCoalescer.arena for the contract.
-	arena  *util.Arena
-	cursor int
+	out     io.Writer
+	slots   [][]byte
+	backing []byte
+	cursor  int
 }
 
-func NewPassthrough(w io.Writer, slots int, arena *util.Arena) *Passthrough {
+func NewPassthrough(w io.Writer) *Passthrough {
+	const baseNumSlots = 128
 	return &Passthrough{
-		out:   w,
-		slots: make([][]byte, 0, slots),
-		arena: arena,
+		out:     w,
+		slots:   make([][]byte, 0, baseNumSlots),
+		backing: make([]byte, 0, baseNumSlots*udp.MTU),
 	}
 }
 
 func (p *Passthrough) Reserve(sz int) []byte {
-	return p.arena.Reserve(sz)
+	if len(p.backing)+sz > cap(p.backing) {
+		// Grow: allocate a fresh backing. Already-committed slices still
+		// reference the old array and remain valid until Flush drops them.
+		newCap := max(cap(p.backing)*2, sz)
+		p.backing = make([]byte, 0, newCap)
+	}
+	start := len(p.backing)
+	p.backing = p.backing[:start+sz]
+	return p.backing[start : start+sz : start+sz] //return zero length, sz-cap slice
 }
 
 func (p *Passthrough) Commit(pkt []byte) error {
@@ -32,6 +40,13 @@ func (p *Passthrough) Commit(pkt []byte) error {
 	return nil
 }
 
+// CommitInbound ignores the hint — Passthrough never coalesces, so there's
+// no IP/L4 re-parse to skip. Present so Passthrough satisfies the RxBatcher
+// interface alongside MultiCoalescer.
+func (p *Passthrough) CommitInbound(pkt []byte, _ *RxParsed) error {
+	return p.Commit(pkt)
+}
+
 func (p *Passthrough) Flush() error {
 	var firstErr error
 	for _, s := range p.slots {
@@ -42,6 +57,6 @@ func (p *Passthrough) Flush() error {
 	}
 	clear(p.slots)
 	p.slots = p.slots[:0]
-	p.arena.Reset()
+	p.backing = p.backing[:0]
 	return firstErr
 }

overlay/batch/rx_batch.go

@@ -1,12 +0,0 @@
-package batch
-
-type RxBatcher interface {
-	// Reserve creates a pkt to borrow
-	Reserve(sz int) []byte
-	// Commit borrows pkt. The caller must keep pkt valid until the next Flush
-	Commit(pkt []byte) error
-	// Flush emits every queued packet in arrival order. Returns the
-	// first error observed; keeps draining so one bad packet doesn't hold up
-	// the rest. After Flush returns, borrowed payload slices may be recycled.
-	Flush() error
-}

overlay/batch/tcp_coalesce.go

@@ -2,7 +2,6 @@ package batch
 
 import (
 	"bytes"
-	"context"
 	"encoding/binary"
 	"io"
 	"log/slog"
@@ -10,8 +9,6 @@ import (
 	"slices"
 
 	"github.com/slackhq/nebula/overlay/tio"
-	"github.com/slackhq/nebula/util"
-	"github.com/slackhq/nebula/wire"
 )
 
 // ipProtoTCP is the IANA protocol number for TCP. Hardcoded instead of
@@ -86,24 +83,18 @@ type TCPCoalescer struct {
 	lastSlot *coalesceSlot
 	pool     []*coalesceSlot // free list for reuse
 
-	// arena is injected; the coalescer borrows slices from it via Reserve
-	// and tells it to release them via Reset on Flush. When wrapped in
-	// MultiCoalescer the same *Arena is shared with the other lanes so
-	// there's exactly one backing slab per Multi instance.
-	arena *util.Arena
-	l     *slog.Logger
+	backing []byte
 }
 
-func NewTCPCoalescer(w tio.Queue, l *slog.Logger, arena *util.Arena) *TCPCoalescer {
+func NewTCPCoalescer(w io.Writer) *TCPCoalescer {
 	c := &TCPCoalescer{
 		plainW:    w,
 		slots:     make([]*coalesceSlot, 0, initialSlots),
 		openSlots: make(map[flowKey]*coalesceSlot, initialSlots),
 		pool:      make([]*coalesceSlot, 0, initialSlots),
-		arena:     arena,
-		l:         l,
+		backing:   make([]byte, 0, initialSlots*65535),
 	}
-	if gw, ok := tio.SupportsGSO(w, wire.GSOProtoTCP); ok {
+	if gw, ok := tio.SupportsGSO(w, tio.GSOProtoTCP); ok {
 		c.gsoW = gw
 	}
 	return c
@@ -180,7 +171,7 @@ func (p parsedTCP) coalesceable() bool {
 }
 
 func (c *TCPCoalescer) Reserve(sz int) []byte {
-	return c.arena.Reserve(sz)
+	return reserveFromBacking(&c.backing, sz)
 }
 
 // Commit borrows pkt. The caller must keep pkt valid until the next Flush,
@@ -280,7 +271,7 @@ func (c *TCPCoalescer) Flush() error {
 	clear(c.openSlots)
 	c.lastSlot = nil
 
-	c.arena.Reset()
+	c.backing = c.backing[:0]
 	return first
 }
 
@@ -421,7 +412,7 @@ func (c *TCPCoalescer) flushSlot(s *coalesceSlot) error {
 	tcsum := s.ipHdrLen + 16
 	binary.BigEndian.PutUint16(hdr[tcsum:tcsum+2], foldOnceNoInvert(psum))
 
-	return c.gsoW.WriteGSO(hdr[:s.ipHdrLen], hdr[s.ipHdrLen:], s.payIovs, wire.GSOProtoTCP)
+	return c.gsoW.WriteGSO(hdr[:s.ipHdrLen], hdr[s.ipHdrLen:], s.payIovs, tio.GSOProtoTCP)
 }
 
 // headersMatch compares two IP+TCP header prefixes for byte-for-byte
@@ -497,25 +488,22 @@ func (c *TCPCoalescer) reorderForFlush() {
 				// the operator can quantify how often it happens; the data
 				// itself still emits in seq order, kernel TCP handles the
 				// gap via its OOO queue.
-				if c.l.Enabled(context.Background(), slog.LevelDebug) {
-					if prev.nextSeq != slotSeedSeq(s) {
-						logged = true
-						gap := int64(slotSeedSeq(s)) - int64(prev.nextSeq)
-						c.l.Debug("tcp coalesce: cross-slot seq gap",
-							"src", flowKeyAddr(s.fk, false),
-							"dst", flowKeyAddr(s.fk, true),
-							"sport", s.fk.sport,
-							"dport", s.fk.dport,
-							"prev_seed_seq", slotSeedSeq(prev),
-							"prev_next_seq", prev.nextSeq,
-							"this_seed_seq", slotSeedSeq(s),
-							"gap_bytes", gap,
-							"prev_seg_count", prev.numSeg,
-							"prev_total_pay", prev.totalPay,
-						)
-					}
+				if prev.nextSeq != slotSeedSeq(s) {
+					logged = true
+					gap := int64(slotSeedSeq(s)) - int64(prev.nextSeq)
+					slog.Default().Warn("tcp coalesce: cross-slot seq gap",
+						"src", flowKeyAddr(s.fk, false),
+						"dst", flowKeyAddr(s.fk, true),
+						"sport", s.fk.sport,
+						"dport", s.fk.dport,
+						"prev_seed_seq", slotSeedSeq(prev),
+						"prev_next_seq", prev.nextSeq,
+						"this_seed_seq", slotSeedSeq(s),
+						"gap_bytes", gap,
+						"prev_seg_count", prev.numSeg,
+						"prev_total_pay", prev.totalPay,
+					)
 				}
-
 				if canMergeSlots(prev, s) {
 					mergeSlots(prev, s)
 					c.release(s)
@@ -526,7 +514,7 @@ func (c *TCPCoalescer) reorderForFlush() {
 		out = append(out, s)
 	}
 	if logged {
-		c.l.Warn("==== end of batch ====")
+		slog.Default().Warn("==== end of batch ====")
 	}
 	c.slots = out
 }

overlay/batch/tcp_coalesce_bench_test.go

@@ -6,19 +6,14 @@ import (
 	"testing"
 
 	"github.com/slackhq/nebula/overlay/tio"
-	"github.com/slackhq/nebula/test"
-	"github.com/slackhq/nebula/util"
-	"github.com/slackhq/nebula/wire"
 )
 
 // nopTunWriter is a zero-alloc tio.GSOWriter for benchmarks. Discards
 // everything but satisfies the interface the coalescer detects.
 type nopTunWriter struct{}
 
-func (nopTunWriter) Write(p []byte) (int, error)                    { return len(p), nil }
-func (nopTunWriter) Read(_ []wire.TunPacket, _ []byte) (int, error) { return 0, nil }
-func (nopTunWriter) Close() error                                   { return nil }
-func (nopTunWriter) WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte, _ wire.GSOProto) error {
+func (nopTunWriter) Write(p []byte) (int, error) { return len(p), nil }
+func (nopTunWriter) WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte, _ tio.GSOProto) error {
 	return nil
 }
 func (nopTunWriter) Capabilities() tio.Capabilities {
@@ -75,7 +70,7 @@ func buildICMPv4() []byte {
 // between batches, and reports per-packet cost.
 func runCommitBench(b *testing.B, pkts [][]byte, batchSize int) {
 	b.Helper()
-	c := NewTCPCoalescer(nopTunWriter{}, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(nopTunWriter{})
 	b.ReportAllocs()
 	b.SetBytes(int64(len(pkts[0])))
 	b.ResetTimer()
@@ -144,7 +139,7 @@ func BenchmarkCommitNonCoalesceableTCP(b *testing.B) {
 // is the bench that shows the savings of skipping the lane's re-parse.
 func runMultiCommitBench(b *testing.B, pkts [][]byte, batchSize int) {
 	b.Helper()
-	m := NewMultiCoalescer(nopTunWriter{}, test.NewLogger(), util.NewArena(0), true, true)
+	m := NewMultiCoalescer(nopTunWriter{}, true, true)
 	b.ReportAllocs()
 	b.SetBytes(int64(len(pkts[0])))
 	b.ResetTimer()

overlay/batch/tcp_coalesce_test.go

@@ -5,9 +5,6 @@ import (
 	"testing"
 
 	"github.com/slackhq/nebula/overlay/tio"
-	"github.com/slackhq/nebula/test"
-	"github.com/slackhq/nebula/util"
-	"github.com/slackhq/nebula/wire"
 )
 
 // fakeTunWriter records plain Writes and WriteGSO calls without touching a
@@ -55,12 +52,7 @@ func (w *fakeTunWriter) Write(p []byte) (int, error) {
 	return len(p), nil
 }
 
-// Read and Close exist solely to satisfy tio.Queue; coalescer tests never
-// invoke them.
-func (w *fakeTunWriter) Read(_ []wire.TunPacket, _ []byte) (int, error) { return 0, nil }
-func (w *fakeTunWriter) Close() error                                   { return nil }
-
-func (w *fakeTunWriter) WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte, _ wire.GSOProto) error {
+func (w *fakeTunWriter) WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte, _ tio.GSOProto) error {
 	hcopy := make([]byte, len(hdr)+len(transportHdr))
 	copy(hcopy, hdr)
 	copy(hcopy[len(hdr):], transportHdr)
@@ -135,7 +127,7 @@ const (
 
 func TestCoalescerPassthroughWhenGSOUnavailable(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: false}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pkt := buildTCPv4(1000, tcpAck, []byte("hello"))
 	if err := c.Commit(pkt); err != nil {
 		t.Fatal(err)
@@ -154,7 +146,7 @@ func TestCoalescerPassthroughWhenGSOUnavailable(t *testing.T) {
 
 func TestCoalescerNonTCPPassthrough(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pkt := make([]byte, 28)
 	pkt[0] = 0x45
 	binary.BigEndian.PutUint16(pkt[2:4], 28)
@@ -174,7 +166,7 @@ func TestCoalescerNonTCPPassthrough(t *testing.T) {
 
 func TestCoalescerSeedThenFlushAlone(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pkt := buildTCPv4(1000, tcpAck, make([]byte, 1000))
 	if err := c.Commit(pkt); err != nil {
 		t.Fatal(err)
@@ -201,7 +193,7 @@ func TestCoalescerSeedThenFlushAlone(t *testing.T) {
 
 func TestCoalescerCoalescesAdjacentACKs(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 	if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
 		t.Fatal(err)
@@ -241,7 +233,7 @@ func TestCoalescerCoalescesAdjacentACKs(t *testing.T) {
 
 func TestCoalescerRejectsSeqGap(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 	if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
 		t.Fatal(err)
@@ -260,7 +252,7 @@ func TestCoalescerRejectsSeqGap(t *testing.T) {
 
 func TestCoalescerRejectsFlagMismatch(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 	if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
 		t.Fatal(err)
@@ -281,7 +273,7 @@ func TestCoalescerRejectsFlagMismatch(t *testing.T) {
 
 func TestCoalescerRejectsFIN(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	fin := buildTCPv4(1000, tcpAck|tcpFin, []byte("x"))
 	if err := c.Commit(fin); err != nil {
 		t.Fatal(err)
@@ -297,7 +289,7 @@ func TestCoalescerRejectsFIN(t *testing.T) {
 
 func TestCoalescerShortLastSegmentClosesChain(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	full := make([]byte, 1200)
 	half := make([]byte, 500)
 	if err := c.Commit(buildTCPv4(1000, tcpAck, full)); err != nil {
@@ -332,7 +324,7 @@ func TestCoalescerShortLastSegmentClosesChain(t *testing.T) {
 
 func TestCoalescerPSHFinalizesChain(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 	if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
 		t.Fatal(err)
@@ -362,7 +354,7 @@ func TestCoalescerPSHFinalizesChain(t *testing.T) {
 // coalescer drops it the sender's push signal never reaches the receiver.
 func TestCoalescerPropagatesPSHFromAppended(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 	// Seed has no PSH; second segment carries PSH and seals the chain.
 	if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
@@ -390,7 +382,7 @@ func TestCoalescerPropagatesPSHFromAppended(t *testing.T) {
 
 func TestCoalescerRejectsDifferentFlow(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 	p1 := buildTCPv4(1000, tcpAck, pay)
 	p2 := buildTCPv4(2200, tcpAck, pay)
@@ -412,7 +404,7 @@ func TestCoalescerRejectsDifferentFlow(t *testing.T) {
 
 func TestCoalescerRejectsIPOptions(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 500)
 	pkt := buildTCPv4(1000, tcpAck, pay)
 	// Bump IHL to 6 to simulate 4 bytes of IP options. Don't actually add
@@ -432,7 +424,7 @@ func TestCoalescerRejectsIPOptions(t *testing.T) {
 
 func TestCoalescerCapBySegments(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 512)
 	seq := uint32(1000)
 	for i := 0; i < tcpCoalesceMaxSegs+5; i++ {
@@ -456,7 +448,7 @@ func TestCoalescerCapBySegments(t *testing.T) {
 // flows coalesce independently in a single Flush.
 func TestCoalescerMultipleFlowsInSameBatch(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 
 	// Flow A: sport 1000. Flow B: sport 3000.
@@ -513,7 +505,7 @@ func TestCoalescerMultipleFlowsInSameBatch(t *testing.T) {
 // writing passthrough packets synchronously.
 func TestCoalescerPreservesArrivalOrder(t *testing.T) {
 	w := &orderedFakeWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	// Sequence: coalesceable TCP, ICMP (passthrough), coalesceable TCP on
 	// a different flow. Expected emit order: gso(X), plain(ICMP), gso(Y).
 	pay := make([]byte, 1200)
@@ -556,12 +548,7 @@ func (w *orderedFakeWriter) Write(p []byte) (int, error) {
 	return len(p), nil
 }
 
-// Read and Close exist solely to satisfy tio.Queue; order tests never
-// invoke them.
-func (w *orderedFakeWriter) Read(_ []wire.TunPacket, _ []byte) (int, error) { return 0, nil }
-func (w *orderedFakeWriter) Close() error                                   { return nil }
-
-func (w *orderedFakeWriter) WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte, _ wire.GSOProto) error {
+func (w *orderedFakeWriter) WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte, _ tio.GSOProto) error {
 	w.events = append(w.events, "gso")
 	return nil
 }
@@ -586,7 +573,7 @@ func stringSliceEq(a, b []string) bool {
 // packet (SYN) mid-flow only flushes its own flow, not others.
 func TestCoalescerInterleavedFlowsPreserveOrdering(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 
 	// Flow A two segments.
@@ -691,7 +678,7 @@ func buildTCPv6(tcLow byte, seq uint32, flags byte, payload []byte) []byte {
 // retains ECE on the wire.
 func TestCoalescerCoalescesEceFlow(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 	flags := byte(tcpAck | tcpEce)
 	if err := c.Commit(buildTCPv4(1000, flags, pay)); err != nil {
@@ -720,7 +707,7 @@ func TestCoalescerCoalescesEceFlow(t *testing.T) {
 // in-flow segment seeds a new slot rather than extending the prior burst.
 func TestCoalescerCwrSealsFlow(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 	if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
 		t.Fatal(err)
@@ -753,7 +740,7 @@ func TestCoalescerCwrSealsFlow(t *testing.T) {
 // a CE-echoing window or none.
 func TestCoalescerEceMismatchReseeds(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 	if err := c.Commit(buildTCPv4(1000, tcpAck|tcpEce, pay)); err != nil {
 		t.Fatal(err)
@@ -778,7 +765,7 @@ func TestCoalescerEceMismatchReseeds(t *testing.T) {
 // CE-marked packet still coalesces, and the merged superpacket carries CE.
 func TestCoalescerMergesCEMark(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 	if err := c.Commit(buildTCPv4WithToS(ecnECT0, 1000, tcpAck, pay)); err != nil {
 		t.Fatal(err)
@@ -809,7 +796,7 @@ func TestCoalescerMergesCEMark(t *testing.T) {
 // headersMatch did not also relax DSCP — different DSCP must still split.
 func TestCoalescerDscpMismatchReseeds(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 	// Same ECN (Not-ECT), different DSCP (0x10 vs 0x20 in upper 6 bits).
 	tosA := byte(0x10<<2) | ecnNotECT
@@ -832,7 +819,7 @@ func TestCoalescerDscpMismatchReseeds(t *testing.T) {
 // TestCoalescerCoalescesEceFlow.
 func TestCoalescerIPv6CoalescesEceFlow(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 	flags := byte(tcpAck | tcpEce)
 	if err := c.Commit(buildTCPv6(0, 1000, flags, pay)); err != nil {
@@ -863,7 +850,7 @@ func TestCoalescerIPv6CoalescesEceFlow(t *testing.T) {
 // seen had the wire never reordered.
 func TestCoalescerSortsReorderedSeedsAndMerges(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 	// Arrival order: seq 1000, 3400, 2200. The 3400 seeds a separate slot
 	// because 3400 != nextSeq=2200, then 2200 fails to extend the 3400 slot
@@ -899,7 +886,7 @@ func TestCoalescerSortsReorderedSeedsAndMerges(t *testing.T) {
 // without any cross-flow contamination.
 func TestCoalescerSortAcrossFlowsMergesEachIndependently(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 	// Flow A (sport 1000) seq 100, 1300; flow B (sport 3000) seq 500, 1700.
 	// Arrival: A.1300, B.1700, A.100, B.500 — every flow reordered.
@@ -950,7 +937,7 @@ func TestCoalescerSortAcrossFlowsMergesEachIndependently(t *testing.T) {
 // boundary by an arbitrary number of segments.
 func TestCoalescerSortKeepsPSHBoundary(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 	// Seq 1000 (no PSH) + 2200 (PSH) → seal one slot with PSH set.
 	// Seq 3400 (no PSH) is contiguous to 3400 from seq 2200+1200; without
@@ -978,7 +965,7 @@ func TestCoalescerSortKeepsPSHBoundary(t *testing.T) {
 // is sorted/merged independently.
 func TestCoalescerSortKeepsPassthroughBarrier(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 	// First two segments seed S1 (then a 3400 reorder seeds S2).
 	if err := c.Commit(buildTCPv4(1000, tcpAck, pay)); err != nil {
@@ -1011,7 +998,7 @@ func TestCoalescerSortKeepsPassthroughBarrier(t *testing.T) {
 // TestCoalescerMergesCEMark. ECN bits live in TC[1:0] = byte 1 mask 0x30.
 func TestCoalescerIPv6MergesCEMark(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w, test.NewLogger(), util.NewArena(0))
+	c := NewTCPCoalescer(w)
 	pay := make([]byte, 1200)
 	// tcLow is the low 4 bits of TC; ECN occupies the bottom 2 of those.
 	if err := c.Commit(buildTCPv6(ecnECT0, 1000, tcpAck, pay)); err != nil {

overlay/batch/tx_batch.go

@@ -1,10 +1,6 @@
 package batch
 
-import (
-	"net/netip"
-
-	"github.com/slackhq/nebula/util"
-)
+import "net/netip"
 
 const SendBatchCap = 128
 
@@ -15,29 +11,37 @@ type batchWriter interface {
 
 // SendBatch accumulates encrypted UDP packets and flushes them via WriteBatch.
 // One SendBatch is owned by each listenIn goroutine; no locking is needed.
-// Slot bytes are borrowed from the injected Arena and remain valid until
-// Flush, which Resets the arena.
+// The backing arena grows on demand: when there isn't room for the next slot
+// we allocate a fresh backing array. Already-committed slices keep referencing
+// the old array and remain valid until Flush drops them.
 type SendBatch struct {
-	out   batchWriter
-	bufs  [][]byte
-	dsts  []netip.AddrPort
-	ecns  []byte
-	arena *util.Arena
+	out     batchWriter
+	bufs    [][]byte
+	dsts    []netip.AddrPort
+	ecns    []byte
+	backing []byte
 }
 
-// NewSendBatch makes a SendBatch with batchCap slots backed by arena.
-func NewSendBatch(out batchWriter, batchCap int, arena *util.Arena) *SendBatch {
+func NewSendBatch(out batchWriter, batchCap, slotCap int) *SendBatch {
 	return &SendBatch{
-		out:   out,
-		bufs:  make([][]byte, 0, batchCap),
-		dsts:  make([]netip.AddrPort, 0, batchCap),
-		ecns:  make([]byte, 0, batchCap),
-		arena: arena,
+		out:     out,
+		bufs:    make([][]byte, 0, batchCap),
+		dsts:    make([]netip.AddrPort, 0, batchCap),
+		ecns:    make([]byte, 0, batchCap),
+		backing: make([]byte, 0, batchCap*slotCap),
 	}
 }
 
 func (b *SendBatch) Reserve(sz int) []byte {
-	return b.arena.Reserve(sz)
+	if len(b.backing)+sz > cap(b.backing) {
+		// Grow: allocate a fresh backing. Already-committed slices still
+		// reference the old array and remain valid until Flush drops them.
+		newCap := max(cap(b.backing)*2, sz)
+		b.backing = make([]byte, 0, newCap)
+	}
+	start := len(b.backing)
+	b.backing = b.backing[:start+sz]
+	return b.backing[start : start+sz : start+sz]
 }
 
 func (b *SendBatch) Commit(pkt []byte, dst netip.AddrPort, outerECN byte) {
@@ -55,6 +59,6 @@ func (b *SendBatch) Flush() error {
 	b.bufs = b.bufs[:0]
 	b.dsts = b.dsts[:0]
 	b.ecns = b.ecns[:0]
-	b.arena.Reset()
+	b.backing = b.backing[:0]
 	return err
 }

overlay/batch/tx_batch_test.go

@@ -3,8 +3,6 @@ package batch
 import (
 	"net/netip"
 	"testing"
-
-	"github.com/slackhq/nebula/util"
 )
 
 type fakeBatchWriter struct {
@@ -29,7 +27,7 @@ func (w *fakeBatchWriter) WriteBatch(bufs [][]byte, addrs []netip.AddrPort, ecns
 
 func TestSendBatchReserveCommitFlush(t *testing.T) {
 	fw := &fakeBatchWriter{}
-	b := NewSendBatch(fw, 4, util.NewArena(32))
+	b := NewSendBatch(fw, 4, 32)
 
 	ap := netip.MustParseAddrPort("10.0.0.1:4242")
 	for i := 0; i < 4; i++ {
@@ -73,7 +71,7 @@ func TestSendBatchReserveCommitFlush(t *testing.T) {
 
 func TestSendBatchSlotsDoNotOverlap(t *testing.T) {
 	fw := &fakeBatchWriter{}
-	b := NewSendBatch(fw, 3, util.NewArena(8))
+	b := NewSendBatch(fw, 3, 8)
 	ap := netip.MustParseAddrPort("10.0.0.1:80")
 
 	for i := 0; i < 3; i++ {
@@ -95,7 +93,7 @@ func TestSendBatchSlotsDoNotOverlap(t *testing.T) {
 func TestSendBatchGrowPreservesCommitted(t *testing.T) {
 	fw := &fakeBatchWriter{}
 	// Tiny initial backing forces a grow on the second Reserve.
-	b := NewSendBatch(fw, 1, util.NewArena(4))
+	b := NewSendBatch(fw, 1, 4)
 	ap := netip.MustParseAddrPort("10.0.0.1:80")
 
 	s1 := b.Reserve(4)

overlay/batch/udp_coalesce.go

@@ -5,8 +5,6 @@ import (
 	"io"
 
 	"github.com/slackhq/nebula/overlay/tio"
-	"github.com/slackhq/nebula/util"
-	"github.com/slackhq/nebula/wire"
 )
 
 // ipProtoUDP is the IANA protocol number for UDP.
@@ -68,8 +66,7 @@ type UDPCoalescer struct {
 	openSlots map[flowKey]*udpSlot
 	pool      []*udpSlot
 
-	// arena is injected; see TCPCoalescer.arena for the contract.
-	arena *util.Arena
+	backing []byte
 }
 
 // NewUDPCoalescer wraps w. The caller is responsible for only constructing
@@ -77,15 +74,15 @@ type UDPCoalescer struct {
 // the kernel may reject GSO_UDP_L4 writes. If w does not implement
 // tio.GSOWriter at all (single-packet Queue), the coalescer degrades to
 // plain Writes — same defensive shape as the TCP coalescer.
-func NewUDPCoalescer(w tio.Queue, arena *util.Arena) *UDPCoalescer {
+func NewUDPCoalescer(w io.Writer) *UDPCoalescer {
 	c := &UDPCoalescer{
 		plainW:    w,
 		slots:     make([]*udpSlot, 0, initialSlots),
 		openSlots: make(map[flowKey]*udpSlot, initialSlots),
 		pool:      make([]*udpSlot, 0, initialSlots),
-		arena:     arena,
+		backing:   make([]byte, 0, initialSlots*udpCoalesceBufSize),
 	}
-	if gw, ok := tio.SupportsGSO(w, wire.GSOProtoUDP); ok {
+	if gw, ok := tio.SupportsGSO(w, tio.GSOProtoUDP); ok {
 		c.gsoW = gw
 	}
 	return c
@@ -129,7 +126,7 @@ func parseUDP(pkt []byte) (parsedUDP, bool) {
 }
 
 func (c *UDPCoalescer) Reserve(sz int) []byte {
-	return c.arena.Reserve(sz)
+	return reserveFromBacking(&c.backing, sz)
 }
 
 // Commit borrows pkt. The caller must keep pkt valid until the next Flush.
@@ -186,7 +183,7 @@ func (c *UDPCoalescer) Flush() error {
 	clear(c.slots)
 	c.slots = c.slots[:0]
 	clear(c.openSlots)
-	c.arena.Reset()
+	c.backing = c.backing[:0]
 	return first
 }
 
@@ -315,7 +312,7 @@ func (c *UDPCoalescer) flushSlot(s *udpSlot) error {
 	udpCsumOff := s.ipHdrLen + 6
 	binary.BigEndian.PutUint16(hdr[udpCsumOff:udpCsumOff+2], foldOnceNoInvert(psum))
 
-	return c.gsoW.WriteGSO(hdr[:s.ipHdrLen], hdr[s.ipHdrLen:], s.payIovs, wire.GSOProtoUDP)
+	return c.gsoW.WriteGSO(hdr[:s.ipHdrLen], hdr[s.ipHdrLen:], s.payIovs, tio.GSOProtoUDP)
 }
 
 // udpHeadersMatch compares two IP+UDP header prefixes for byte-equality on

overlay/batch/udp_coalesce_test.go

@@ -3,8 +3,6 @@ package batch
 import (
 	"encoding/binary"
 	"testing"
-
-	"github.com/slackhq/nebula/util"
 )
 
 // buildUDPv4 builds a minimal IPv4+UDP packet with the given payload and ports.
@@ -62,7 +60,7 @@ func buildUDPv6(sport, dport uint16, payload []byte) []byte {
 
 func TestUDPCoalescerPassthroughWhenGSOUnavailable(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: false}
-	c := NewUDPCoalescer(w, util.NewArena(0))
+	c := NewUDPCoalescer(w)
 	pkt := buildUDPv4(1000, 53, make([]byte, 100))
 	if err := c.Commit(pkt); err != nil {
 		t.Fatal(err)
@@ -80,7 +78,7 @@ func TestUDPCoalescerPassthroughWhenGSOUnavailable(t *testing.T) {
 
 func TestUDPCoalescerNonUDPPassthrough(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewUDPCoalescer(w, util.NewArena(0))
+	c := NewUDPCoalescer(w)
 	// ICMP packet
 	pkt := make([]byte, 28)
 	pkt[0] = 0x45
@@ -101,7 +99,7 @@ func TestUDPCoalescerNonUDPPassthrough(t *testing.T) {
 
 func TestUDPCoalescerSeedThenFlushAlone(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewUDPCoalescer(w, util.NewArena(0))
+	c := NewUDPCoalescer(w)
 	pkt := buildUDPv4(1000, 53, make([]byte, 800))
 	if err := c.Commit(pkt); err != nil {
 		t.Fatal(err)
@@ -118,7 +116,7 @@ func TestUDPCoalescerSeedThenFlushAlone(t *testing.T) {
 
 func TestUDPCoalescerCoalescesEqualSized(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewUDPCoalescer(w, util.NewArena(0))
+	c := NewUDPCoalescer(w)
 	pay := make([]byte, 1200)
 	for i := 0; i < 3; i++ {
 		if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
@@ -158,7 +156,7 @@ func TestUDPCoalescerCoalescesEqualSized(t *testing.T) {
 // Last segment may be shorter, sealing the chain.
 func TestUDPCoalescerShortLastSegmentSeals(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewUDPCoalescer(w, util.NewArena(0))
+	c := NewUDPCoalescer(w)
 	full := make([]byte, 1200)
 	tail := make([]byte, 600)
 	if err := c.Commit(buildUDPv4(1000, 53, full)); err != nil {
@@ -191,7 +189,7 @@ func TestUDPCoalescerShortLastSegmentSeals(t *testing.T) {
 // A larger-than-gsoSize packet cannot extend the slot — it reseeds.
 func TestUDPCoalescerLargerThanSeedReseeds(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewUDPCoalescer(w, util.NewArena(0))
+	c := NewUDPCoalescer(w)
 	if err := c.Commit(buildUDPv4(1000, 53, make([]byte, 800))); err != nil {
 		t.Fatal(err)
 	}
@@ -209,7 +207,7 @@ func TestUDPCoalescerLargerThanSeedReseeds(t *testing.T) {
 // Different 5-tuples must not coalesce.
 func TestUDPCoalescerDifferentFlowsKeepSeparate(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewUDPCoalescer(w, util.NewArena(0))
+	c := NewUDPCoalescer(w)
 	pay := make([]byte, 800)
 	if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
 		t.Fatal(err)
@@ -240,7 +238,7 @@ func TestUDPCoalescerDifferentFlowsKeepSeparate(t *testing.T) {
 // Caps at udpCoalesceMaxSegs.
 func TestUDPCoalescerCapsAtMaxSegs(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewUDPCoalescer(w, util.NewArena(0))
+	c := NewUDPCoalescer(w)
 	pay := make([]byte, 100)
 	for i := 0; i < udpCoalesceMaxSegs+5; i++ {
 		if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
@@ -266,7 +264,7 @@ func TestUDPCoalescerCapsAtMaxSegs(t *testing.T) {
 // CE marks on appended segments must be merged into the seed's IP TOS.
 func TestUDPCoalescerMergesCEMark(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewUDPCoalescer(w, util.NewArena(0))
+	c := NewUDPCoalescer(w)
 	pay := make([]byte, 800)
 	pkt0 := buildUDPv4(1000, 53, pay) // ECN=00
 	pkt1 := buildUDPv4(1000, 53, pay)
@@ -295,7 +293,7 @@ func TestUDPCoalescerMergesCEMark(t *testing.T) {
 // IPv6 path: same flow, equal-sized → coalesced.
 func TestUDPCoalescerIPv6Coalesces(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewUDPCoalescer(w, util.NewArena(0))
+	c := NewUDPCoalescer(w)
 	pay := make([]byte, 1200)
 	for i := 0; i < 3; i++ {
 		if err := c.Commit(buildUDPv6(1000, 53, pay)); err != nil {
@@ -331,7 +329,7 @@ func TestUDPCoalescerIPv6Coalesces(t *testing.T) {
 // DSCP differences must reseed (headers don't match outside ECN).
 func TestUDPCoalescerDSCPMismatchReseeds(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewUDPCoalescer(w, util.NewArena(0))
+	c := NewUDPCoalescer(w)
 	pay := make([]byte, 800)
 	pkt0 := buildUDPv4(1000, 53, pay)
 	pkt1 := buildUDPv4(1000, 53, pay)
@@ -353,7 +351,7 @@ func TestUDPCoalescerDSCPMismatchReseeds(t *testing.T) {
 // Fragmented IPv4 must not be coalesced.
 func TestUDPCoalescerFragmentedIPv4PassesThrough(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewUDPCoalescer(w, util.NewArena(0))
+	c := NewUDPCoalescer(w)
 	pkt := buildUDPv4(1000, 53, make([]byte, 200))
 	binary.BigEndian.PutUint16(pkt[6:8], 0x2000) // MF=1
 	if err := c.Commit(pkt); err != nil {
@@ -370,7 +368,7 @@ func TestUDPCoalescerFragmentedIPv4PassesThrough(t *testing.T) {
 // IPv4 with options is not admissible (we require IHL=5).
 func TestUDPCoalescerIPv4WithOptionsPassesThrough(t *testing.T) {
 	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewUDPCoalescer(w, util.NewArena(0))
+	c := NewUDPCoalescer(w)
 	pkt := buildUDPv4(1000, 53, make([]byte, 200))
 	pkt[0] = 0x46 // IHL = 6 (24-byte IPv4 header — has options)
 	if err := c.Commit(pkt); err != nil {

overlay/overlaytest/noop.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
-	"github.com/slackhq/nebula/wire"
 )
 
 // NoopTun is an overlay.Device that silently discards every read and write.
@@ -16,10 +15,6 @@ import (
 // exercise the datapath.
 type NoopTun struct{}
 
-func (NoopTun) Capabilities() tio.Capabilities {
-	return tio.Capabilities{}
-}
-
 func (NoopTun) RoutesFor(addr netip.Addr) routing.Gateways {
 	return routing.Gateways{}
 }
@@ -36,8 +31,8 @@ func (NoopTun) Name() string {
 	return "noop"
 }
 
-func (NoopTun) Read(p []wire.TunPacket, mem []byte) (int, error) {
-	return 0, nil
+func (NoopTun) Read() ([]tio.Packet, error) {
+	return nil, nil
 }
 
 func (NoopTun) Write([]byte) (int, error) {

overlay/tio/queueset_gso_linux.go

@@ -62,10 +62,6 @@ func (c *offloadQueueSet) wakeForShutdown() error {
 }
 
 func (c *offloadQueueSet) Close() error {
-	if c.shutdownFd < 0 {
-		return nil
-	}
-
 	errs := []error{}
 
 	// Signal all readers blocked in poll to wake up and exit
@@ -79,12 +75,5 @@ func (c *offloadQueueSet) Close() error {
 		}
 	}
 
-	// All Offloads reference shutdownFd in their pollfd arrays, so close it
-	// only after every Offload.Close has returned.
-	if err := unix.Close(c.shutdownFd); err != nil {
-		errs = append(errs, err)
-	}
-	c.shutdownFd = -1
-
 	return errors.Join(errs...)
 }

overlay/tio/queueset_poll_linux.go

@@ -48,15 +48,11 @@ func (c *pollQueueSet) Add(fd int) error {
 func (c *pollQueueSet) wakeForShutdown() error {
 	var buf [8]byte
 	binary.NativeEndian.PutUint64(buf[:], 1)
-	_, err := unix.Write(c.shutdownFd, buf[:])
+	_, err := unix.Write(int(c.shutdownFd), buf[:])
 	return err
 }
 
 func (c *pollQueueSet) Close() error {
-	if c.shutdownFd < 0 {
-		return nil
-	}
-
 	errs := []error{}
 
 	if err := c.wakeForShutdown(); err != nil {
@@ -69,12 +65,5 @@ func (c *pollQueueSet) Close() error {
 		}
 	}
 
-	// All Polls reference shutdownFd in their pollfd arrays, so close it
-	// only after every Poll.Close has returned.
-	if err := unix.Close(c.shutdownFd); err != nil {
-		errs = append(errs, err)
-	}
-	c.shutdownFd = -1
-
 	return errors.Join(errs...)
 }

overlay/tio/segment_bench_test.go

@@ -2,11 +2,7 @@
 
 package tio
 
-import (
-	"testing"
-
-	"github.com/slackhq/nebula/wire"
-)
+import "testing"
 
 // fakeBatch stands in for batch.TxBatcher inside the bench — same shape
 // of pointer-capturing closure that sendInsideMessage builds.
@@ -25,27 +21,27 @@ type fakeIface struct {
 }
 
 // BenchmarkSegmentSuperpacketAllocsTSO measures allocation per
-// PerSegment call when a closure captures pointer-bearing receivers — the
-// realistic shape of sendInsideMessage's closure.
+// SegmentSuperpacket call when a closure captures pointer-bearing
+// receivers — the realistic shape of sendInsideMessage's closure.
 func BenchmarkSegmentSuperpacketAllocsTSO(b *testing.B) {
 	const mss = 1400
 	const numSeg = 32
 	pkt := buildTSOv6(mss*numSeg, mss)
-	gso := wire.GSOInfo{
+	gso := GSOInfo{
 		Size:      mss,
 		HdrLen:    60, // 40 (IPv6) + 20 (TCP)
 		CsumStart: 40,
-		Proto:     wire.GSOProtoTCP,
+		Proto:     GSOProtoTCP,
 	}
-	p := wire.TunPacket{Bytes: pkt, Meta: gso}
+	p := Packet{Bytes: pkt, GSO: gso}
 
 	hi := &fakeHostInfo{remoteIndexId: 0xdeadbeef}
 	f := &fakeIface{rebindCount: 7, hi: hi}
 	fb := &fakeBatch{}
 
-	// PerSegment consumes pkt destructively; refresh from a master copy
-	// each iter (matches the production pattern where every TUN read hands
-	// the segmenter a fresh kernel-supplied buffer).
+	// SegmentSuperpacket consumes pkt destructively; refresh from a master
+	// copy each iter (matches the production pattern where every TUN read
+	// hands the segmenter a fresh kernel-supplied buffer).
 	master := append([]byte(nil), pkt...)
 	work := make([]byte, len(pkt))
 	p.Bytes = work
@@ -54,7 +50,7 @@ func BenchmarkSegmentSuperpacketAllocsTSO(b *testing.B) {
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		copy(work, master)
-		err := p.PerSegment(func(seg []byte) error {
+		err := SegmentSuperpacket(p, func(seg []byte) error {
 			out := fb.Reserve(16 + len(seg) + 16)
 			out[0] = byte(f.rebindCount)
 			out[1] = byte(hi.counter)
@@ -63,7 +59,7 @@ func BenchmarkSegmentSuperpacketAllocsTSO(b *testing.B) {
 			return nil
 		})
 		if err != nil {
-			b.Fatalf("PerSegment: %v", err)
+			b.Fatalf("SegmentSuperpacket: %v", err)
 		}
 	}
 }

overlay/tio/segment_other.go

@@ -0,0 +1,22 @@
+//go:build !linux || android || e2e_testing
+
+package tio
+
+import "fmt"
+
+func protoFromGSOType(_ uint8) (GSOProto, error) {
+	return 0, fmt.Errorf("GSO unsupported")
+}
+
+// SegmentSuperpacket invokes fn once per segment of pkt. On non-Linux
+// builds (and Android/e2e_testing) this package does not provide a Queue
+// implementation, so any caller that does construct a Packet here can only
+// be operating on non-superpacket bytes and the stub forwards them
+// directly. A non-zero GSO field is a programming error from the caller
+// and returns an explicit error rather than silently misbehaving.
+func SegmentSuperpacket(pkt Packet, fn func(seg []byte) error) error {
+	if pkt.GSO.IsSuperpacket() {
+		return fmt.Errorf("tio: GSO superpacket on platform without segmentation support")
+	}
+	return fn(pkt.Bytes)
+}

overlay/tio/tio.go

@@ -2,8 +2,6 @@ package tio
 
 import (
 	"io"
-
-	"github.com/slackhq/nebula/wire"
 )
 
 // QueueSet holds one or many Queue objects and helps close them in an orderly way.
@@ -15,8 +13,10 @@ type QueueSet interface {
 	Add(fd int) error
 }
 
-// Capabilities advertises which kernel offload features a Queue successfully negotiated.
-// Callers consult this to decide which coalescers to wire onto the write path.
+// Capabilities advertises which kernel offload features a Queue
+// successfully negotiated. Callers consult this to decide which coalescers
+// to wire onto the write path — a Queue without TSO can't usefully accept a
+// TCPCoalescer, and a Queue without USO can't accept a UDPCoalescer.
 type Capabilities struct {
 	// TSO means the FD was opened with IFF_VNET_HDR and the kernel agreed
 	// to TUN_F_TSO4|TSO6 — i.e. WriteGSO with GSOProtoTCP is safe.
@@ -31,21 +31,96 @@ type Capabilities struct {
 type Queue interface {
 	io.Closer
 
-	// Read will read at least 1 packet from the tun (up to len(p)).
-	// mem will be used to provide the backing for each of p[n].Bytes.
-	// Callers should size mem and p to avoid exhausting mem before p.
-	// Returns the number of packets actually read, or error.
-	Read(p []wire.TunPacket, mem []byte) (int, error)
+	// Read returns one or more packets. The returned Packet.Bytes slices
+	// are borrowed from the Queue's internal buffer and are only valid
+	// until the next Read or Close on this Queue - callers must encrypt
+	// or copy each slice before the next call. A Packet may carry a
+	// GSO/USO superpacket (see GSOInfo); when GSO.IsSuperpacket() is
+	// true the caller must segment Bytes before treating it as a single
+	// IP datagram. Not safe for concurrent Reads.
+	Read() ([]Packet, error)
 
 	// Write emits a single packet on the plaintext (outside→inside)
-	// delivery path.
+	// delivery path. Not safe for concurrent Writes.
 	Write(p []byte) (int, error)
+}
 
-	// Capabilities returns the Queue's negotiated offload capabilities,
-	// or the zero value when q does not advertise any.
+// Packet is the unit Queue.Read returns. Bytes points into the queue's
+// internal buffer and is only valid until the next Read or Close on the
+// queue that produced it. GSO is the zero value for an already-segmented
+// IP datagram; when non-zero it describes a kernel-supplied TSO/USO
+// superpacket the caller must segment before consuming.
+type Packet struct {
+	Bytes []byte
+	GSO   GSOInfo
+}
+
+// GSOInfo describes a kernel-supplied superpacket sitting in Packet.Bytes.
+// The zero value means "not a superpacket" — Bytes is one regular IP
+// datagram and no segmentation is required.
+type GSOInfo struct {
+	// Size is the GSO segment size: max payload bytes per segment
+	// (== TCP MSS for TSO, == UDP payload chunk for USO). Zero means
+	// not a superpacket.
+	Size uint16
+	// HdrLen is the total L3+L4 header length within Bytes (already
+	// corrected via correctHdrLen, so safe to slice on).
+	HdrLen uint16
+	// CsumStart is the L4 header offset inside Bytes (== L3 header
+	// length).
+	CsumStart uint16
+	// Proto picks the L4 protocol (TCP or UDP) so the segmenter knows
+	// which checksum/header layout to apply.
+	Proto GSOProto
+}
+
+// IsSuperpacket reports whether g describes a multi-segment GSO/USO
+// superpacket that needs segmentation before its bytes can be encrypted
+// and sent on the wire.
+func (g GSOInfo) IsSuperpacket() bool { return g.Size > 0 }
+
+// Clone returns a Packet whose Bytes is a freshly allocated copy of p.Bytes,
+// safe to retain past the next Read or Close on the originating Queue.
+// GSO metadata is copied verbatim. Use this only when a caller genuinely
+// needs to outlive the borrowed-slice contract — the hot path reads should
+// continue to consume the borrow synchronously to avoid the allocation.
+func (p Packet) Clone() Packet {
+	if p.Bytes == nil {
+		return p
+	}
+	cp := make([]byte, len(p.Bytes))
+	copy(cp, p.Bytes)
+	return Packet{Bytes: cp, GSO: p.GSO}
+}
+
+// CapsProvider is an optional interface implemented by Queues that
+// successfully negotiated kernel offload features at open time. Callers
+// pick a write-path coalescer based on the result. Queues that don't
+// implement it are treated as having no offload capability — callers must
+// fall back to plain per-packet writes.
+type CapsProvider interface {
 	Capabilities() Capabilities
 }
 
+// QueueCapabilities returns q's negotiated offload capabilities, or the
+// zero value when q does not advertise any.
+func QueueCapabilities(q Queue) Capabilities {
+	if cp, ok := q.(CapsProvider); ok {
+		return cp.Capabilities()
+	}
+	return Capabilities{}
+}
+
+// GSOProto selects the L4 protocol for a GSO superpacket. Determines which
+// VIRTIO_NET_HDR_GSO_* type the writer stamps and which checksum offset
+// inside the transport header virtio NEEDS_CSUM expects.
+type GSOProto uint8
+
+const (
+	GSOProtoTCP GSOProto = iota
+	GSOProtoUDP
+)
+
 // GSOWriter is implemented by Queues that can emit a TCP or UDP superpacket
 // assembled from a header prefix plus one or more borrowed payload
 // fragments, in a single vectored write (writev with a leading
@@ -63,28 +138,33 @@ type Queue interface {
 // in pays except possibly the last is exactly the same size. proto picks
 // the L4 protocol so the writer knows which GSOType / CsumOffset to set.
 //
-// Callers should also consult Queue.Capabilities (via SupportsGSO) for
-// the per-protocol negotiated capability; an implementation of GSOWriter
-// is necessary but not sufficient since USO may not have been negotiated
-// even when TSO was.
+// Callers should also consult CapsProvider (via SupportsGSO or
+// QueueCapabilities) for the per-protocol negotiated capability; an
+// implementation of GSOWriter is necessary but not sufficient since USO
+// may not have been negotiated even when TSO was.
 type GSOWriter interface {
-	WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte, proto wire.GSOProto) error
+	WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte, proto GSOProto) error
 }
 
 // SupportsGSO reports whether w implements GSOWriter and the underlying
-// queue advertises the negotiated capability for `want` via Capabilities.
-func SupportsGSO(w Queue, want wire.GSOProto) (GSOWriter, bool) {
+// queue advertises the negotiated capability for `want`. A writer that
+// implements GSOWriter but not CapsProvider is treated as permissive
+// (used by tests and fakes that don't negotiate).
+func SupportsGSO(w any, want GSOProto) (GSOWriter, bool) {
 	gw, ok := w.(GSOWriter)
 	if !ok {
 		return nil, false
 	}
-	caps := w.Capabilities()
-	switch want {
-	case wire.GSOProtoTCP:
-		return gw, caps.TSO
-	case wire.GSOProtoUDP:
-		return gw, caps.USO
-	default:
-		return gw, false
+	cp, ok := w.(CapsProvider)
+	if !ok {
+		return gw, true
 	}
+	caps := cp.Capabilities()
+	switch want {
+	case GSOProtoTCP:
+		return gw, caps.TSO
+	case GSOProtoUDP:
+		return gw, caps.USO
+	}
+	return gw, false
 }

overlay/tio/tio_gso_linux.go

@@ -10,20 +10,31 @@ import (
 	"syscall"
 	"unsafe"
 
-	"github.com/slackhq/nebula/wire"
 	"golang.org/x/sys/unix"
 
 	"github.com/slackhq/nebula/overlay/tio/virtio"
 )
 
-// tunRxBufSize is the per-Read worst-case footprint for one kernel-supplied
-// packet body, which is at most ~64 KiB (tunReadBufSize). Segmentation
-// happens at encrypt time via wire.TunPacket.PerSegment on a per-routine
-// MTU-sized scratch, so the caller-supplied read buffer only holds raw
-// kernel-supplied bytes. Used by Read's drain loop to gate further reads
-// on whether the remaining buffer can still hold one worst-case packet.
+// tunRxBufSize is the per-Read worst-case footprint inside rxBuf: one
+// kernel-supplied packet body, which is at most ~64 KiB (tunReadBufSize).
+// Segmentation happens at encrypt time on a per-routine MTU-sized scratch
+// (see SegmentSuperpacket), so rxBuf only holds raw kernel-supplied bytes.
+// We round up to give comfortable margin for the drain headroom check
+// below.
 const tunRxBufSize = 64 * 1024
 
+// tunRxBufCap is the total size we allocate for the per-reader rx
+// buffer. With reads landing directly in rxBuf, each drain iteration
+// consumes up to tunRxBufSize of headroom for the kernel-supplied bytes.
+// Sized to two such iterations so the initial blocking read plus one
+// drain read both fit without partial-drop.
+const tunRxBufCap = tunRxBufSize * 2
+
+// tunDrainCap caps how many packets a single Read will accumulate via
+// the post-wake drain loop. Sized to soak up a burst of small ACKs while
+// bounding how much work a single caller holds before handing off.
+const tunDrainCap = 64
+
 // gsoMaxIovs caps the iovec budget WriteGSO assembles per call: 3 fixed
 // entries (virtio_net_hdr, IP hdr, transport hdr) plus up to gsoMaxIovs-3
 // payload fragments. Sized comfortably above the typical kernel GSO
@@ -38,7 +49,7 @@ const gsoMaxIovs = 256
 // CHECKSUM_UNNECESSARY so the receiving network stack skips L4 checksum
 // verification. All packets that reach the plain Write paths already carry
 // a valid L4 checksum (either supplied by a remote peer whose ciphertext we
-// AEAD-authenticated, produced by virtio.SegmentTCP/SegmentUDP during
+// AEAD-authenticated, produced by segmentTCPYield/segmentUDPYield during
 // superpacket segmentation, or built locally by CreateRejectPacket), so
 // trusting them is safe.
 var validVnetHdr = [virtio.Size]byte{unix.VIRTIO_NET_HDR_F_DATA_VALID}
@@ -56,15 +67,18 @@ type Offload struct {
 	// events.
 	writeLock sync.Mutex
 	closed    atomic.Bool
+	rxBuf     []byte   // backing store for kernel-handed packets read this drain
+	rxOff     int      // cursor into rxBuf for the current Read drain
+	pending   []Packet // packets returned from the most recent Read
 
 	// readVnetScratch holds the 10-byte virtio_net_hdr split off the front of
 	// every TUN read via readv(2). Decoupling the header from the packet body
-	// lets us read the body directly into the caller-supplied mem at the
-	// current rxOff with no userspace copy on the GSO_NONE fast path.
+	// lets us read the body directly into rxBuf at the current rxOff with
+	// no userspace copy on the GSO_NONE fast path.
 	readVnetScratch [virtio.Size]byte
 	// readIovs is the readv(2) iovec scratch wired once at construction —
 	// iovec[0] points at readVnetScratch; iovec[1].Base/Len is updated per
-	// read to address the caller-supplied mem slot.
+	// read to address the current rxBuf slot.
 	readIovs [2]unix.Iovec
 
 	// usoEnabled records whether the kernel agreed to TUN_F_USO* on this FD,
@@ -101,15 +115,16 @@ func newOffload(fd int, shutdownFd int, usoEnabled bool) (*Offload, error) {
 			{Fd: int32(shutdownFd), Events: unix.POLLIN},
 		},
 		writeLock: sync.Mutex{},
-		gsoIovs:   make([]unix.Iovec, 2, gsoMaxIovs),
+
+		rxBuf:   make([]byte, tunRxBufCap),
+		gsoIovs: make([]unix.Iovec, 2, gsoMaxIovs),
 	}
 
 	out.gsoIovs[0].Base = &out.gsoHdrBuf[0]
 	out.gsoIovs[0].SetLen(virtio.Size)
 
 	// readIovs[0] is wired once to the virtio_net_hdr scratch; per-read we
-	// only repoint readIovs[1] at the next caller-supplied mem slot
-	// (see readPacket).
+	// only repoint readIovs[1] at the next rxBuf slot (see readPacket).
 	out.readIovs[0].Base = &out.readVnetScratch[0]
 	out.readIovs[0].SetLen(virtio.Size)
 
@@ -171,19 +186,20 @@ func (r *Offload) blockOnWrite() error {
 }
 
 // readPacket issues a single readv(2) splitting the virtio_net_hdr off
-// into readVnetScratch and reading the packet body directly into mem.
-// Returns the body length (zero virtio header bytes, just the IP
-// packet/superpacket). block controls whether EAGAIN is retried via poll:
-// the initial read of a drain blocks; subsequent drain reads do not.
+// into readVnetScratch and reading the packet body directly into rxBuf at
+// the current rxOff. Returns the body length (zero virtio header bytes,
+// just the IP packet/superpacket). block controls whether EAGAIN is
+// retried via poll: the initial read of a drain blocks; subsequent drain
+// reads do not.
 //
-// The body iovec capacity is always tunReadBufSize; the Read drain loop
-// gates entry on len(mem)-rxOff >= tunRxBufSize, sized to hold one
-// worst-case kernel-supplied packet body. Without that gate the body
-// iovec could be smaller than the next inbound packet and the kernel
-// would truncate.
-func (r *Offload) readPacket(mem []byte, block bool) (int, error) {
+// The body iovec capacity is always tunReadBufSize; callers (the Read
+// drain loop) gate entry on tunRxBufCap-rxOff >= tunRxBufSize, sized to
+// hold one worst-case kernel-supplied packet body. Without that gate the
+// body iovec could be smaller than the next inbound packet and the
+// kernel would truncate.
+func (r *Offload) readPacket(block bool) (int, error) {
 	for {
-		r.readIovs[1].Base = &mem[0]
+		r.readIovs[1].Base = &r.rxBuf[r.rxOff]
 		r.readIovs[1].SetLen(tunReadBufSize)
 		n, _, errno := syscall.Syscall(unix.SYS_READV, uintptr(r.fd), uintptr(unsafe.Pointer(&r.readIovs[0])), uintptr(len(r.readIovs)))
 		if errno == 0 {
@@ -211,43 +227,39 @@ func (r *Offload) readPacket(mem []byte, block bool) (int, error) {
 	}
 }
 
-// Read returns one or more packets from the tun. Each wire.TunPacket
-// either carries a single ready-to-use IP datagram (GSO zero) or a TSO/USO
-// superpacket plus the wire.GSOInfo a caller needs to segment it (see
-// wire.TunPacket.PerSegment). The first read blocks via poll; once the fd
-// is known readable we drain additional packets non-blocking until the
-// kernel queue is empty (EAGAIN), p is full, or mem no longer has room
-// for another worst-case packet (tunRxBufSize). This amortizes the poll
-// wake over bursts of small packets (e.g. TCP ACKs). The Bytes slices on
-// returned packets point into the caller-supplied mem and are only valid
-// until the next Read or Close on this Queue.
-func (r *Offload) Read(p []wire.TunPacket, mem []byte) (int, error) {
-	maxP := len(p)
-	maxM := len(mem)
-	p = p[:0]
-	rxOff := 0
+// Read returns one or more packets from the tun. Each Packet either
+// carries a single ready-to-use IP datagram (GSO zero) or a TSO/USO
+// superpacket plus the GSOInfo a caller needs to segment it (see
+// SegmentSuperpacket). The first read blocks via poll; once the fd is
+// known readable we drain additional packets non-blocking until the
+// kernel queue is empty (EAGAIN), we've collected tunDrainCap packets,
+// or we're out of rxBuf headroom. This amortizes the poll wake over
+// bursts of small packets (e.g. TCP ACKs). Packet.Bytes slices point
+// into the Offload's internal buffer and are only valid until the next
+// Read or Close on this Queue.
+func (r *Offload) Read() ([]Packet, error) {
+	r.pending = r.pending[:0]
+	r.rxOff = 0
 
 	// Initial (blocking) read. Retry on decode errors so a single bad
 	// packet does not stall the reader.
 	for {
-		n, err := r.readPacket(mem, true)
+		n, err := r.readPacket(true)
 		if err != nil {
-			return 0, err
+			return nil, err
 		}
-		if p, err = r.decodeRead(p, mem, n); err != nil {
+		if err := r.decodeRead(n); err != nil {
 			// Drop and read again — a bad packet should not kill the reader.
 			continue
 		}
-
-		rxOff += n
 		break
 	}
 
-	// Drain: non-blocking reads until the kernel queue is empty, p is full,
-	// or mem no longer has room for another worst-case kernel-supplied
-	// packet (tunRxBufSize).
-	for len(p) < maxP && maxM-rxOff >= tunRxBufSize {
-		n, err := r.readPacket(mem[rxOff:], false)
+	// Drain: non-blocking reads until the kernel queue is empty, the drain
+	// cap is reached, or rxBuf no longer has room for another worst-case
+	// kernel-supplied packet (tunRxBufSize).
+	for len(r.pending) < tunDrainCap && tunRxBufCap-r.rxOff >= tunRxBufSize {
+		n, err := r.readPacket(false)
 		if err != nil {
 			// EAGAIN / EINTR / anything else: stop draining. We already
 			// have a valid batch from the first read.
@@ -256,66 +268,68 @@ func (r *Offload) Read(p []wire.TunPacket, mem []byte) (int, error) {
 		if n <= 0 {
 			break
 		}
-		if p, err = r.decodeRead(p, mem[rxOff:], n); err != nil {
+		if err := r.decodeRead(n); err != nil {
 			// Drop this packet and stop the drain; we'd rather hand off
 			// what we have than keep spinning here.
 			break
 		}
-		rxOff += n
 	}
 
-	return len(p), nil
+	return r.pending, nil
 }
 
-// decodeRead processes the packet sitting at mem[:pktLen]. The bytes stay
-// in mem — for GSO_NONE we slice them as a regular IP datagram (running
-// finishChecksum if NEEDS_CSUM is set); for TSO/USO superpackets we attach
-// the corrected GSO metadata so the caller can segment lazily at encrypt
-// time. The caller advances its own rxOff past the kernel-supplied body
-// and nothing else, since segmentation no longer writes back into mem.
-func (r *Offload) decodeRead(p []wire.TunPacket, mem []byte, pktLen int) ([]wire.TunPacket, error) {
+// decodeRead processes the packet sitting in rxBuf at rxOff (length
+// pktLen). The bytes stay in rxBuf — for GSO_NONE we slice them as a
+// regular IP datagram (running finishChecksum if NEEDS_CSUM is set);
+// for TSO/USO superpackets we attach the corrected GSO metadata so the
+// caller can segment lazily at encrypt time. rxOff advances past the
+// kernel-supplied body and nothing else, since segmentation no longer
+// writes back into rxBuf.
+func (r *Offload) decodeRead(pktLen int) error {
 	if pktLen <= 0 {
-		return p, fmt.Errorf("short tun read: %d", pktLen)
+		return fmt.Errorf("short tun read: %d", pktLen)
 	}
 	var hdr virtio.Hdr
 	hdr.Decode(r.readVnetScratch[:])
 
-	body := mem[:pktLen]
+	body := r.rxBuf[r.rxOff : r.rxOff+pktLen]
 
 	if hdr.GSOType == unix.VIRTIO_NET_HDR_GSO_NONE {
 		if hdr.Flags&unix.VIRTIO_NET_HDR_F_NEEDS_CSUM != 0 {
 			if err := virtio.FinishChecksum(body, hdr); err != nil {
-				return p, err
+				return err
 			}
 		}
-		p = append(p, wire.TunPacket{Bytes: body})
-		return p, nil
+		r.pending = append(r.pending, Packet{Bytes: body})
+		r.rxOff += pktLen
+		return nil
 	}
 
 	// GSO superpacket: validate, fix the kernel-supplied HdrLen on the
 	// FORWARD path (CorrectHdrLen), pick the L4 protocol, and attach
-	// the metadata. The bytes stay in mem untouched; segmentation
-	// happens in wire.TunPacket.PerSegment at encrypt time.
+	// the metadata. The bytes stay in rxBuf untouched, segmentation
+	// happens in SegmentSuperpacket at encrypt time.
 	if err := virtio.CheckValid(body, hdr); err != nil {
-		return p, err
+		return err
 	}
 	if err := virtio.CorrectHdrLen(body, &hdr); err != nil {
-		return p, err
+		return err
 	}
 	proto, err := protoFromGSOType(hdr.GSOType)
 	if err != nil {
-		return p, err
+		return err
 	}
-	p = append(p, wire.TunPacket{
+	r.pending = append(r.pending, Packet{
 		Bytes: body,
-		Meta: wire.GSOInfo{
+		GSO: GSOInfo{
 			Size:      hdr.GSOSize,
 			HdrLen:    hdr.HdrLen,
 			CsumStart: hdr.CsumStart,
 			Proto:     proto,
 		},
 	})
-	return p, nil
+	r.rxOff += pktLen
+	return nil
 }
 
 func (r *Offload) Write(buf []byte) (int, error) {
@@ -370,7 +384,7 @@ func (r *Offload) Capabilities() Capabilities {
 	return Capabilities{TSO: true, USO: r.usoEnabled}
 }
 
-func (r *Offload) WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte, proto wire.GSOProto) error {
+func (r *Offload) WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte, proto GSOProto) error {
 	if len(hdr) == 0 || len(pays) == 0 || len(transportHdr) == 0 {
 		return nil
 	}
@@ -378,7 +392,7 @@ func (r *Offload) WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte, proto
 	// seq/ack/dataoff/flags/window), UDP=6 (after sport/dport/length).
 	var csumOff uint16
 	switch proto {
-	case wire.GSOProtoUDP:
+	case GSOProtoUDP:
 		csumOff = 6
 	default:
 		csumOff = 16
@@ -393,7 +407,7 @@ func (r *Offload) WriteGSO(hdr []byte, transportHdr []byte, pays [][]byte, proto
 	if len(pays) > 1 {
 		ipVer := hdr[0] >> 4
 		switch {
-		case proto == wire.GSOProtoUDP && (ipVer == 4 || ipVer == 6):
+		case proto == GSOProtoUDP && (ipVer == 4 || ipVer == 6):
 			vhdr.GSOType = unix.VIRTIO_NET_HDR_GSO_UDP_L4
 		case ipVer == 6:
 			vhdr.GSOType = unix.VIRTIO_NET_HDR_GSO_TCPV6

overlay/tio/tio_poll_linux.go

@@ -5,7 +5,6 @@ import (
 	"os"
 	"sync/atomic"
 
-	"github.com/slackhq/nebula/wire"
 	"golang.org/x/sys/unix"
 )
 
@@ -20,6 +19,9 @@ type Poll struct {
 	readPoll  [2]unix.PollFd
 	writePoll [2]unix.PollFd
 	closed    atomic.Bool
+
+	readBuf  []byte
+	batchRet [1]Packet
 }
 
 func newPoll(fd int, shutdownFd int) (*Poll, error) {
@@ -29,7 +31,8 @@ func newPoll(fd int, shutdownFd int) (*Poll, error) {
 	}
 
 	out := &Poll{
-		fd: fd,
+		fd:      fd,
+		readBuf: make([]byte, tunReadBufSize),
 		readPoll: [2]unix.PollFd{
 			{Fd: int32(fd), Events: unix.POLLIN},
 			{Fd: int32(shutdownFd), Events: unix.POLLIN},
@@ -94,17 +97,13 @@ func (t *Poll) blockOnWrite() error {
 	return nil
 }
 
-func (t *Poll) Read(p []wire.TunPacket, mem []byte) (int, error) {
-	if len(p) == 0 || len(mem) == 0 {
-		return 0, nil //todo should this be an err?
-	}
-	p[0].Meta = wire.GSOInfo{}
-	n, err := t.readOne(mem)
+func (t *Poll) Read() ([]Packet, error) {
+	n, err := t.readOne(t.readBuf)
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
-	p[0].Bytes = mem[:n]
-	return 1, nil
+	t.batchRet[0] = Packet{Bytes: t.readBuf[:n]}
+	return t.batchRet[:], nil
 }
 
 func (t *Poll) readOne(to []byte) (int, error) {
@@ -163,7 +162,3 @@ func (t *Poll) Close() error {
 
 	return err
 }
-
-func (t *Poll) Capabilities() Capabilities {
-	return Capabilities{}
-}

overlay/tio/tun_file_linux_test.go

@@ -10,7 +10,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/slackhq/nebula/wire"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sys/unix"
 )
@@ -28,13 +27,16 @@ func newReadPipe(t *testing.T) int {
 }
 
 func TestPoll_WakeForShutdown_WakesFriends(t *testing.T) {
+	pipe1 := newReadPipe(t)
+	pipe2 := newReadPipe(t)
 	parent, err := NewPollQueueSet()
 	require.NoError(t, err)
-	require.NoError(t, parent.Add(newReadPipe(t)))
-	require.NoError(t, parent.Add(newReadPipe(t)))
-	// QueueSet.Close owns the read fds we Added — don't register a separate
-	// Cleanup to close them or we'll double-close whatever fd the kernel
-	// has since reused.
+	require.NoError(t, parent.Add(pipe1))
+	require.NoError(t, parent.Add(pipe2))
+	t.Cleanup(func() {
+		_ = unix.Close(pipe1)
+		_ = unix.Close(pipe2)
+	})
 
 	readers := parent.Queues()
 	errs := make([]error, len(readers))
@@ -43,8 +45,7 @@ func TestPoll_WakeForShutdown_WakesFriends(t *testing.T) {
 		wg.Add(1)
 		go func(i int, r Queue) {
 			defer wg.Done()
-			pkts := make([]wire.TunPacket, 1)
-			_, errs[i] = r.Read(pkts, make([]byte, 64))
+			_, errs[i] = r.Read()
 		}(i, r)
 	}
 
@@ -70,11 +71,7 @@ func TestPoll_WakeForShutdown_WakesFriends(t *testing.T) {
 }
 
 func TestPoll_Close_Idempotent(t *testing.T) {
-	shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = unix.Close(shutdownFd) })
-
-	tf, err := newPoll(newReadPipe(t), shutdownFd)
+	tf, err := newPoll(newReadPipe(t), 1)
 	require.NoError(t, err)
 	if err := tf.Close(); err != nil {
 		t.Fatalf("first Close: %v", err)
@@ -83,24 +80,3 @@ func TestPoll_Close_Idempotent(t *testing.T) {
 		t.Fatalf("second Close should be a no-op, got %v", err)
 	}
 }
-
-func TestPollQueueSet_Close_ClosesEventfd(t *testing.T) {
-	qs, err := NewPollQueueSet()
-	require.NoError(t, err)
-	require.NoError(t, qs.Add(newReadPipe(t)))
-
-	fd := qs.(*pollQueueSet).shutdownFd
-	require.NoError(t, qs.Close())
-
-	// Closing the eventfd again should fail with EBADF, proving Close
-	// actually released it.
-	if err := unix.Close(fd); err == nil {
-		t.Fatalf("eventfd %d still open after QueueSet.Close", fd)
-	}
-
-	// Second Close must be a no-op (and must not double-close the eventfd
-	// in case the kernel handed it out to another caller in the meantime).
-	if err := qs.Close(); err != nil {
-		t.Fatalf("second Close: %v", err)
-	}
-}

overlay/tio/tun_linux_offload.go

@@ -6,20 +6,46 @@ package tio
 import (
 	"fmt"
 
-	"github.com/slackhq/nebula/wire"
 	"golang.org/x/sys/unix"
+
+	"github.com/slackhq/nebula/overlay/tio/virtio"
 )
 
 // protoFromGSOType maps a virtio_net_hdr GSOType to the GSOProto value the
 // segment-time helpers use. Returns an error for GSO_NONE or any unknown
 // value — the caller should only invoke this on a confirmed superpacket.
-func protoFromGSOType(t uint8) (wire.GSOProto, error) {
+func protoFromGSOType(t uint8) (GSOProto, error) {
 	switch t {
 	case unix.VIRTIO_NET_HDR_GSO_TCPV4, unix.VIRTIO_NET_HDR_GSO_TCPV6:
-		return wire.GSOProtoTCP, nil
+		return GSOProtoTCP, nil
 	case unix.VIRTIO_NET_HDR_GSO_UDP_L4:
-		return wire.GSOProtoUDP, nil
+		return GSOProtoUDP, nil
 	default:
 		return 0, fmt.Errorf("unsupported virtio gso type: %d", t)
 	}
 }
+
+// SegmentSuperpacket invokes fn once per segment of pkt. For non-GSO pkts
+// fn is called once with pkt.Bytes (no segmentation, no copy). For GSO/USO
+// superpackets fn is called once per segment with a slice of pkt.Bytes
+// holding that segment's plaintext (a freshly-patched L3+L4 header sliced
+// in front of the original payload chunk). The slide is destructive: pkt is
+// consumed by this call and its bytes are in an undefined state when
+// SegmentSuperpacket returns. Callers must not retain pkt or any earlier
+// seg slice past fn's return for that segment. The scratch parameter is
+// unused on the destructive path and kept only for cross-platform
+// signature compatibility. Aborts and returns the first error from fn or
+// from per-segment construction.
+func SegmentSuperpacket(pkt Packet, fn func(seg []byte) error) error {
+	if !pkt.GSO.IsSuperpacket() {
+		return fn(pkt.Bytes)
+	}
+	switch pkt.GSO.Proto {
+	case GSOProtoTCP:
+		return virtio.SegmentTCP(pkt.Bytes, pkt.GSO.HdrLen, pkt.GSO.CsumStart, pkt.GSO.Size, fn)
+	case GSOProtoUDP:
+		return virtio.SegmentUDP(pkt.Bytes, pkt.GSO.HdrLen, pkt.GSO.CsumStart, pkt.GSO.Size, fn)
+	default:
+		return fmt.Errorf("unsupported gso proto: %d", pkt.GSO.Proto)
+	}
+}

overlay/tio/tun_linux_offload_test.go

@@ -12,7 +12,6 @@ import (
 	"gvisor.dev/gvisor/pkg/tcpip/checksum"
 
 	"github.com/slackhq/nebula/overlay/tio/virtio"
-	"github.com/slackhq/nebula/wire"
 )
 
 // testSegScratchSize is a generous segmentation scratch sized to fit any
@@ -27,11 +26,12 @@ func verifyChecksum(b []byte, pseudo uint16) bool {
 }
 
 // segmentForTest is the test-only counterpart to the production
-// wire.TunPacket.PerSegment path. It handles GSO_NONE (with optional
+// SegmentSuperpacket path. It handles GSO_NONE (with optional
 // finishChecksum) inline and dispatches GSO superpackets through
-// PerSegment, draining each yielded segment into a freshly-copied [][]byte
-// slot so callers can iterate after the call returns. Tests pre-set
-// hdr.HdrLen correctly, so correctHdrLen is not invoked here.
+// SegmentSuperpacket, draining each yielded segment into a
+// freshly-copied [][]byte slot so callers can iterate after the call
+// returns. Tests pre-set hdr.HdrLen correctly, so correctHdrLen is not
+// invoked here.
 func segmentForTest(pkt []byte, hdr virtio.Hdr, out *[][]byte, scratch []byte) error {
 	if hdr.GSOType == unix.VIRTIO_NET_HDR_GSO_NONE {
 		cp := append([]byte(nil), pkt...)
@@ -47,16 +47,13 @@ func segmentForTest(pkt []byte, hdr virtio.Hdr, out *[][]byte, scratch []byte) e
 	if err != nil {
 		return err
 	}
-	p := wire.TunPacket{
-		Bytes: pkt,
-		Meta: wire.GSOInfo{
-			Size:      hdr.GSOSize,
-			HdrLen:    hdr.HdrLen,
-			CsumStart: hdr.CsumStart,
-			Proto:     proto,
-		},
+	gso := GSOInfo{
+		Size:      hdr.GSOSize,
+		HdrLen:    hdr.HdrLen,
+		CsumStart: hdr.CsumStart,
+		Proto:     proto,
 	}
-	return p.PerSegment(func(seg []byte) error {
+	return SegmentSuperpacket(Packet{Bytes: pkt, GSO: gso}, func(seg []byte) error {
 		*out = append(*out, append([]byte(nil), seg...))
 		return nil
 	})
@@ -595,8 +592,8 @@ func BenchmarkSegmentTCPv4(b *testing.B) {
 			scratch := make([]byte, testSegScratchSize)
 			out := make([][]byte, 0, 64)
 
-			// PerSegment consumes its input destructively; restore pkt from
-			// a master copy each iteration. The restore mirrors the
+			// SegmentSuperpacket consumes its input destructively; restore
+			// pkt from a master copy each iteration. The restore mirrors the
 			// kernel→userspace copy that hands a fresh GSO blob to the
 			// segmenter in production, so it's representative cost rather
 			// than bench overhead.
@@ -676,21 +673,24 @@ func buildTSOv6(payLen, gso int) []byte {
 	return pkt
 }
 
-// TestDecodeReadFitsMaxTSO proves decodeRead can absorb a worst-case
-// 64KiB TSO superpacket without dropping it. With segmentation deferred to
-// encrypt time, decodeRead writes nothing — it just slices the
-// caller-supplied mem and attaches GSO metadata — so the size requirement
-// is just "fit one worst-case input."
+// TestDecodeReadFitsMaxTSOAtDrainThreshold proves the rxBuf sizing is
+// correct: when rxOff is at the maximum value the drain headroom check
+// allows, decodeRead must still be able to absorb a worst-case 64KiB
+// TSO superpacket without dropping the burst. With segmentation deferred
+// to encrypt time, decodeRead writes only the kernel-supplied bytes into
+// rxBuf, so the size requirement is just "fit one worst-case input."
 //
 // Regression history: in a prior layout the rx buffer doubled as the
 // segmentation output, a near-threshold drain read returned "scratch too
 // small", the whole 45-segment TSO burst was dropped, and the remote's TCP
-// fast-retransmit collapsed cwnd. Keeping this test guards against
-// re-introducing per-call sizing assumptions inside decodeRead.
-func TestDecodeReadFitsMaxTSO(t *testing.T) {
+// fast-retransmit collapsed cwnd. Keeping this test in the new layout
+// guards against re-introducing a drain headroom shortfall.
+func TestDecodeReadFitsMaxTSOAtDrainThreshold(t *testing.T) {
 	const ipv6HdrLen = 40
 	const tcpHdrLen = 20
 	const headerLen = ipv6HdrLen + tcpHdrLen
+	// Maximum TUN read body. The tunReadBufSize cap on readv's body iovec
+	// is what bounds the kernel's superpacket length.
 	pktLen := tunReadBufSize
 	payLen := pktLen - headerLen
 	const targetSegs = 64
@@ -701,12 +701,16 @@ func TestDecodeReadFitsMaxTSO(t *testing.T) {
 		t.Fatalf("buildTSOv6 produced %d bytes, want %d", len(pkt), pktLen)
 	}
 
-	o := &Offload{}
-	// mem is sized exactly to one worst-case packet — the caller-side
-	// invariant the drain loop in Read enforces. decodeRead must process
-	// the burst within that window.
-	mem := make([]byte, pktLen)
-	copy(mem, pkt)
+	o := &Offload{
+		rxBuf: make([]byte, tunRxBufCap),
+	}
+	// rxOff at the maximum value the drain headroom check permits before
+	// it would refuse another read. Any drain-time read up to this
+	// threshold MUST still process correctly.
+	o.rxOff = tunRxBufCap - tunRxBufSize
+
+	// Stage the body in rxBuf as if readv(2) just placed it there.
+	copy(o.rxBuf[o.rxOff:], pkt)
 
 	// Encode the matching virtio_net_hdr.
 	hdr := virtio.Hdr{
@@ -719,42 +723,50 @@ func TestDecodeReadFitsMaxTSO(t *testing.T) {
 	}
 	hdr.Encode(o.readVnetScratch[:])
 
-	var pkts []wire.TunPacket
-	pkts, err := o.decodeRead(pkts, mem, pktLen)
-	if err != nil {
-		t.Fatalf("decodeRead returned %v — sizing regression: "+
+	startRxOff := o.rxOff
+	if err := o.decodeRead(pktLen); err != nil {
+		t.Fatalf("decodeRead at drain threshold returned %v — rxBuf sizing regression: "+
 			"tunRxBufSize=%d must hold one worst-case input (%d)",
 			err, tunRxBufSize, pktLen)
 	}
 
-	if len(pkts) != 1 {
-		t.Fatalf("got %d packets, want 1 superpacket entry", len(pkts))
+	if len(o.pending) != 1 {
+		t.Fatalf("got %d packets, want 1 superpacket entry", len(o.pending))
 	}
-	got := pkts[0]
-	if !got.Meta.IsSuperpacket() {
-		t.Fatalf("expected superpacket GSO metadata, got %+v", got.Meta)
+	got := o.pending[0]
+	if !got.GSO.IsSuperpacket() {
+		t.Fatalf("expected superpacket GSO metadata, got %+v", got.GSO)
 	}
-	if got.Meta.Proto != wire.GSOProtoTCP {
-		t.Errorf("Meta.Proto=%d want TCP", got.Meta.Proto)
+	if got.GSO.Proto != GSOProtoTCP {
+		t.Errorf("GSO.Proto=%d want TCP", got.GSO.Proto)
 	}
-	if got.Meta.Size != uint16(gsoSize) {
-		t.Errorf("Meta.Size=%d want %d", got.Meta.Size, gsoSize)
+	if got.GSO.Size != uint16(gsoSize) {
+		t.Errorf("GSO.Size=%d want %d", got.GSO.Size, gsoSize)
 	}
-	if got.Meta.HdrLen != uint16(headerLen) {
-		t.Errorf("Meta.HdrLen=%d want %d", got.Meta.HdrLen, headerLen)
+	if got.GSO.HdrLen != uint16(headerLen) {
+		t.Errorf("GSO.HdrLen=%d want %d", got.GSO.HdrLen, headerLen)
 	}
-	if got.Meta.CsumStart != uint16(ipv6HdrLen) {
-		t.Errorf("Meta.CsumStart=%d want %d", got.Meta.CsumStart, ipv6HdrLen)
+	if got.GSO.CsumStart != uint16(ipv6HdrLen) {
+		t.Errorf("GSO.CsumStart=%d want %d", got.GSO.CsumStart, ipv6HdrLen)
 	}
 	if len(got.Bytes) != pktLen {
 		t.Errorf("len(Bytes)=%d want %d", len(got.Bytes), pktLen)
 	}
 
+	// rxOff advances exactly by the kernel-supplied body length — no
+	// segmentation output to account for any more.
+	if o.rxOff != startRxOff+pktLen {
+		t.Errorf("rxOff=%d want %d", o.rxOff, startRxOff+pktLen)
+	}
+	if o.rxOff > tunRxBufCap {
+		t.Fatalf("rxOff=%d overran rxBuf (cap=%d)", o.rxOff, tunRxBufCap)
+	}
+
 	// Validate that segmenting the returned superpacket reproduces the
 	// expected per-segment IPv6 payload length and TCP checksum.
 	wantSegs := (payLen + gsoSize - 1) / gsoSize
 	gotSegs := 0
-	if err := got.PerSegment(func(seg []byte) error {
+	if err := SegmentSuperpacket(got, func(seg []byte) error {
 		defer func() { gotSegs++ }()
 		if len(seg) < headerLen+1 {
 			t.Errorf("seg %d too short: %d", gotSegs, len(seg))
@@ -774,7 +786,7 @@ func TestDecodeReadFitsMaxTSO(t *testing.T) {
 		}
 		return nil
 	}); err != nil {
-		t.Fatalf("PerSegment: %v", err)
+		t.Fatalf("SegmentSuperpacket: %v", err)
 	}
 	if gotSegs != wantSegs {
 		t.Fatalf("got %d segments, want %d", gotSegs, wantSegs)

overlay/tun_android.go

@@ -16,7 +16,6 @@ import (
 	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
-	"github.com/slackhq/nebula/wire"
 )
 
 type tun struct {
@@ -26,19 +25,18 @@ type tun struct {
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
 	l           *slog.Logger
+
+	readBuf  []byte
+	batchRet [1]tio.Packet
 }
 
-func (t *tun) Read(p []wire.TunPacket, mem []byte) (int, error) {
-	if len(p) == 0 || len(mem) == 0 {
-		return 0, nil //todo should this be an err?
-	}
-	p[0].Meta = struct{}{}
-	n, err := t.rwc.Read(mem)
+func (t *tun) Read() ([]tio.Packet, error) {
+	n, err := t.rwc.Read(t.readBuf)
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
-	p[0].Bytes = mem[:n]
-	return 1, nil
+	t.batchRet[0] = tio.Packet{Bytes: t.readBuf[:n]}
+	return t.batchRet[:], nil
 }
 
 func (t *tun) Write(p []byte) (int, error) {
@@ -59,6 +57,7 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
 		fd:          deviceFd,
 		vpnNetworks: vpnNetworks,
 		l:           l,
+		readBuf:     make([]byte, defaultBatchBufSize),
 	}
 
 	err := t.reload(c, true)
@@ -129,7 +128,3 @@ func (t *tun) NewMultiQueueReader() error {
 func (t *tun) Readers() []tio.Queue {
 	return []tio.Queue{t}
 }
-
-func (t *tun) Capabilities() tio.Capabilities {
-	return tio.Capabilities{}
-}

overlay/tun_darwin.go

@@ -19,7 +19,6 @@ import (
 	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
-	"github.com/slackhq/nebula/wire"
 	netroute "golang.org/x/net/route"
 	"golang.org/x/sys/unix"
 )
@@ -36,6 +35,9 @@ type tun struct {
 
 	// cache out buffer since we need to prepend 4 bytes for tun metadata
 	out []byte
+
+	readBuf  []byte
+	batchRet [1]tio.Packet
 }
 
 type ifReq struct {
@@ -131,6 +133,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 		vpnNetworks: vpnNetworks,
 		DefaultMTU:  c.GetInt("tun.mtu", DefaultMTU),
 		l:           l,
+		readBuf:     make([]byte, defaultBatchBufSize),
 	}
 
 	err = t.reload(c, true)
@@ -504,17 +507,22 @@ func delRoute(prefix netip.Prefix, gateway netroute.Addr) error {
 	return nil
 }
 
-func (t *tun) Read(p []wire.TunPacket, mem []byte) (int, error) {
-	if len(p) == 0 || len(mem) <= 4 {
-		return 0, nil //todo should this be an err?
-	}
-	p[0].Meta = struct{}{}
-	n, err := t.rwc.Read(mem)
+func (t *tun) readOne(to []byte) (int, error) {
+	buf := make([]byte, len(to)+4)
+
+	n, err := t.rwc.Read(buf)
+
+	copy(to, buf[4:])
+	return n - 4, err
+}
+
+func (t *tun) Read() ([]tio.Packet, error) {
+	n, err := t.readOne(t.readBuf)
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
-	p[0].Bytes = mem[4:n]
-	return 1, nil
+	t.batchRet[0] = tio.Packet{Bytes: t.readBuf[:n]}
+	return t.batchRet[:], nil
 }
 
 // Write is only valid for single threaded use
@@ -565,7 +573,3 @@ func (t *tun) NewMultiQueueReader() error {
 func (t *tun) Readers() []tio.Queue {
 	return []tio.Queue{t}
 }
-
-func (t *tun) Capabilities() tio.Capabilities {
-	return tio.Capabilities{}
-}

overlay/tun_disabled.go

@@ -12,7 +12,6 @@ import (
 	"github.com/slackhq/nebula/iputil"
 	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
-	"github.com/slackhq/nebula/wire"
 )
 
 type disabledTun struct {
@@ -22,8 +21,43 @@ type disabledTun struct {
 	// Track these metrics since we don't have the tun device to do it for us
 	tx         metrics.Counter
 	rx         metrics.Counter
-	numReaders int
 	l          *slog.Logger
+	numReaders int
+}
+
+// disabledQueue is one tio.Queue view onto a shared disabledTun. Each queue
+// owns a private batchRet so concurrent Read calls from different reader
+// goroutines do not race on the returned slice.
+type disabledQueue struct {
+	parent   *disabledTun
+	batchRet [1]tio.Packet
+}
+
+func (q *disabledQueue) Read() ([]tio.Packet, error) {
+	r, ok := <-q.parent.read
+	if !ok {
+		return nil, io.EOF
+	}
+
+	q.parent.tx.Inc(1)
+	if q.parent.l.Enabled(context.Background(), slog.LevelDebug) {
+		q.parent.l.Debug("Write payload", "raw", prettyPacket(r))
+	}
+
+	q.batchRet[0] = tio.Packet{Bytes: r}
+	return q.batchRet[:], nil
+}
+
+// Write on a queue forwards to the underlying disabledTun. All queues share
+// one ICMP-handling/log path so this is a thin pass-through.
+func (q *disabledQueue) Write(b []byte) (int, error) {
+	return q.parent.Write(b)
+}
+
+// Close on a queue is a no-op. The shared channel and metrics are owned by
+// the disabledTun; Close on the device tears them down once for everybody.
+func (q *disabledQueue) Close() error {
+	return nil
 }
 
 func newDisabledTun(vpnNetworks []netip.Prefix, queueLen int, metricsEnabled bool, l *slog.Logger) *disabledTun {
@@ -61,37 +95,6 @@ func (*disabledTun) Name() string {
 	return "disabled"
 }
 
-func (t *disabledTun) readOne(b []byte) (int, error) {
-	r, ok := <-t.read
-	if !ok {
-		return 0, io.EOF
-	}
-
-	if len(r) > len(b) {
-		return 0, fmt.Errorf("packet larger than mtu: %d > %d bytes", len(r), len(b))
-	}
-
-	t.tx.Inc(1)
-	if t.l.Enabled(context.Background(), slog.LevelDebug) {
-		t.l.Debug("Write payload", "raw", prettyPacket(r))
-	}
-
-	return copy(b, r), nil
-}
-
-func (t *disabledTun) Read(p []wire.TunPacket, mem []byte) (int, error) {
-	if len(p) == 0 || len(mem) == 0 {
-		return 0, nil //todo should this be an err?
-	}
-	p[0].Meta = wire.GSOInfo{}
-	n, err := t.readOne(mem)
-	if err != nil {
-		return 0, err
-	}
-	p[0].Bytes = mem[:n]
-	return 1, nil
-}
-
 func (t *disabledTun) handleICMPEchoRequest(b []byte) bool {
 	out := make([]byte, len(b))
 	out = iputil.CreateICMPEchoResponse(b, out)
@@ -135,15 +138,11 @@ func (t *disabledTun) NewMultiQueueReader() error {
 func (t *disabledTun) Readers() []tio.Queue {
 	out := make([]tio.Queue, t.numReaders)
 	for i := range t.numReaders {
-		out[i] = t
+		out[i] = &disabledQueue{parent: t}
 	}
 	return out
 }
 
-func (t *disabledTun) Capabilities() tio.Capabilities {
-	return tio.Capabilities{}
-}
-
 func (t *disabledTun) Close() error {
 	if t.read != nil {
 		close(t.read)

overlay/tun_freebsd.go

@@ -17,7 +17,6 @@ import (
 	"unsafe"
 
 	"github.com/gaissmai/bart"
-	"github.com/slackhq/nebula/wire"
 
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/overlay/tio"
@@ -103,6 +102,9 @@ type tun struct {
 	readPoll  [2]unix.PollFd
 	writePoll [2]unix.PollFd
 	closed    atomic.Bool
+
+	readBuf  []byte
+	batchRet [1]tio.Packet
 }
 
 // blockOnRead waits until the tun fd is readable or shutdown has been signaled.
@@ -157,17 +159,13 @@ func (t *tun) blockOnWrite() error {
 	return nil
 }
 
-func (t *tun) Read(p []wire.TunPacket, mem []byte) (int, error) {
-	if len(p) == 0 || len(mem) == 0 {
-		return 0, nil //todo should this be an err?
-	}
-	p[0].Meta = struct{}{}
-	n, err := t.readOne(mem)
+func (t *tun) Read() ([]tio.Packet, error) {
+	n, err := t.readOne(t.readBuf)
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
-	p[0].Bytes = mem[:n]
-	return 1, nil
+	t.batchRet[0] = tio.Packet{Bytes: t.readBuf[:n]}
+	return t.batchRet[:], nil
 }
 
 func (t *tun) readOne(to []byte) (int, error) {
@@ -388,6 +386,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 		MTU:         c.GetInt("tun.mtu", DefaultMTU),
 		l:           l,
 		fd:          fd,
+		readBuf:     make([]byte, defaultBatchBufSize),
 		shutdownR:   shutdownR,
 		shutdownW:   shutdownW,
 		readPoll: [2]unix.PollFd{
@@ -610,10 +609,6 @@ func (t *tun) Readers() []tio.Queue {
 	return []tio.Queue{t}
 }
 
-func (t *tun) Capabilities() tio.Capabilities {
-	return tio.Capabilities{}
-}
-
 func (t *tun) removeRoutes(routes []Route) error {
 	for _, r := range routes {
 		if !r.Install {

overlay/tun_ios.go

@@ -19,7 +19,6 @@ import (
 	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
-	"github.com/slackhq/nebula/wire"
 )
 
 type tun struct {
@@ -28,19 +27,18 @@ type tun struct {
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
 	l           *slog.Logger
+
+	readBuf  []byte
+	batchRet [1]tio.Packet
 }
 
-func (t *tun) Read(p []wire.TunPacket, mem []byte) (int, error) {
-	if len(p) == 0 || len(mem) <= 4 {
-		return 0, nil //todo should this be an err?
-	}
-	p[0].Meta = struct{}{}
-	n, err := t.rwc.Read(mem)
+func (t *tun) Read() ([]tio.Packet, error) {
+	n, err := t.rwc.Read(t.readBuf)
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
-	p[0].Bytes = mem[4:n]
-	return 1, nil
+	t.batchRet[0] = tio.Packet{Bytes: t.readBuf[:n]}
+	return t.batchRet[:], nil
 }
 
 func (t *tun) Write(p []byte) (int, error) {
@@ -61,6 +59,7 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
 		vpnNetworks: vpnNetworks,
 		rwc:         &tunReadCloser{f: file},
 		l:           l,
+		readBuf:     make([]byte, defaultBatchBufSize),
 	}
 
 	err := t.reload(c, true)
@@ -119,9 +118,18 @@ type tunReadCloser struct {
 	wBuf []byte
 }
 
-// Read returns a packet with the BSD 4-byte header, watch out!
 func (tr *tunReadCloser) Read(to []byte) (int, error) {
-	return tr.f.Read(to)
+	tr.rMu.Lock()
+	defer tr.rMu.Unlock()
+
+	if cap(tr.rBuf) < len(to)+4 {
+		tr.rBuf = make([]byte, len(to)+4)
+	}
+	tr.rBuf = tr.rBuf[:len(to)+4]
+
+	n, err := tr.f.Read(tr.rBuf)
+	copy(to, tr.rBuf[4:])
+	return n - 4, err
 }
 
 func (tr *tunReadCloser) Write(from []byte) (int, error) {
@@ -176,7 +184,3 @@ func (t *tun) NewMultiQueueReader() error {
 func (t *tun) Readers() []tio.Queue {
 	return []tio.Queue{t}
 }
-
-func (t *tun) Capabilities() tio.Capabilities {
-	return tio.Capabilities{}
-}

overlay/tun_netbsd.go

@@ -19,7 +19,6 @@ import (
 	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
-	"github.com/slackhq/nebula/wire"
 	netroute "golang.org/x/net/route"
 	"golang.org/x/sys/unix"
 )
@@ -67,29 +66,24 @@ type tun struct {
 	l           *slog.Logger
 	f           *os.File
 	fd          int
+
+	readBuf  []byte
+	batchRet [1]tio.Packet
 }
 
-func (t *tun) Read(p []wire.TunPacket, mem []byte) (int, error) {
-	if len(p) == 0 || len(mem) == 0 {
-		return 0, nil //todo should this be an err?
-	}
-	p[0].Meta = struct{}{}
-	n, err := t.readOne(mem)
+func (t *tun) Read() ([]tio.Packet, error) {
+	n, err := t.readOne(t.readBuf)
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
-	p[0].Bytes = mem[:n]
-	return 1, nil
+	t.batchRet[0] = tio.Packet{Bytes: t.readBuf[:n]}
+	return t.batchRet[:], nil
 }
 
 func (t *tun) Readers() []tio.Queue {
 	return []tio.Queue{t}
 }
 
-func (t *tun) Capabilities() tio.Capabilities {
-	return tio.Capabilities{}
-}
-
 var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
 
 func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) {
@@ -124,6 +118,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 		vpnNetworks: vpnNetworks,
 		MTU:         c.GetInt("tun.mtu", DefaultMTU),
 		l:           l,
+		readBuf:     make([]byte, defaultBatchBufSize),
 	}
 
 	err = t.reload(c, true)

overlay/tun_openbsd.go

@@ -19,7 +19,6 @@ import (
 	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
-	"github.com/slackhq/nebula/wire"
 	netroute "golang.org/x/net/route"
 	"golang.org/x/sys/unix"
 )
@@ -60,19 +59,18 @@ type tun struct {
 	fd          int
 	// cache out buffer since we need to prepend 4 bytes for tun metadata
 	out []byte
+
+	readBuf  []byte
+	batchRet [1]tio.Packet
 }
 
-func (t *tun) Read(p []wire.TunPacket, mem []byte) (int, error) {
-	if len(p) == 0 || len(mem) <= 4 {
-		return 0, nil //todo should this be an err?
-	}
-	p[0].Meta = struct{}{}
-	n, err := t.f.Read(mem)
+func (t *tun) Read() ([]tio.Packet, error) {
+	n, err := t.readOne(t.readBuf)
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
-	p[0].Bytes = mem[4:n]
-	return 1, nil
+	t.batchRet[0] = tio.Packet{Bytes: t.readBuf[:n]}
+	return t.batchRet[:], nil
 }
 
 var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
@@ -109,6 +107,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 		vpnNetworks: vpnNetworks,
 		MTU:         c.GetInt("tun.mtu", DefaultMTU),
 		l:           l,
+		readBuf:     make([]byte, defaultBatchBufSize),
 	}
 
 	err = t.reload(c, true)
@@ -138,6 +137,15 @@ func (t *tun) Close() error {
 	return nil
 }
 
+func (t *tun) readOne(to []byte) (int, error) {
+	buf := make([]byte, len(to)+4)
+
+	n, err := t.f.Read(buf)
+
+	copy(to, buf[4:])
+	return n - 4, err
+}
+
 // Write is only valid for single threaded use
 func (t *tun) Write(from []byte) (int, error) {
 	buf := t.out
@@ -375,10 +383,6 @@ func (t *tun) Readers() []tio.Queue {
 	return []tio.Queue{t}
 }
 
-func (t *tun) Capabilities() tio.Capabilities {
-	return tio.Capabilities{}
-}
-
 func addRoute(prefix netip.Prefix, gateways []netip.Prefix) error {
 	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
 	if err != nil {

overlay/tun_tester.go

@@ -17,7 +17,6 @@ import (
 	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/udp"
-	"github.com/slackhq/nebula/wire"
 )
 
 type TestTun struct {
@@ -30,6 +29,8 @@ type TestTun struct {
 	closed    atomic.Bool
 	rxPackets chan []byte // Packets to receive into nebula
 	TxPackets chan []byte // Packets transmitted outside by nebula
+
+	batchRet [1]tio.Packet
 }
 
 func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*TestTun, error) {
@@ -50,6 +51,9 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*T
 		l:           l,
 		rxPackets:   make(chan []byte, 10),
 		TxPackets:   make(chan []byte, 10),
+		batchRet: [1]tio.Packet{
+			tio.Packet{Bytes: make([]byte, udp.MTU)},
+		},
 	}, nil
 }
 
@@ -164,17 +168,14 @@ func (t *TestTun) Close() error {
 	return nil
 }
 
-func (t *TestTun) Read(p []wire.TunPacket, mem []byte) (int, error) {
-	if len(p) == 0 || len(mem) == 0 {
-		return 0, nil //todo should this be an err?
-	}
-	p[0].Meta = struct{}{}
-	n, err := t.read(mem)
+func (t *TestTun) Read() ([]tio.Packet, error) {
+	t.batchRet[0].Bytes = t.batchRet[0].Bytes[:udp.MTU]
+	n, err := t.read(t.batchRet[0].Bytes)
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
-	p[0].Bytes = mem[:n]
-	return 1, nil
+	t.batchRet[0].Bytes = t.batchRet[0].Bytes[:n]
+	return t.batchRet[:], nil
 }
 
 func (t *TestTun) read(b []byte) (int, error) {
@@ -196,10 +197,6 @@ func (t *TestTun) Readers() []tio.Queue {
 	return []tio.Queue{t}
 }
 
-func (t *TestTun) Capabilities() tio.Capabilities {
-	return tio.Capabilities{}
-}
-
 func (t *TestTun) SupportsMultiqueue() bool {
 	return false
 }

overlay/tun_windows.go

@@ -21,7 +21,6 @@ import (
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	"github.com/slackhq/nebula/wintun"
-	"github.com/slackhq/nebula/wire"
 	"golang.org/x/sys/windows"
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
 )
@@ -46,19 +45,18 @@ type winTun struct {
 	l               *slog.Logger
 
 	tun *wintun.NativeTun
+
+	readBuf  []byte
+	batchRet [1]tio.Packet
 }
 
-func (t *winTun) Read(p []wire.TunPacket, mem []byte) (int, error) {
-	if len(p) == 0 || len(mem) == 0 {
-		return 0, nil //todo should this be an err?
-	}
-	p[0].Meta = struct{}{}
-	n, err := t.tun.Read(mem, 0)
+func (t *winTun) Read() ([]tio.Packet, error) {
+	n, err := t.tun.Read(t.readBuf, 0)
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
-	p[0].Bytes = mem[:n]
-	return 1, nil
+	t.batchRet[0] = tio.Packet{Bytes: t.readBuf[:n]}
+	return t.batchRet[:], nil
 }
 
 func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (Device, error) {
@@ -83,6 +81,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w
 	}
 
 	t := &winTun{
+		readBuf:     make([]byte, defaultBatchBufSize),
 		Device:          deviceName,
 		vpnNetworks:     vpnNetworks,
 		MTU:             c.GetInt("tun.mtu", DefaultMTU),
@@ -285,10 +284,6 @@ func (t *winTun) Readers() []tio.Queue {
 	return []tio.Queue{t}
 }
 
-func (t *winTun) Capabilities() tio.Capabilities {
-	return tio.Capabilities{}
-}
-
 func (t *winTun) Close() error {
 	// It seems that the Windows networking stack doesn't like it when we destroy interfaces that have active routes,
 	// so to be certain, just remove everything before destroying.

overlay/user.go

@@ -8,7 +8,6 @@ import (
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
-	"github.com/slackhq/nebula/wire"
 )
 
 func NewUserDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
@@ -38,23 +37,21 @@ type UserDevice struct {
 
 	inboundReader *io.PipeReader
 	inboundWriter *io.PipeWriter
+
+	readBuf  []byte
+	batchRet [1]tio.Packet
 }
 
-func (d *UserDevice) Capabilities() tio.Capabilities {
-	return tio.Capabilities{}
-}
-
-func (d *UserDevice) Read(p []wire.TunPacket, mem []byte) (int, error) {
-	if len(p) == 0 || len(mem) == 0 {
-		return 0, nil //todo should this be an err?
+func (d *UserDevice) Read() ([]tio.Packet, error) {
+	if d.readBuf == nil {
+		d.readBuf = make([]byte, defaultBatchBufSize)
 	}
-	p[0].Meta = wire.GSOInfo{}
-	n, err := d.outboundReader.Read(mem)
+	n, err := d.outboundReader.Read(d.readBuf)
 	if err != nil {
-		return 0, err
+		return nil, err
 	}
-	p[0].Bytes = mem[:n]
-	return 1, nil
+	d.batchRet[0] = tio.Packet{Bytes: d.readBuf[:n]}
+	return d.batchRet[:], nil
 }
 
 func (d *UserDevice) Activate() error {

udp/udp_ecn_outer_linux_test.go

@@ -12,7 +12,7 @@ import (
 // kernel stamps one outer codepoint per entry, so a run that straddled the
 // boundary would silently lose information).
 func TestPlanRunBreaksOnECNChange(t *testing.T) {
-	u := &StdConn{gsoSupported: true, maxGSOSegments: 63}
+	u := &StdConn{gsoSupported: true}
 	dst := netip.MustParseAddrPort("10.0.0.1:4242")
 
 	bufs := [][]byte{

udp/udp_linux.go

@@ -6,13 +6,10 @@ package udp
 import (
 	"context"
 	"encoding/binary"
-	"errors"
 	"fmt"
 	"log/slog"
 	"net"
 	"net/netip"
-	"strconv"
-	"strings"
 	"syscall"
 	"unsafe"
 
@@ -58,8 +55,7 @@ type StdConn struct {
 	// probed once at socket creation. When true, WriteBatch packs same-
 	// destination consecutive packets into a single sendmmsg entry with a
 	// UDP_SEGMENT cmsg; otherwise each packet is its own entry.
-	gsoSupported   bool
-	maxGSOSegments int
+	gsoSupported bool
 
 	// UDP GRO (recvmsg with UDP_GRO cmsg) support. groSupported is probed
 	// once at socket creation. When true, listenOutBatch allocates larger
@@ -106,11 +102,10 @@ func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int)
 	}
 	//gotta find out if we got an AF_INET6 socket or not:
 	out := &StdConn{
-		udpConn:        udpConn,
-		rawConn:        rawConn,
-		l:              l,
-		batch:          batch,
-		maxGSOSegments: 1,
+		udpConn: udpConn,
+		rawConn: rawConn,
+		l:       l,
+		batch:   batch,
 	}
 
 	af, err := out.getSockOptInt(unix.SO_DOMAIN)
@@ -193,6 +188,19 @@ func (u *StdConn) prepareWriteMessages(n int) {
 	}
 }
 
+// maxGSOSegments caps the per-sendmsg GSO fan-out. Linux kernels have
+// historically capped UDP_MAX_SEGMENTS at 64; newer kernels raise it to 128.
+// We stay one below 64 because the kernel's check is
+//
+//	if (cork->length > cork->gso_size * UDP_MAX_SEGMENTS) return -EINVAL;
+//
+// and cork->length includes the 8-byte UDP header (udp_sendmsg passes
+// ulen = len + sizeof(udphdr) to ip_append_data). Packing exactly 64
+// same-size segments puts cork->length at gso_size*64 + 8, which is one
+// UDP-header over the bound and the kernel rejects the whole sendmmsg
+// with EINVAL. 63 leaves room for the header for any segSize >= 8.
+const maxGSOSegments = 63
+
 // maxGSOBytes bounds the total payload per sendmsg() when UDP_SEGMENT is
 // set. The kernel stitches all iovecs into a single skb whose length the
 // UDP length field can represent, and also enforces sk_gso_max_size (which
@@ -203,8 +211,6 @@ const maxGSOBytes = 65000
 // prepareGSO probes UDP_SEGMENT support and sets u.gsoSupported on success.
 // Best-effort; failure leaves it false.
 func (u *StdConn) prepareGSO() {
-	u.maxGSOSegments = 63 //gotta be one less than the max so we can still attach a header
-
 	var probeErr error
 	if err := u.rawConn.Control(func(fd uintptr) {
 		probeErr = unix.SetsockoptInt(int(fd), unix.IPPROTO_UDP, unix.UDP_SEGMENT, 0)
@@ -218,20 +224,8 @@ func (u *StdConn) prepareGSO() {
 		recordCapability("udp.gso.enabled", false)
 		return
 	}
-
-	var un unix.Utsname
-	if err := unix.Uname(&un); err != nil {
-		u.l.Info("udp: GSO disabled", "reason", "kernel uname probe failed", "error", err)
-		recordCapability("udp.gso.enabled", false)
-		return
-	}
-	major, minor := parseRelease(string(un.Release[:]))
-	if major > 5 || (major == 5 && minor >= 5) {
-		u.maxGSOSegments = 127
-	}
-
 	u.gsoSupported = true
-	u.l.Info("udp: GSO enabled", "maxGSOSegments", u.maxGSOSegments)
+	u.l.Info("udp: GSO enabled")
 	recordCapability("udp.gso.enabled", true)
 }
 
@@ -275,41 +269,26 @@ func (u *StdConn) prepareGRO() {
 // codepoint through the EncReader for RFC 6040 combine on the decap side.
 // Best-effort: we keep going on failure.
 func (u *StdConn) prepareECNRecv() {
-	var v4err, v6err error
+	var probeErr error
 	if err := u.rawConn.Control(func(fd uintptr) {
-		v4err = unix.SetsockoptInt(int(fd), unix.IPPROTO_IP, unix.IP_RECVTOS, 1)
-		if !u.isV4 {
-			v6err = unix.SetsockoptInt(int(fd), unix.IPPROTO_IPV6, unix.IPV6_RECVTCLASS, 1)
+		if u.isV4 {
+			probeErr = unix.SetsockoptInt(int(fd), unix.IPPROTO_IP, unix.IP_RECVTOS, 1)
+		} else {
+			probeErr = unix.SetsockoptInt(int(fd), unix.IPPROTO_IPV6, unix.IPV6_RECVTCLASS, 1)
 		}
 	}); err != nil {
 		u.l.Info("udp: outer-ECN RX disabled", "reason", "rawconn control failed", "error", err)
 		recordCapability("udp.ecn_rx.enabled", false)
 		return
 	}
-	if u.isV4 { //only check the V4 attempt
-		if v4err != nil {
-			u.l.Info("udp: outer-ECN RX disabled", "reason", "kernel rejected probe", "error", v4err)
-			recordCapability("udp.ecn_rx.enabled", false)
-		} else {
-			u.ecnRecvSupported = true
-			u.l.Info("udp: outer-ECN RX enabled")
-			recordCapability("udp.ecn_rx.enabled", true)
-		}
-		return
-	} else {
-		if v6err != nil { //no V6 ECN? disable it.
-			u.l.Info("udp: outer-ECN RX disabled", "reason", "kernel rejected probe", "error", errors.Join(v4err, v6err))
-			recordCapability("udp.ecn_rx.enabled", false)
-			return
-		} else if v4err != nil { //no V4, but yes V6? Low level warning. Could be a V6-specific bind.
-			u.l.Debug("udp: outer-ECN RX degraded", "reason", "kernel rejected probe on IPv4", "error", v4err)
-		}
-		// all good
-		u.ecnRecvSupported = true
-		u.l.Info("udp: outer-ECN RX enabled")
-		recordCapability("udp.ecn_rx.enabled", true)
+	if probeErr != nil {
+		u.l.Info("udp: outer-ECN RX disabled", "reason", "kernel rejected probe", "error", probeErr)
+		recordCapability("udp.ecn_rx.enabled", false)
 		return
 	}
+	u.ecnRecvSupported = true
+	u.l.Info("udp: outer-ECN RX enabled")
+	recordCapability("udp.ecn_rx.enabled", true)
 }
 
 // recordCapability registers (or updates) a boolean gauge for one of the
@@ -522,6 +501,15 @@ func (u *StdConn) listenOutBatch(r EncReader, flush func()) error {
 	}
 }
 
+// headerCounter returns the big-endian uint64 message counter at bytes
+// [8:16] of a nebula packet, or 0 if the buffer is too short.
+func headerCounter(buf []byte) uint64 {
+	if len(buf) < 16 {
+		return 0
+	}
+	return binary.BigEndian.Uint64(buf[8:16])
+}
+
 // parseRecvCmsg walks the per-slot ancillary buffer once and extracts up to
 // two values of interest in a single pass: the UDP_GRO gso_size (when
 // wantGRO is true) and the outer IP-level ECN codepoint stamped on the
@@ -724,7 +712,7 @@ func (u *StdConn) planRun(bufs [][]byte, addrs []netip.AddrPort, ecns []byte, st
 	if ecns != nil {
 		ecn = ecns[start]
 	}
-	maxLen := u.maxGSOSegments
+	maxLen := maxGSOSegments
 	if iovBudget < maxLen {
 		maxLen = iovBudget
 	}
@@ -811,7 +799,9 @@ func writeSockaddr(buf []byte, addr netip.AddrPort, isV4 bool) (int, error) {
 		binary.BigEndian.PutUint16(buf[2:4], addr.Port())
 		ip4 := ap.As4()
 		copy(buf[4:8], ip4[:])
-		clear(buf[8:16])
+		for j := 8; j < 16; j++ {
+			buf[j] = 0
+		}
 		return unix.SizeofSockaddrInet4, nil
 	}
 	// struct sockaddr_in6: { sa_family_t(2), in_port_t(2, BE), flowinfo(4), in6_addr(16), scope_id(4) }
@@ -929,22 +919,3 @@ func NewUDPStatsEmitter(udpConns []Conn) func() {
 		}
 	}
 }
-
-func parseRelease(r string) (major, minor int) {
-	// strip anything after the second dot or any non-digit
-	parts := strings.SplitN(r, ".", 3)
-	if len(parts) < 2 {
-		return 0, 0
-	}
-	major, _ = strconv.Atoi(parts[0])
-	// minor may have trailing junk like "15-generic"
-	mp := parts[1]
-	for i, c := range mp {
-		if c < '0' || c > '9' {
-			mp = mp[:i]
-			break
-		}
-	}
-	minor, _ = strconv.Atoi(mp)
-	return
-}

util/arena.go

@@ -1,39 +0,0 @@
-package util
-
-// Arena is an injectable byte-slab that hands out non-overlapping borrowed
-// slices via Reserve and releases them in bulk via Reset.
-//
-// Arena is not safe for concurrent use.
-//
-// Reserve borrows; the slice is valid until the next Reset. The slab grows
-// (by allocating a fresh, larger backing array) if a Reserve doesn't fit;
-// pre-size the arena via NewArena to avoid that path on the hot path.
-type Arena struct {
-	buf []byte
-}
-
-// NewArena returns an Arena with a pre-allocated backing of the given capacity
-func NewArena(capacity int) *Arena {
-	return &Arena{buf: make([]byte, 0, capacity)}
-}
-
-// Reserve hands out a non-overlapping sz-byte slice from the arena. If the
-// request doesn't fit the current backing, a fresh, larger backing is
-// allocated; already-borrowed slices reference the old backing and remain
-// valid until Reset.
-func (a *Arena) Reserve(sz int) []byte {
-	if len(a.buf)+sz > cap(a.buf) {
-		newCap := max(cap(a.buf)*2, sz)
-		a.buf = make([]byte, 0, newCap)
-	}
-	start := len(a.buf)
-	a.buf = a.buf[:start+sz]
-	return a.buf[start : start+sz : start+sz]
-}
-
-// Reset releases every slice handed out since the last Reset. Callers must
-// not use any previously-borrowed slice after this returns. The underlying
-// backing array is retained so subsequent Reserves don't re-allocate.
-func (a *Arena) Reset() {
-	a.buf = a.buf[:0]
-}

wire/wire.go

@@ -1,46 +0,0 @@
-package wire
-
-// TunPacket is the unit a read from a tun device returns.
-// On supported platforms, it may be a superpacket, but a single TunPacket will never have more than one destination.
-type TunPacket struct {
-	// Bytes contains the actual packet
-	Bytes []byte
-	// Meta contains other information to help process the packet correctly, such as offsets for segmentation offloads
-	// Fields in Meta should be as portable/platform-agnostic as possible.
-	Meta GSOInfo
-}
-
-// GSOInfo describes a kernel-supplied superpacket sitting in Packet.Bytes.
-// The zero value means "not a superpacket" — Bytes is one regular IP
-// datagram and no segmentation is required.
-type GSOInfo struct {
-	// Size is the GSO segment size: max payload bytes per segment
-	// (== TCP MSS for TSO, == UDP payload chunk for USO). Zero means
-	// not a superpacket.
-	Size uint16
-	// HdrLen is the total L3+L4 header length within Bytes (already
-	// corrected via correctHdrLen, so safe to slice on).
-	HdrLen uint16
-	// CsumStart is the L4 header offset inside Bytes (== L3 header
-	// length).
-	CsumStart uint16
-	// Proto picks the L4 protocol (TCP or UDP) so the segmenter knows
-	// which checksum/header layout to apply.
-	Proto GSOProto
-}
-
-// GSOProto selects the L4 protocol for a GSO superpacket. Determines which
-// VIRTIO_NET_HDR_GSO_* type the writer stamps and which checksum offset
-// inside the transport header virtio NEEDS_CSUM expects.
-type GSOProto uint8
-
-const (
-	GSOProtoNone GSOProto = iota
-	GSOProtoTCP
-	GSOProtoUDP
-)
-
-// IsSuperpacket reports whether g describes a multi-segment GSO/USO
-// superpacket that needs segmentation before its bytes can be encrypted
-// and sent on the wire.
-func (g GSOInfo) IsSuperpacket() bool { return g.Size > 0 }

wire/wire_generic.go

@@ -1,10 +0,0 @@
-//go:build !linux
-// +build !linux
-
-package wire
-
-// PerSegment invokes fn once per segment of pkt.
-// This is a stub implementation that does not actually support segmentation
-func (t *TunPacket) PerSegment(fn func(seg []byte) error) error {
-	return fn(t.Bytes)
-}

wire/wire_linux.go

@@ -1,30 +0,0 @@
-package wire
-
-import (
-	"fmt"
-
-	"github.com/slackhq/nebula/overlay/tio/virtio"
-)
-
-// PerSegment invokes fn once per segment of t. For non-GSO packets fn is
-// called once with t.Bytes (no segmentation, no copy). For GSO/USO
-// superpackets fn is called once per segment with a slice of t.Bytes
-// holding that segment's plaintext (a freshly-patched L3+L4 header sliced
-// in front of the original payload chunk). The slide is destructive: t is
-// consumed by this call and its bytes are in an undefined state when
-// PerSegment returns. Callers must not retain t or any earlier seg slice
-// past fn's return for that segment. Aborts and returns the first error
-// from fn or from per-segment construction.
-func (t *TunPacket) PerSegment(fn func(seg []byte) error) error {
-	if !t.Meta.IsSuperpacket() {
-		return fn(t.Bytes)
-	}
-	switch t.Meta.Proto {
-	case GSOProtoTCP:
-		return virtio.SegmentTCP(t.Bytes, t.Meta.HdrLen, t.Meta.CsumStart, t.Meta.Size, fn)
-	case GSOProtoUDP:
-		return virtio.SegmentUDP(t.Bytes, t.Meta.HdrLen, t.Meta.CsumStart, t.Meta.Size, fn)
-	default:
-		return fmt.Errorf("unsupported gso proto: %d", t.Meta.Proto)
-	}
-}