PMTUD exploration, start small then grow

connection_manager.go

@@ -145,6 +145,7 @@ func (cm *connectionManager) getAndResetTrafficCheck(h *HostInfo, now time.Time)
 func (cm *connectionManager) AddTrafficWatch(h *HostInfo) {
 	if h.out.Swap(true) == false {
 		cm.trafficTimer.Add(h.localIndexId, cm.checkInterval)
+		cm.intf.pmtudManager.OnTunnelUp(h)
 	}
 }
 
@@ -153,8 +154,8 @@ func (cm *connectionManager) Start(ctx context.Context) {
 	defer clockSource.Stop()
 
 	p := []byte("")
-	// Long-lived buf for the traffic-check goroutine; never released.
+	nb := make([]byte, 12, 12)
-	buf := cm.intf.bufAlloc.Acquire()
+	out := make([]byte, mtu)
 
 	for {
 		select {
@@ -169,17 +170,18 @@ func (cm *connectionManager) Start(ctx context.Context) {
 					break
 				}
 
-				cm.doTrafficCheck(localIndex, p, buf, now)
+				cm.doTrafficCheck(localIndex, p, nb, out, now)
 			}
 		}
 	}
 }
 
-func (cm *connectionManager) doTrafficCheck(localIndex uint32, p []byte, buf *WireBuffer, now time.Time) {
+func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) {
 	decision, hostinfo, primary := cm.makeTrafficDecision(localIndex, now)
 
 	switch decision {
 	case deleteTunnel:
+		cm.intf.pmtudManager.OnTunnelDown(hostinfo)
 		if cm.hostMap.DeleteHostInfo(hostinfo) {
 			// Only clearing the lighthouse cache if this is the last hostinfo for this vpn ip in the hostmap
 			cm.intf.lightHouse.DeleteVpnAddrs(hostinfo.vpnAddrs)
@@ -199,7 +201,14 @@ func (cm *connectionManager) doTrafficCheck(localIndex uint32, p []byte, buf *Wi
 		cm.tryRehandshake(hostinfo)
 
 	case sendTestPacket:
-		cm.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, p, buf)
+		// Defer to pmtud if it has a confirmed PMTU > floor for this peer:
+		// the probe at the confirmed size verifies both liveness AND that
+		// the discovered PMTU still fits, so we don't burn a separate test
+		// packet on top of it. If pmtud declines (disabled, peer unsupported,
+		// or no confirmed size yet) we fall back to the regular test.
+		if !cm.intf.pmtudManager.MaybeProbeAsTest(hostinfo) {
+			cm.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, p, nb, out)
+		}
 	}
 
 	cm.resetRelayTrafficCheck(hostinfo)
@@ -308,9 +317,7 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
 		if err != nil {
 			cm.l.Error("failed to marshal Control message to migrate relay", "error", err)
 		} else {
-			migBuf := cm.intf.bufAlloc.Acquire()
+			cm.intf.SendMessageToHostInfo(header.Control, 0, newhostinfo, msg, make([]byte, 12), make([]byte, mtu))
-			cm.intf.SendMessageToHostInfo(header.Control, 0, newhostinfo, msg, migBuf)
-			cm.intf.bufAlloc.Release(migBuf)
 			cm.l.Info("send CreateRelayRequest",
 				"relayFrom", req.RelayFromAddr,
 				"relayTo", req.RelayToAddr,

connection_manager_test.go

@@ -67,9 +67,11 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	punchy := NewPunchyFromConfig(test.NewLogger(), conf)
 	nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy)
 	nc.intf = ifce
-
+	ifce.pmtudManager = newPMTUDManagerFromConfig(test.NewLogger(), conf, ifce.inside)
+	ifce.pmtudManager.intf = ifce
 	p := []byte("")
-	buf := NewWireBuffer(mtu, 0)
+	nb := make([]byte, 12, 12)
+	out := make([]byte, mtu)
 
 	// Add an ip we have established a connection w/ to hostmap
 	hostinfo := &HostInfo{
@@ -92,7 +94,7 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	assert.True(t, hostinfo.in.Load())
 
 	// Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded
-	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
 	assert.False(t, hostinfo.pendingDeletion.Load())
 	assert.False(t, hostinfo.out.Load())
 	assert.False(t, hostinfo.in.Load())
@@ -100,7 +102,7 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	// Do another traffic check tick, this host should be pending deletion now
 	nc.Out(hostinfo)
 	assert.True(t, hostinfo.out.Load())
-	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
 	assert.True(t, hostinfo.pendingDeletion.Load())
 	assert.False(t, hostinfo.out.Load())
 	assert.False(t, hostinfo.in.Load())
@@ -108,7 +110,7 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
 
 	// Do a final traffic check tick, the host should now be removed
-	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
 	assert.NotContains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs)
 	assert.NotContains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 }
@@ -149,9 +151,11 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 	punchy := NewPunchyFromConfig(test.NewLogger(), conf)
 	nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy)
 	nc.intf = ifce
-
+	ifce.pmtudManager = newPMTUDManagerFromConfig(test.NewLogger(), conf, ifce.inside)
+	ifce.pmtudManager.intf = ifce
 	p := []byte("")
-	buf := NewWireBuffer(mtu, 0)
+	nb := make([]byte, 12, 12)
+	out := make([]byte, mtu)
 
 	// Add an ip we have established a connection w/ to hostmap
 	hostinfo := &HostInfo{
@@ -174,14 +178,14 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 
 	// Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded
-	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
 	assert.False(t, hostinfo.pendingDeletion.Load())
 	assert.False(t, hostinfo.out.Load())
 	assert.False(t, hostinfo.in.Load())
 
 	// Do another traffic check tick, this host should be pending deletion now
 	nc.Out(hostinfo)
-	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
 	assert.True(t, hostinfo.pendingDeletion.Load())
 	assert.False(t, hostinfo.out.Load())
 	assert.False(t, hostinfo.in.Load())
@@ -190,7 +194,7 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 
 	// We saw traffic, should no longer be pending deletion
 	nc.In(hostinfo)
-	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
 	assert.False(t, hostinfo.pendingDeletion.Load())
 	assert.False(t, hostinfo.out.Load())
 	assert.False(t, hostinfo.in.Load())
@@ -361,6 +365,8 @@ func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) {
 	punchy := NewPunchyFromConfig(test.NewLogger(), conf)
 	nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy)
 	nc.intf = ifce
+	ifce.pmtudManager = newPMTUDManagerFromConfig(test.NewLogger(), conf, ifce.inside)
+	ifce.pmtudManager.intf = ifce
 	ifce.connectionManager = nc
 
 	hostinfo := &HostInfo{

control.go

@@ -54,6 +54,7 @@ type Control struct {
 	dnsStart               func()
 	lighthouseStart        func()
 	connectionManagerStart func(context.Context)
+	pmtudManagerStart      func(context.Context)
 }
 
 type ControlHostInfo struct {
@@ -107,6 +108,9 @@ func (c *Control) Start() (func() error, error) {
 	if c.connectionManagerStart != nil {
 		go c.connectionManagerStart(c.ctx)
 	}
+	if c.pmtudManagerStart != nil {
+		go c.pmtudManagerStart(c.ctx)
+	}
 	if c.lighthouseStart != nil {
 		c.lighthouseStart()
 	}
@@ -278,9 +282,15 @@ func (c *Control) CloseTunnel(vpnIp netip.Addr, localOnly bool) bool {
 	}
 
 	if !localOnly {
-		buf := c.f.bufAlloc.Acquire()
+		c.f.send(
-		c.f.send(header.CloseTunnel, 0, hostInfo.ConnectionState, hostInfo, []byte{}, buf)
+			header.CloseTunnel,
-		c.f.bufAlloc.Release(buf)
+			0,
+			hostInfo.ConnectionState,
+			hostInfo,
+			[]byte{},
+			make([]byte, 12, 12),
+			make([]byte, mtu),
+		)
 	}
 
 	c.f.closeTunnel(hostInfo)
@@ -290,14 +300,11 @@ func (c *Control) CloseTunnel(vpnIp netip.Addr, localOnly bool) bool {
 // CloseAllTunnels is just like CloseTunnel except it goes through and shuts them all down, optionally you can avoid shutting down lighthouse tunnels
 // the int returned is a count of tunnels closed
 func (c *Control) CloseAllTunnels(excludeLighthouses bool) (closed int) {
-	// One WireBuffer for the whole shutdown loop.
-	buf := c.f.bufAlloc.Acquire()
-	defer c.f.bufAlloc.Release(buf)
 	shutdown := func(h *HostInfo) {
 		if excludeLighthouses && c.f.lightHouse.IsAnyLighthouseAddr(h.vpnAddrs) {
 			return
 		}
-		c.f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, buf)
+		c.f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, make([]byte, 12, 12), make([]byte, mtu))
 		c.f.closeTunnel(h)
 
 		c.l.Debug("Sending close tunnel message",

control_tester.go

@@ -5,6 +5,8 @@ package nebula
 import (
 	"net/netip"
 
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/overlay"
 	"github.com/slackhq/nebula/udp"
@@ -20,9 +22,7 @@ func (c *Control) WaitForType(msgType header.MessageType, subType header.Message
 			panic(err)
 		}
 		pipeTo.InjectUDPPacket(p)
-		match := h.Type == msgType && h.Subtype == subType
+		if h.Type == msgType && h.Subtype == subType {
-		p.Release()
-		if match {
 			return
 		}
 	}
@@ -38,9 +38,7 @@ func (c *Control) WaitForTypeByIndex(toIndex uint32, msgType header.MessageType,
 			panic(err)
 		}
 		pipeTo.InjectUDPPacket(p)
-		match := h.RemoteIndex == toIndex && h.Type == msgType && h.Subtype == subType
+		if h.RemoteIndex == toIndex && h.Type == msgType && h.Subtype == subType {
-		p.Release()
-		if match {
 			return
 		}
 	}
@@ -92,15 +90,65 @@ func (c *Control) GetTunTxChan() <-chan []byte {
 	return c.f.inside.(*overlay.TestTun).TxPackets
 }
 
-// InjectUDPPacket injects a packet into the udp side. We copy internally so the caller keeps ownership of p.
+// InjectUDPPacket will inject a packet into the udp side of nebula
-// The copy comes from the freelist so steady-state alloc is zero.
 func (c *Control) InjectUDPPacket(p *udp.Packet) {
-	c.f.outside.(*udp.TesterConn).Send(p.Copy())
+	c.f.outside.(*udp.TesterConn).Send(p)
 }
 
-// InjectTunPacket pushes an IP packet onto the tun interface.
+// InjectTunUDPPacket puts a udp packet on the tun interface. Using UDP here because it's a simpler protocol
-func (c *Control) InjectTunPacket(packet []byte) {
+func (c *Control) InjectTunUDPPacket(toAddr netip.Addr, toPort uint16, fromAddr netip.Addr, fromPort uint16, data []byte) {
-	c.f.inside.(*overlay.TestTun).Send(packet)
+	serialize := make([]gopacket.SerializableLayer, 0)
+	var netLayer gopacket.NetworkLayer
+	if toAddr.Is6() {
+		if !fromAddr.Is6() {
+			panic("Cant send ipv6 to ipv4")
+		}
+		ip := &layers.IPv6{
+			Version:    6,
+			NextHeader: layers.IPProtocolUDP,
+			SrcIP:      fromAddr.Unmap().AsSlice(),
+			DstIP:      toAddr.Unmap().AsSlice(),
+		}
+		serialize = append(serialize, ip)
+		netLayer = ip
+	} else {
+		if !fromAddr.Is4() {
+			panic("Cant send ipv4 to ipv6")
+		}
+
+		ip := &layers.IPv4{
+			Version:  4,
+			TTL:      64,
+			Protocol: layers.IPProtocolUDP,
+			SrcIP:    fromAddr.Unmap().AsSlice(),
+			DstIP:    toAddr.Unmap().AsSlice(),
+		}
+		serialize = append(serialize, ip)
+		netLayer = ip
+	}
+
+	udp := layers.UDP{
+		SrcPort: layers.UDPPort(fromPort),
+		DstPort: layers.UDPPort(toPort),
+	}
+	err := udp.SetNetworkLayerForChecksum(netLayer)
+	if err != nil {
+		panic(err)
+	}
+
+	buffer := gopacket.NewSerializeBuffer()
+	opt := gopacket.SerializeOptions{
+		ComputeChecksums: true,
+		FixLengths:       true,
+	}
+
+	serialize = append(serialize, &udp, gopacket.Payload(data))
+	err = gopacket.SerializeLayers(buffer, opt, serialize...)
+	if err != nil {
+		panic(err)
+	}
+
+	c.f.inside.(*overlay.TestTun).Send(buffer.Bytes())
 }
 
 func (c *Control) GetVpnAddrs() []netip.Addr {

e2e/bench_test.go

@@ -1,68 +0,0 @@
-//go:build e2e_testing
-// +build e2e_testing
-
-package e2e
-
-import (
-	"testing"
-	"time"
-
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/e2e/router"
-)
-
-// BenchmarkHandshake measures end-to-end tunnel establishment time. The two
-// nodes and the router are constructed once before the loop so the timed window
-// is just the handshake itself: trigger packet -> handshake1 -> handshake2 ->
-// cached packet replay -> arrival on the remote TUN. Between iterations we
-// tear down both sides locally (no CloseTunnel notification on the wire) and
-// re-inject the lighthouse address that closeTunnel cleared, so the next
-// iteration runs through a fresh handshake against the same harness.
-func BenchmarkHandshake(b *testing.B) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	// Default try_interval is 100ms. The handshake manager schedules handshake1
-	// on its OutboundHandshakeTimer rather than firing immediately on trigger
-	// (the trigger channel only fast-paths static hosts), so a 100ms default
-	// drowns the actual handshake cost. Drop it to 1ms so the bench reflects
-	// the computation, not the wheel cadence.
-	bovr := m{"handshakes": m{"try_interval": "1ms"}}
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.1/24", bovr)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.2/24", bovr)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	myControl.Start()
-	theirControl.Start()
-	defer myControl.Stop()
-	defer theirControl.Stop()
-
-	r := router.NewR(b, myControl, theirControl)
-	r.CancelFlowLogs()
-	r.EnableFanIn()
-
-	trigger := BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-
-	b.ReportAllocs()
-	b.ResetTimer()
-
-	for n := 0; n < b.N; n++ {
-		myControl.InjectTunPacket(trigger)
-		// RouteForAllUntilTxTun returns the moment the cached packet arrives at
-		// the remote TUN, which is also when both sides are fully established.
-		_ = r.RouteForAllUntilTxTun(theirControl)
-
-		b.StopTimer()
-		// Local-only close removes hostmap state on both sides without putting a
-		// CloseTunnel packet on the wire that we'd then have to drain. The
-		// closeTunnel path also clears learned lighthouse state for the peer
-		// when the last hostinfo for that addr goes away, so we re-inject.
-		myControl.CloseTunnel(theirVpnIpNet[0].Addr(), true)
-		theirControl.CloseTunnel(myVpnIpNet[0].Addr(), true)
-		myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-		theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-		b.StartTimer()
-	}
-}

e2e/handshake_manager_test.go

@@ -47,7 +47,7 @@ func TestHandshakeRetransmitDuplicate(t *testing.T) {
 	defer r.RenderFlow()
 
 	t.Log("Trigger handshake from me to them")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
 
 	t.Log("Grab my msg1")
 	msg1 := myControl.GetFromUDP(true)
@@ -97,7 +97,7 @@ func TestHandshakeTruncatedPacketRecovery(t *testing.T) {
 	defer r.RenderFlow()
 
 	t.Log("Trigger handshake")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
 
 	t.Log("Get msg1 and deliver to responder")
 	msg1 := myControl.GetFromUDP(true)
@@ -146,7 +146,7 @@ func TestHandshakeOrphanedMsg2Dropped(t *testing.T) {
 	defer r.RenderFlow()
 
 	t.Log("Complete a normal handshake")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
 	r.RouteForAllUntilTxTun(theirControl)
 	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
 
@@ -248,7 +248,7 @@ func TestHandshakeLateResponse(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger handshake from me")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
 
 	t.Log("Grab msg1 but don't deliver")
 	msg1 := myControl.GetFromUDP(true)
@@ -292,7 +292,7 @@ func TestHandshakeSelfConnectionRejected(t *testing.T) {
 	myControl.Start()
 
 	t.Log("Trigger handshake from me")
-	myControl.InjectTunPacket(BuildTunUDPPacket(netip.MustParseAddr("10.128.0.2"), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi")))
+	myControl.InjectTunUDPPacket(netip.MustParseAddr("10.128.0.2"), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
 	msg1 := myControl.GetFromUDP(true)
 
 	t.Log("Drain any handshake retransmits before injecting")
@@ -375,7 +375,7 @@ func TestHandshakeRemoteAllowList(t *testing.T) {
 	defer r.RenderFlow()
 
 	t.Log("Trigger handshake from them")
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi")))
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi"))
 	msg1 := theirControl.GetFromUDP(true)
 
 	t.Log("Rewrite the source to a blocked IP and inject")
@@ -426,7 +426,7 @@ func TestHandshakeAlreadySeenPreferredRemote(t *testing.T) {
 	defer r.RenderFlow()
 
 	t.Log("Complete a normal handshake via the router")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
 	r.RouteForAllUntilTxTun(theirControl)
 	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
 
@@ -437,7 +437,7 @@ func TestHandshakeAlreadySeenPreferredRemote(t *testing.T) {
 	originalRemote := hi.CurrentRemote
 
 	t.Log("Re-trigger traffic to cause a new handshake attempt (ErrAlreadySeen)")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("roam")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("roam"))
 	r.RouteForAllUntilTxTun(theirControl)
 
 	t.Log("Verify tunnel still works")
@@ -475,8 +475,8 @@ func TestHandshakeWrongResponderPacketStore(t *testing.T) {
 	evilControl.Start()
 
 	t.Log("Send multiple packets to them (cached during handshake)")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet1")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet1"))
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet2")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet2"))
 
 	t.Log("Route until evil tunnel is closed")
 	h := &header.H{}
@@ -540,7 +540,7 @@ func TestHandshakeRelayComplete(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger handshake via relay")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi via relay")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi via relay"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	assertUdpPacket(t, []byte("Hi via relay"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
@@ -568,7 +568,7 @@ func TestHandshakeRelayComplete(t *testing.T) {
 }
 
 // NOTE: Relay V1 cert + IPv6 rejection is not tested here because
-// BuildTunUDPPacket from a V4 node to a V6 address panics in the test
+// InjectTunUDPPacket from a V4 node to a V6 address panics in the test
 // framework. The check is in handshake_manager.go handleOutbound relay
 // logic (lines ~304-313): if the relay host has a V1 cert and either
 // address is IPv6, the relay is skipped.

e2e/handshakes_test.go

@@ -16,7 +16,6 @@ import (
 	"github.com/slackhq/nebula/cert_test"
 	"github.com/slackhq/nebula/e2e/router"
 	"github.com/slackhq/nebula/header"
-	"github.com/slackhq/nebula/overlay"
 	"github.com/slackhq/nebula/udp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -40,22 +39,11 @@ func BenchmarkHotPath(b *testing.B) {
 	r.CancelFlowLogs()
 
 	assertTunnel(b, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	// Pre-build the IP packet bytes once so the bench measures the data plane,
-	// not gopacket SerializeLayers overhead.
-	prebuilt := BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-
-	// EnableFanIn switches the router to a 0-alloc routing path. Required
-	// for hot-path benchmarks; would conflict with GetFromUDP-using tests.
-	r.EnableFanIn()
-
 	b.ResetTimer()
 
 	for n := 0; n < b.N; n++ {
-		myControl.InjectTunPacket(prebuilt)
+		myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-		// Release the TUN-side bytes back to the harness freelist; the bench
+		_ = r.RouteForAllUntilTxTun(theirControl)
-		// just confirms a packet arrived, the contents aren't inspected.
-		overlay.ReleaseTunBuf(r.RouteForAllUntilTxTun(theirControl))
 	}
 
 	myControl.Stop()
@@ -83,15 +71,11 @@ func BenchmarkHotPathRelay(b *testing.B) {
 	theirControl.Start()
 
 	assertTunnel(b, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
-
-	prebuilt := BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-	r.EnableFanIn()
-
 	b.ResetTimer()
 
 	for n := 0; n < b.N; n++ {
-		myControl.InjectTunPacket(prebuilt)
+		myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-		overlay.ReleaseTunBuf(r.RouteForAllUntilTxTun(theirControl))
+		_ = r.RouteForAllUntilTxTun(theirControl)
 	}
 
 	myControl.Stop()
@@ -113,7 +97,7 @@ func TestGoodHandshake(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Send a udp packet through to begin standing up the tunnel, this should come out the other side")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	t.Log("Have them consume my stage 0 packet. They have a tunnel now")
 	theirControl.InjectUDPPacket(myControl.GetFromUDP(true))
@@ -165,7 +149,7 @@ func TestGoodHandshakeNoOverlap(t *testing.T) {
 
 	empty := []byte{}
 	t.Log("do something to cause a handshake")
-	myControl.GetF().SendMessageToVpnAddr(header.Test, header.MessageNone, theirVpnIpNet[0].Addr(), empty, nebula.NewWireBuffer(9001, 0))
+	myControl.GetF().SendMessageToVpnAddr(header.Test, header.MessageNone, theirVpnIpNet[0].Addr(), empty, empty, empty)
 
 	t.Log("Have them consume my stage 0 packet. They have a tunnel now")
 	theirControl.InjectUDPPacket(myControl.GetFromUDP(true))
@@ -207,7 +191,7 @@ func TestWrongResponderHandshake(t *testing.T) {
 	evilControl.Start()
 
 	t.Log("Start the handshake process, we will route until we see the evil tunnel closed")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	h := &header.H{}
 	r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType {
@@ -289,7 +273,7 @@ func TestWrongResponderHandshakeStaticHostMap(t *testing.T) {
 	evilControl.Start()
 
 	t.Log("Start the handshake process, we will route until we see the evil tunnel closed")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	h := &header.H{}
 	r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType {
@@ -368,8 +352,8 @@ func TestStage1Race(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger a handshake to start on both me and them")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them")))
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))
 
 	t.Log("Get both stage 1 handshake packets")
 	myHsForThem := myControl.GetFromUDP(true)
@@ -446,7 +430,7 @@ func TestUncleanShutdownRaceLoser(t *testing.T) {
 	theirControl.Start()
 
 	r.Log("Trigger a handshake from me to them")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
@@ -457,7 +441,7 @@ func TestUncleanShutdownRaceLoser(t *testing.T) {
 	myHostmap.Indexes = map[uint32]*nebula.HostInfo{}
 	myHostmap.RemoteIndexes = map[uint32]*nebula.HostInfo{}
 
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me again")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me again"))
 	p = r.RouteForAllUntilTxTun(theirControl)
 	assertUdpPacket(t, []byte("Hi from me again"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
 
@@ -496,7 +480,7 @@ func TestUncleanShutdownRaceWinner(t *testing.T) {
 	theirControl.Start()
 
 	r.Log("Trigger a handshake from me to them")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
@@ -508,7 +492,7 @@ func TestUncleanShutdownRaceWinner(t *testing.T) {
 	theirHostmap.Indexes = map[uint32]*nebula.HostInfo{}
 	theirHostmap.RemoteIndexes = map[uint32]*nebula.HostInfo{}
 
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them again")))
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them again"))
 	p = r.RouteForAllUntilTxTun(myControl)
 	assertUdpPacket(t, []byte("Hi from them again"), p, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), 80, 80)
 	r.RenderHostmaps("Derp hostmaps", myControl, theirControl)
@@ -551,7 +535,7 @@ func TestRelays(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	r.Log("Assert the tunnel works")
@@ -581,7 +565,7 @@ func TestRelaysDontCareAboutIps(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	r.Log("Assert the tunnel works")
@@ -611,14 +595,14 @@ func TestReestablishRelays(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	r.Log("Assert the tunnel works")
 	assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
 
 	t.Log("Ensure packet traversal from them to me via the relay")
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them")))
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))
 
 	p = r.RouteForAllUntilTxTun(myControl)
 	r.Log("Assert the tunnel works")
@@ -633,7 +617,7 @@ func TestReestablishRelays(t *testing.T) {
 	for curIndexes >= start {
 		curIndexes = len(myControl.GetHostmap().Indexes)
 		r.Logf("Wait for the dead index to go away:start=%v indexes, current=%v indexes", start, curIndexes)
-		myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me should fail")))
+		myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me should fail"))
 
 		r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType {
 			return router.RouteAndExit
@@ -650,7 +634,7 @@ func TestReestablishRelays(t *testing.T) {
 		myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
 		myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
 		relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-		myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+		myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 		p = r.RouteForAllUntilTxTun(theirControl)
 		r.Log("Assert the tunnel works")
@@ -685,7 +669,7 @@ func TestReestablishRelays(t *testing.T) {
 	t.Log("Assert the tunnel works the other way, too")
 	for {
 		t.Log("RouteForAllUntilTxTun")
-		theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them")))
+		theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))
 
 		p = r.RouteForAllUntilTxTun(myControl)
 		r.Log("Assert the tunnel works")
@@ -755,8 +739,8 @@ func TestStage1RaceRelays(t *testing.T) {
 	assertTunnel(t, theirVpnIpNet[0].Addr(), relayVpnIpNet[0].Addr(), theirControl, relayControl, r)
 
 	r.Log("Trigger a handshake from both them and me via relay to them and me")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them")))
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))
 
 	r.Log("Wait for a packet from them to me")
 	p := r.RouteForAllUntilTxTun(myControl)
@@ -803,8 +787,8 @@ func TestStage1RaceRelays2(t *testing.T) {
 	assertTunnel(t, theirVpnIpNet[0].Addr(), relayVpnIpNet[0].Addr(), theirControl, relayControl, r)
 
 	r.Log("Trigger a handshake from both them and me via relay to them and me")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them")))
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))
 
 	//r.RouteUntilAfterMsgType(myControl, header.Control, header.MessageNone)
 	//r.RouteUntilAfterMsgType(theirControl, header.Control, header.MessageNone)
@@ -868,7 +852,7 @@ func TestRehandshakingRelays(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	r.Log("Assert the tunnel works")
@@ -973,7 +957,7 @@ func TestRehandshakingRelaysPrimary(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	r.Log("Assert the tunnel works")
@@ -1275,8 +1259,8 @@ func TestRaceRegression(t *testing.T) {
 	//them rx stage:2 initiatorIndex=120607833 responderIndex=4209862089
 
 	t.Log("Start both handshakes")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them")))
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them"))
 
 	t.Log("Get both stage 1")
 	myStage1ForThem := myControl.GetFromUDP(true)
@@ -1492,7 +1476,7 @@ func TestGoodHandshakeUnsafeDest(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Send a udp packet through to begin standing up the tunnel, this should come out the other side")
-	myControl.InjectTunPacket(BuildTunUDPPacket(spookyDest, 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(spookyDest, 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 
 	t.Log("Have them consume my stage 0 packet. They have a tunnel now")
 	theirControl.InjectUDPPacket(myControl.GetFromUDP(true))
@@ -1520,7 +1504,7 @@ func TestGoodHandshakeUnsafeDest(t *testing.T) {
 	assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIpNet[0].Addr(), spookyDest, 80, 80)
 
 	//reply
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, spookyDest, 80, []byte("Hi from the spookyman")))
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, spookyDest, 80, []byte("Hi from the spookyman"))
 	//wait for reply
 	theirControl.WaitForType(1, 0, myControl)
 	theirCachedPacket := myControl.GetFromTun(true)

e2e/helpers_test.go

@@ -294,12 +294,12 @@ func deadline(t *testing.T, seconds time.Duration) doneCb {
 
 func assertTunnel(t testing.TB, vpnIpA, vpnIpB netip.Addr, controlA, controlB *nebula.Control, r *router.R) {
 	// Send a packet from them to me
-	controlB.InjectTunPacket(BuildTunUDPPacket(vpnIpA, 80, vpnIpB, 90, []byte("Hi from B")))
+	controlB.InjectTunUDPPacket(vpnIpA, 80, vpnIpB, 90, []byte("Hi from B"))
 	bPacket := r.RouteForAllUntilTxTun(controlA)
 	assertUdpPacket(t, []byte("Hi from B"), bPacket, vpnIpB, vpnIpA, 90, 80)
 
 	// And once more from me to them
-	controlA.InjectTunPacket(BuildTunUDPPacket(vpnIpB, 80, vpnIpA, 90, []byte("Hello from A")))
+	controlA.InjectTunUDPPacket(vpnIpB, 80, vpnIpA, 90, []byte("Hello from A"))
 	aPacket := r.RouteForAllUntilTxTun(controlB)
 	assertUdpPacket(t, []byte("Hello from A"), aPacket, vpnIpA, vpnIpB, 90, 80)
 }
@@ -408,58 +408,3 @@ func testLogLevelName() string {
 	}
 	return "info"
 }
-
-// BuildTunUDPPacket assembles an IP+UDP packet suitable for Control.InjectTunPacket.
-// Using UDP here because it's a simpler protocol.
-func BuildTunUDPPacket(toAddr netip.Addr, toPort uint16, fromAddr netip.Addr, fromPort uint16, data []byte) []byte {
-	serialize := make([]gopacket.SerializableLayer, 0)
-	var netLayer gopacket.NetworkLayer
-	if toAddr.Is6() {
-		if !fromAddr.Is6() {
-			panic("Cant send ipv6 to ipv4")
-		}
-		ip := &layers.IPv6{
-			Version:    6,
-			NextHeader: layers.IPProtocolUDP,
-			SrcIP:      fromAddr.Unmap().AsSlice(),
-			DstIP:      toAddr.Unmap().AsSlice(),
-		}
-		serialize = append(serialize, ip)
-		netLayer = ip
-	} else {
-		if !fromAddr.Is4() {
-			panic("Cant send ipv4 to ipv6")
-		}
-
-		ip := &layers.IPv4{
-			Version:  4,
-			TTL:      64,
-			Protocol: layers.IPProtocolUDP,
-			SrcIP:    fromAddr.Unmap().AsSlice(),
-			DstIP:    toAddr.Unmap().AsSlice(),
-		}
-		serialize = append(serialize, ip)
-		netLayer = ip
-	}
-
-	udp := layers.UDP{
-		SrcPort: layers.UDPPort(fromPort),
-		DstPort: layers.UDPPort(toPort),
-	}
-	if err := udp.SetNetworkLayerForChecksum(netLayer); err != nil {
-		panic(err)
-	}
-
-	buffer := gopacket.NewSerializeBuffer()
-	opt := gopacket.SerializeOptions{
-		ComputeChecksums: true,
-		FixLengths:       true,
-	}
-
-	serialize = append(serialize, &udp, gopacket.Payload(data))
-	if err := gopacket.SerializeLayers(buffer, opt, serialize...); err != nil {
-		panic(err)
-	}
-
-	return buffer.Bytes()
-}

e2e/router/router.go

@@ -13,7 +13,6 @@ import (
 	"regexp"
 	"sort"
 	"sync"
-	"sync/atomic"
 	"testing"
 	"time"
 
@@ -25,19 +24,6 @@ import (
 	"golang.org/x/exp/maps"
 )
 
-// outNatKey is the (from, to) pair used by outNat. Comparable struct, so it works as a map key without the
-// allocation cost of a string-concat key.
-type outNatKey struct {
-	from, to netip.AddrPort
-}
-
-// fannedPacket pairs a UDP TX packet with its source control so the router can route it after popping from
-// the fan-in channel.
-type fannedPacket struct {
-	from *nebula.Control
-	pkt  *udp.Packet
-}
-
 type R struct {
 	// Simple map of the ip:port registered on a control to the control
 	// Basically a router, right?
@@ -48,28 +34,12 @@ type R struct {
 
 	// A last used map, if an inbound packet hit the inNat map then
 	// all return packets should use the same last used inbound address for the outbound sender
-	outNat map[outNatKey]netip.AddrPort
+	// map[from address + ":" + to address] => ip:port to rewrite in the udp packet to receiver
+	outNat map[string]netip.AddrPort
 
 	// A map of vpn ip to the nebula control it belongs to
 	vpnControls map[netip.Addr]*nebula.Control
 
-	// Cached select infrastructure for RouteForAllUntilTxTun.
-	// The controls map is immutable after NewR so the cases are good for the test lifetime.
-	// We only rebuild if a different receiver is asked.
-	selRecvCtl *nebula.Control
-	selCases   []reflect.SelectCase
-	selCtls    []*nebula.Control
-
-	// Optional fan-in mode for hot-path benchmarks: one forwarder goroutine per control drains UDP TX into udpFanIn,
-	// so RouteForAllUntilTxTun can do a fixed 2-way native select instead of paying reflect.Select per call.
-	// Off by default (would otherwise interleave with tests that use GetFromUDP directly on the same control).
-	// Enabled by EnableFanIn.
-	udpFanIn  chan fannedPacket
-	stopFanIn chan struct{}
-	fanInWG   sync.WaitGroup
-	fanInMu   sync.Mutex
-	fanInOn   atomic.Bool
-
 	ignoreFlows []ignoreFlow
 	flow        []flowEntry
 
@@ -149,7 +119,7 @@ func NewR(t testing.TB, controls ...*nebula.Control) *R {
 		controls:     make(map[netip.AddrPort]*nebula.Control),
 		vpnControls:  make(map[netip.Addr]*nebula.Control),
 		inNat:        make(map[netip.AddrPort]*nebula.Control),
-		outNat:       make(map[outNatKey]netip.AddrPort),
+		outNat:       make(map[string]netip.AddrPort),
 		flow:         []flowEntry{},
 		ignoreFlows:  []ignoreFlow{},
 		fn:           filepath.Join("mermaid", fmt.Sprintf("%s.md", t.Name())),
@@ -183,10 +153,8 @@ func NewR(t testing.TB, controls ...*nebula.Control) *R {
 			case <-ctx.Done():
 				return
 			case <-clockSource.C:
-				r.Lock()
 				r.renderHostmaps("clock tick")
 				r.renderFlow()
-				r.Unlock()
 			}
 		}
 	}()
@@ -212,21 +180,15 @@ func (r *R) AddRoute(ip netip.Addr, port uint16, c *nebula.Control) {
 // RenderFlow renders the packet flow seen up until now and stops further automatic renders from happening.
 func (r *R) RenderFlow() {
 	r.cancelRender()
-	r.Lock()
-	defer r.Unlock()
 	r.renderFlow()
 }
 
 // CancelFlowLogs stops flow logs from being tracked and destroys any logs already collected
 func (r *R) CancelFlowLogs() {
 	r.cancelRender()
-	r.Lock()
 	r.flow = nil
-	r.Unlock()
 }
 
-// renderFlow writes the flow log to disk. Caller must hold r.Lock. renderFlow reads r.flow / r.additionalGraphs and
-// the *packet pointers stashed inside, all of which are mutated under the same lock by routing paths.
 func (r *R) renderFlow() {
 	if r.flow == nil {
 		return
@@ -472,157 +434,68 @@ func (r *R) RouteUntilTxTun(sender *nebula.Control, receiver *nebula.Control) []
 				panic("No control for udp tx " + a.String())
 			}
 			fp := r.unlockedInjectFlow(sender, c, p, false)
-			c.InjectUDPPacket(p) // copies internally; original is ours to release
+			c.InjectUDPPacket(p)
 			fp.WasReceived()
 			r.Unlock()
-			p.Release()
 		}
 	}
 }
 
-// RouteForAllUntilTxTun will route for everyone and return when a packet is seen on the receiver's tun.
+// RouteForAllUntilTxTun will route for everyone and return when a packet is seen on receivers tun
-// If a control's UDP TX address can't be matched to a registered control, we panic.
+// If the router doesn't have the nebula controller for that address, we panic
-//
-// For allocation-sensitive callers (hot-path benchmarks, in particular relay
-// benches with 3+ controls), call EnableFanIn() first.
 func (r *R) RouteForAllUntilTxTun(receiver *nebula.Control) []byte {
-	if r.fanInOn.Load() {
-		return r.routeFanIn(receiver)
-	}
-	return r.routeReflect(receiver)
-}
-
-// routeFanIn is the alloc-free path used when EnableFanIn is in effect.
-func (r *R) routeFanIn(receiver *nebula.Control) []byte {
-	tunTx := receiver.GetTunTxChan()
-	for {
-		select {
-		case p := <-tunTx:
-			r.Lock()
-			if r.flow != nil {
-				np := udp.Packet{Data: make([]byte, len(p))}
-				copy(np.Data, p)
-				r.unlockedInjectFlow(receiver, receiver, &np, true)
-			}
-			r.Unlock()
-			return p
-		case fp := <-r.udpFanIn:
-			r.routeUDP(fp.from, fp.pkt)
-		}
-	}
-}
-
-// routeReflect is the default reflect.Select-based path. Pays the boxing allocation per call but doesn't interfere
-// with tests that pull packets directly from controls' UDP TX channels via GetFromUDP.
-func (r *R) routeReflect(receiver *nebula.Control) []byte {
-	sc, cm := r.selectCasesFor(receiver)
-	for {
-		x, rx, _ := reflect.Select(sc)
-		if x == 0 {
-			p := rx.Interface().([]byte)
-			r.Lock()
-			if r.flow != nil {
-				np := udp.Packet{Data: make([]byte, len(p))}
-				copy(np.Data, p)
-				r.unlockedInjectFlow(cm[x], cm[x], &np, true)
-			}
-			r.Unlock()
-			return p
-		}
-		r.routeUDP(cm[x], rx.Interface().(*udp.Packet))
-	}
-}
-
-// EnableFanIn switches RouteForAllUntilTxTun to the alloc-free fan-in path.
-// One forwarder goroutine per registered control drains UDP TX into a shared channel that RouteForAllUntilTxTun selects
-// on alongside the receiver's TUN TX channel.
-func (r *R) EnableFanIn() {
-	r.fanInMu.Lock()
-	defer r.fanInMu.Unlock()
-	if r.fanInOn.Load() {
-		return
-	}
-	r.udpFanIn = make(chan fannedPacket, 32)
-	r.stopFanIn = make(chan struct{})
-	for _, c := range r.controls {
-		r.startFanInWorker(c)
-	}
-	r.fanInOn.Store(true)
-	r.t.Cleanup(r.stopFanInWorkers)
-}
-
-// startFanInWorker spawns a goroutine that drains c's UDP TX into r.udpFanIn.
-func (r *R) startFanInWorker(c *nebula.Control) {
-	r.fanInWG.Add(1)
-	udpTx := c.GetUDPTxChan()
-	go func() {
-		defer r.fanInWG.Done()
-		for {
-			select {
-			case <-r.stopFanIn:
-				return
-			case p := <-udpTx:
-				select {
-				case <-r.stopFanIn:
-					p.Release()
-					return
-				case r.udpFanIn <- fannedPacket{from: c, pkt: p}:
-				}
-			}
-		}
-	}()
-}
-
-// stopFanInWorkers signals the fan-in goroutines to exit and waits for them.
-func (r *R) stopFanInWorkers() {
-	r.fanInMu.Lock()
-	wasOn := r.fanInOn.Swap(false)
-	r.fanInMu.Unlock()
-	if !wasOn {
-		return
-	}
-	close(r.stopFanIn)
-	r.fanInWG.Wait()
-}
-
-// routeUDP forwards a UDP TX packet from the named source control to the destination control derived from p.To,
-// releasing the source packet after InjectUDPPacket has copied its bytes into a fresh pool slot.
-func (r *R) routeUDP(from *nebula.Control, p *udp.Packet) {
-	r.Lock()
-	defer r.Unlock()
-	a := from.GetUDPAddr()
-	c := r.getControl(a, p.To, p)
-	if c == nil {
-		panic(fmt.Sprintf("No control for udp tx %s", p.To))
-	}
-	fp := r.unlockedInjectFlow(from, c, p, false)
-	c.InjectUDPPacket(p) // copies internally; original is ours to release
-	fp.WasReceived()
-	p.Release()
-}
-
-// selectCasesFor returns the SelectCase array used by routeReflect: one slot for the receiver's TUN TX channel followed
-// by one per control's UDP TX channel. Cached for the test lifetime, only rebuilt if the receiver changes.
-func (r *R) selectCasesFor(receiver *nebula.Control) ([]reflect.SelectCase, []*nebula.Control) {
-	r.Lock()
-	defer r.Unlock()
-	if r.selRecvCtl == receiver && r.selCases != nil {
-		return r.selCases, r.selCtls
-	}
 	sc := make([]reflect.SelectCase, len(r.controls)+1)
 	cm := make([]*nebula.Control, len(r.controls)+1)
-	sc[0] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(receiver.GetTunTxChan())}
+
-	cm[0] = receiver
+	i := 0
-	i := 1
+	sc[i] = reflect.SelectCase{
+		Dir:  reflect.SelectRecv,
+		Chan: reflect.ValueOf(receiver.GetTunTxChan()),
+		Send: reflect.Value{},
+	}
+	cm[i] = receiver
+
+	i++
 	for _, c := range r.controls {
-		sc[i] = reflect.SelectCase{Dir: reflect.SelectRecv, Chan: reflect.ValueOf(c.GetUDPTxChan())}
+		sc[i] = reflect.SelectCase{
+			Dir:  reflect.SelectRecv,
+			Chan: reflect.ValueOf(c.GetUDPTxChan()),
+			Send: reflect.Value{},
+		}
+
 		cm[i] = c
 		i++
 	}
-	r.selRecvCtl = receiver
+
-	r.selCases = sc
+	for {
-	r.selCtls = cm
+		x, rx, _ := reflect.Select(sc)
-	return sc, cm
+		r.Lock()
+
+		if x == 0 {
+			// we are the tun tx, we can exit
+			p := rx.Interface().([]byte)
+			np := udp.Packet{Data: make([]byte, len(p))}
+			copy(np.Data, p)
+
+			r.unlockedInjectFlow(cm[x], cm[x], &np, true)
+			r.Unlock()
+			return p
+
+		} else {
+			// we are a udp tx, route and continue
+			p := rx.Interface().(*udp.Packet)
+			a := cm[x].GetUDPAddr()
+			c := r.getControl(a, p.To, p)
+			if c == nil {
+				r.Unlock()
+				panic(fmt.Sprintf("No control for udp tx %s", p.To))
+			}
+			fp := r.unlockedInjectFlow(cm[x], c, p, false)
+			c.InjectUDPPacket(p)
+			fp.WasReceived()
+		}
+		r.Unlock()
+	}
 }
 
 // RouteExitFunc will call the whatDo func with each udp packet from sender.
@@ -649,7 +522,6 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
 		switch e {
 		case ExitNow:
 			r.Unlock()
-			p.Release()
 			return
 
 		case RouteAndExit:
@@ -657,7 +529,6 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
 			receiver.InjectUDPPacket(p)
 			fp.WasReceived()
 			r.Unlock()
-			p.Release()
 			return
 
 		case KeepRouting:
@@ -670,7 +541,6 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
 		}
 
 		r.Unlock()
-		p.Release()
 	}
 }
 
@@ -771,7 +641,6 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
 		switch e {
 		case ExitNow:
 			r.Unlock()
-			p.Release()
 			return
 
 		case RouteAndExit:
@@ -779,7 +648,6 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
 			receiver.InjectUDPPacket(p)
 			fp.WasReceived()
 			r.Unlock()
-			p.Release()
 			return
 
 		case KeepRouting:
@@ -791,7 +659,6 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
 			panic(fmt.Sprintf("Unknown exitFunc return: %v", e))
 		}
 		r.Unlock()
-		p.Release()
 	}
 }
 
@@ -835,20 +702,19 @@ func (r *R) FlushAll() {
 		}
 		receiver.InjectUDPPacket(p)
 		r.Unlock()
-		p.Release()
 	}
 }
 
 // getControl performs or seeds NAT translation and returns the control for toAddr, p from fields may change
 // This is an internal router function, the caller must hold the lock
 func (r *R) getControl(fromAddr, toAddr netip.AddrPort, p *udp.Packet) *nebula.Control {
-	if newAddr, ok := r.outNat[outNatKey{from: fromAddr, to: toAddr}]; ok {
+	if newAddr, ok := r.outNat[fromAddr.String()+":"+toAddr.String()]; ok {
 		p.From = newAddr
 	}
 
 	c, ok := r.inNat[toAddr]
 	if ok {
-		r.outNat[outNatKey{from: c.GetUDPAddr(), to: fromAddr}] = toAddr
+		r.outNat[c.GetUDPAddr().String()+":"+fromAddr.String()] = toAddr
 		return c
 	}
 

e2e/tunnels_test.go

@@ -355,14 +355,14 @@ func TestCrossStackRelaysWork(t *testing.T) {
 	theirControl.Start()
 
 	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnV6.Addr(), 80, myVpnV6.Addr(), 80, []byte("Hi from me")))
+	myControl.InjectTunUDPPacket(theirVpnV6.Addr(), 80, myVpnV6.Addr(), 80, []byte("Hi from me"))
 
 	p := r.RouteForAllUntilTxTun(theirControl)
 	r.Log("Assert the tunnel works")
 	assertUdpPacket(t, []byte("Hi from me"), p, myVpnV6.Addr(), theirVpnV6.Addr(), 80, 80)
 
 	t.Log("reply?")
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnV6.Addr(), 80, theirVpnV6.Addr(), 80, []byte("Hi from them")))
+	theirControl.InjectTunUDPPacket(myVpnV6.Addr(), 80, theirVpnV6.Addr(), 80, []byte("Hi from them"))
 	p = r.RouteForAllUntilTxTun(myControl)
 	assertUdpPacket(t, []byte("Hi from them"), p, theirVpnV6.Addr(), myVpnV6.Addr(), 80, 80)
 

handshake_manager.go

@@ -971,11 +971,11 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			hostinfo.logger(f.l).Debug("Sending stored packets", "count", len(hh.packetStore))
 		}
-		buf := f.bufAlloc.Acquire()
+		nb := make([]byte, 12, 12)
+		out := make([]byte, mtu)
 		for _, cp := range hh.packetStore {
-			cp.callback(cp.messageType, cp.messageSubType, hostinfo, cp.packet, buf)
+			cp.callback(cp.messageType, cp.messageSubType, hostinfo, cp.packet, nb, out)
 		}
-		f.bufAlloc.Release(buf)
 		f.cachedPacketMetrics.sent.Inc(int64(len(hh.packetStore)))
 	}
 
@@ -1085,9 +1085,7 @@ func (hm *HandshakeManager) sendHandshakeResponse(via ViaSender, msg []byte, hos
 		// We received a valid handshake on this relay, so make sure the relay
 		// state reflects that, in case it had been marked Disestablished.
 		via.relayHI.relayState.UpdateRelayForByIdxState(via.remoteIdx, Established)
-		buf := f.bufAlloc.Acquire()
+		f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false)
-		f.SendVia(via.relayHI, via.relay, msg, buf)
-		f.bufAlloc.Release(buf)
 		f.l.Info("Handshake message sent", append(logFields, "relay", via.relayHI.vpnAddrs[0])...)
 	}
 }
@@ -1104,9 +1102,7 @@ func (hm *HandshakeManager) handleCheckAndCompleteError(err error, existing, hos
 	switch err {
 	case ErrAlreadySeen:
 		if existing.SetRemoteIfPreferred(f.hostMap, via) {
-			buf := f.bufAlloc.Acquire()
+			f.SendMessageToVpnAddr(header.Test, header.TestRequest, hostinfo.vpnAddrs[0], []byte(""), make([]byte, 12, 12), make([]byte, mtu))
-			f.SendMessageToVpnAddr(header.Test, header.TestRequest, hostinfo.vpnAddrs[0], []byte(""), buf)
-			f.bufAlloc.Release(buf)
 		}
 		// Resend the original response. The peer is committed to that response's
 		// ephemeral keys; a freshly-built one would have different keys and break
@@ -1129,9 +1125,7 @@ func (hm *HandshakeManager) handleCheckAndCompleteError(err error, existing, hos
 			"responderIndex", hostinfo.localIndexId,
 			"handshake", hsFields,
 		)
-		buf := f.bufAlloc.Acquire()
+		f.SendMessageToVpnAddr(header.Test, header.TestRequest, hostinfo.vpnAddrs[0], []byte(""), make([]byte, 12, 12), make([]byte, mtu))
-		f.SendMessageToVpnAddr(header.Test, header.TestRequest, hostinfo.vpnAddrs[0], []byte(""), buf)
-		f.bufAlloc.Release(buf)
 
 	case ErrLocalIndexCollision:
 		f.l.Error("Failed to add HostInfo due to localIndex collision",

handshake_manager_test.go

@@ -80,15 +80,15 @@ func testCountTimerWheelEntries(tw *LockingTimerWheel[netip.Addr]) (c int) {
 type mockEncWriter struct {
 }
 
-func (mw *mockEncWriter) SendMessageToVpnAddr(_ header.MessageType, _ header.MessageSubType, _ netip.Addr, _ []byte, _ *WireBuffer) {
+func (mw *mockEncWriter) SendMessageToVpnAddr(_ header.MessageType, _ header.MessageSubType, _ netip.Addr, _, _, _ []byte) {
 	return
 }
 
-func (mw *mockEncWriter) SendVia(_ *HostInfo, _ *Relay, _ []byte, _ *WireBuffer) {
+func (mw *mockEncWriter) SendVia(_ *HostInfo, _ *Relay, _, _, _ []byte, _ bool) {
 	return
 }
 
-func (mw *mockEncWriter) SendMessageToHostInfo(_ header.MessageType, _ header.MessageSubType, _ *HostInfo, _ []byte, _ *WireBuffer) {
+func (mw *mockEncWriter) SendMessageToHostInfo(_ header.MessageType, _ header.MessageSubType, _ *HostInfo, _, _, _ []byte) {
 	return
 }
 

header/header.go

@@ -57,6 +57,8 @@ const (
 const (
 	TestRequest      MessageSubType = 0
 	TestReply        MessageSubType = 1
+	MTUDProbeRequest MessageSubType = 2
+	MTUDProbeReply   MessageSubType = 3
 )
 
 const (
@@ -69,6 +71,8 @@ var ErrHeaderTooShort = errors.New("header is too short")
 var subTypeTestMap = map[MessageSubType]string{
 	TestRequest:      "testRequest",
 	TestReply:        "testReply",
+	MTUDProbeRequest: "mtudProbeRequest",
+	MTUDProbeReply:   "mtudProbeReply",
 }
 
 var subTypeNoneMap = map[MessageSubType]string{0: "none"}

hostmap.go

@@ -308,7 +308,7 @@ type cachedPacket struct {
 	packet         []byte
 }
 
-type packetCallback func(t header.MessageType, st header.MessageSubType, h *HostInfo, p []byte, buf *WireBuffer)
+type packetCallback func(t header.MessageType, st header.MessageSubType, h *HostInfo, p, nb, out []byte)
 
 type cachedPacketMetrics struct {
 	sent    metrics.Counter
@@ -691,7 +691,6 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interfac
 			}
 		}
 
-		buf := ifce.bufAlloc.Acquire()
 		i.remotes.ForEach(preferredRanges, func(addr netip.AddrPort, preferred bool) {
 			if remote.IsValid() && (!addr.IsValid() || !preferred) {
 				return
@@ -699,9 +698,8 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interfac
 
 			// Try to send a test packet to that host, this should
 			// cause it to detect a roaming event and switch remotes
-			ifce.sendTo(header.Test, header.TestRequest, i.ConnectionState, i, addr, []byte(""), buf)
+			ifce.sendTo(header.Test, header.TestRequest, i.ConnectionState, i, addr, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
 		})
-		ifce.bufAlloc.Release(buf)
 	}
 
 	// Re query our lighthouses for new remotes occasionally

inside.go

@@ -8,13 +8,12 @@ import (
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/noiseutil"
 	"github.com/slackhq/nebula/routing"
 )
 
-func (f *Interface) consumeInsidePacket(buf *WireBuffer, q int, localCache firewall.ConntrackCache) {
+func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache) {
-	packet := buf.IPPacket()
+	err := newPacket(packet, false, fwPacket)
-
-	err := newPacket(packet, false, buf.FwPacket)
 	if err != nil {
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			f.l.Debug("Error while validating outbound packet",
@@ -27,12 +26,12 @@ func (f *Interface) consumeInsidePacket(buf *WireBuffer, q int, localCache firew
 
 	// Ignore local broadcast packets
 	if f.dropLocalBroadcast {
-		if f.myBroadcastAddrsTable.Contains(buf.FwPacket.RemoteAddr) {
+		if f.myBroadcastAddrsTable.Contains(fwPacket.RemoteAddr) {
 			return
 		}
 	}
 
-	if f.myVpnAddrsTable.Contains(buf.FwPacket.RemoteAddr) {
+	if f.myVpnAddrsTable.Contains(fwPacket.RemoteAddr) {
 		// Immediately forward packets from self to self.
 		// This should only happen on Darwin-based and FreeBSD hosts, which
 		// routes packets from the Nebula addr to the Nebula addr through the Nebula
@@ -49,20 +48,20 @@ func (f *Interface) consumeInsidePacket(buf *WireBuffer, q int, localCache firew
 	}
 
 	// Ignore multicast packets
-	if f.dropMulticast && buf.FwPacket.RemoteAddr.IsMulticast() {
+	if f.dropMulticast && fwPacket.RemoteAddr.IsMulticast() {
 		return
 	}
 
-	hostinfo, ready := f.getOrHandshakeConsiderRouting(buf.FwPacket, func(hh *HandshakeHostInfo) {
+	hostinfo, ready := f.getOrHandshakeConsiderRouting(fwPacket, func(hh *HandshakeHostInfo) {
 		hh.cachePacket(f.l, header.Message, 0, packet, f.sendMessageNow, f.cachedPacketMetrics)
 	})
 
 	if hostinfo == nil {
-		f.rejectInside(packet, buf.Out, q)
+		f.rejectInside(packet, out, q)
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			f.l.Debug("dropping outbound packet, vpnAddr not in our vpn networks or in unsafe networks",
-				"vpnAddr", buf.FwPacket.RemoteAddr,
+				"vpnAddr", fwPacket.RemoteAddr,
-				"fwPacket", buf.FwPacket,
+				"fwPacket", fwPacket,
 			)
 		}
 		return
@@ -72,15 +71,15 @@ func (f *Interface) consumeInsidePacket(buf *WireBuffer, q int, localCache firew
 		return
 	}
 
-	dropReason := f.firewall.Drop(*buf.FwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
+	dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason == nil {
-		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, buf, q)
+		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
 
 	} else {
-		f.rejectInside(packet, buf.Out, q)
+		f.rejectInside(packet, out, q)
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			hostinfo.logger(f.l).Debug("dropping outbound packet",
-				"fwPacket", buf.FwPacket,
+				"fwPacket", fwPacket,
 				"reason", dropReason,
 			)
 		}
@@ -103,27 +102,27 @@ func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
 	}
 }
 
-func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *HostInfo, scratch []byte, buf *WireBuffer, q int) {
+func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *HostInfo, nb, out []byte, q int) {
 	if !f.firewall.OutSendReject {
 		return
 	}
 
-	rejectIP := iputil.CreateRejectPacket(packet, scratch)
+	out = iputil.CreateRejectPacket(packet, out)
-	if len(rejectIP) == 0 {
+	if len(out) == 0 {
 		return
 	}
 
-	if len(rejectIP) > iputil.MaxRejectPacketSize {
+	if len(out) > iputil.MaxRejectPacketSize {
 		if f.l.Enabled(context.Background(), slog.LevelInfo) {
 			f.l.Info("rejectOutside: packet too big, not sending",
 				"packet", packet,
-				"outPacket", rejectIP,
+				"outPacket", out,
 			)
 		}
 		return
 	}
 
-	f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, rejectIP, buf, q)
+	f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, out, nb, packet, q)
 }
 
 // Handshake will attempt to initiate a tunnel with the provided vpn address. This is a no-op if the tunnel is already established or being established
@@ -216,7 +215,7 @@ func (f *Interface) getOrHandshakeConsiderRouting(fwPacket *firewall.Packet, cac
 
 }
 
-func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p []byte, buf *WireBuffer) {
+func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte) {
 	fp := &firewall.Packet{}
 	err := newPacket(p, false, fp)
 	if err != nil {
@@ -236,12 +235,12 @@ func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubTyp
 		return
 	}
 
-	f.sendNoMetrics(header.Message, st, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, p, buf, 0)
+	f.sendNoMetrics(header.Message, st, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, p, nb, out, 0)
 }
 
 // SendMessageToVpnAddr handles real addr:port lookup and sends to the current best known address for vpnAddr.
 // This function ignores myVpnNetworksTable, and will always attempt to treat the address as a vpnAddr
-func (f *Interface) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p []byte, buf *WireBuffer) {
+func (f *Interface) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p, nb, out []byte) {
 	hostInfo, ready := f.handshakeManager.GetOrHandshake(vpnAddr, func(hh *HandshakeHostInfo) {
 		hh.cachePacket(f.l, t, st, p, f.SendMessageToHostInfo, f.cachedPacketMetrics)
 	})
@@ -259,73 +258,113 @@ func (f *Interface) SendMessageToVpnAddr(t header.MessageType, st header.Message
 		return
 	}
 
-	f.SendMessageToHostInfo(t, st, hostInfo, p, buf)
+	f.SendMessageToHostInfo(t, st, hostInfo, p, nb, out)
 }
 
-func (f *Interface) SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hi *HostInfo, p []byte, buf *WireBuffer) {
+func (f *Interface) SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hi *HostInfo, p, nb, out []byte) {
-	f.send(t, st, hi.ConnectionState, hi, p, buf)
+	f.send(t, st, hi.ConnectionState, hi, p, nb, out)
 }
 
-func (f *Interface) send(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, p []byte, buf *WireBuffer) {
+func (f *Interface) send(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, p, nb, out []byte) {
 	f.messageMetrics.Tx(t, st, 1)
-	f.sendNoMetrics(t, st, ci, hostinfo, netip.AddrPort{}, p, buf, 0)
+	f.sendNoMetrics(t, st, ci, hostinfo, netip.AddrPort{}, p, nb, out, 0)
 }
 
-func (f *Interface) sendTo(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p []byte, buf *WireBuffer) {
+func (f *Interface) sendTo(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte) {
 	f.messageMetrics.Tx(t, st, 1)
-	f.sendNoMetrics(t, st, ci, hostinfo, remote, p, buf, 0)
+	f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out, 0)
 }
 
 // SendVia sends a payload through a Relay tunnel. No authentication or encryption is done
 // to the payload for the ultimate target host, making this a useful method for sending
 // handshake messages to peers through relay tunnels.
-//
+// via is the HostInfo through which the message is relayed.
-// via is the HostInfo through which the message is relayed. ad is staged into
+// ad is the plaintext data to authenticate, but not encrypt
-// the inner-payload slot of buf and then AAD-only sealed under via's key by
+// nb is a buffer used to store the nonce value, re-used for performance reasons.
-// SealRelayInPlace. The sendNoMetrics relay-forward path skips this entry
+// out is a buffer used to store the result of the Encrypt operation
-// point and calls sendViaInPlace directly because its inner ciphertext is
+// q indicates which writer to use to send the packet.
-// already in place from the encrypt step.
+func (f *Interface) SendVia(via *HostInfo,
-func (f *Interface) SendVia(via *HostInfo, relay *Relay, ad []byte, buf *WireBuffer) {
+	relay *Relay,
-	if header.Len+len(ad)+via.ConnectionState.eKey.Overhead() > cap(buf.Out) {
+	ad,
+	nb,
+	out []byte,
+	nocopy bool,
+) {
+	if noiseutil.EncryptLockNeeded {
+		// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
+		via.ConnectionState.writeLock.Lock()
+	}
+	c := via.ConnectionState.messageCounter.Add(1)
+
+	out = header.Encode(out, header.Version, header.Message, header.MessageRelay, relay.RemoteIndex, c)
+	f.connectionManager.Out(via)
+
+	// Authenticate the header and payload, but do not encrypt for this message type.
+	// The payload consists of the inner, unencrypted Nebula header, as well as the end-to-end encrypted payload.
+	if len(out)+len(ad)+via.ConnectionState.eKey.Overhead() > cap(out) {
+		if noiseutil.EncryptLockNeeded {
+			via.ConnectionState.writeLock.Unlock()
+		}
 		via.logger(f.l).Error("SendVia out buffer not large enough for relay",
-			"outCap", cap(buf.Out),
+			"outCap", cap(out),
 			"payloadLen", len(ad),
-			"headerLen", header.Len,
+			"headerLen", len(out),
 			"cipherOverhead", via.ConnectionState.eKey.Overhead(),
 		)
 		return
 	}
-	buf.StageRelayInner(ad)
-	f.sendViaInPlace(via, relay, len(ad), buf)
-}
 
-// sendViaInPlace stamps the outer relay header, AAD-seals over the [outer
+	// The header bytes are written to the 'out' slice; Grow the slice to hold the header and associated data payload.
-// header | inner-already-staged] region, and writes the result to via.remote.
+	offset := len(out)
-// Called from SendVia (after staging ad) and from sendNoMetrics' relay-forward
+	out = out[:offset+len(ad)]
-// path (where the inner ciphertext is already in place from SealForRelay).
+
-func (f *Interface) sendViaInPlace(via *HostInfo, relay *Relay, innerLen int, buf *WireBuffer) {
+	// In one call path, the associated data _is_ already stored in out. In other call paths, the associated data must
-	f.connectionManager.Out(via)
+	// be copied into 'out'.
-	out, err := buf.SealRelayInPlace(via.ConnectionState, relay.RemoteIndex, innerLen)
+	if !nocopy {
+		copy(out[offset:], ad)
+	}
+
+	var err error
+	out, err = via.ConnectionState.eKey.EncryptDanger(out, out, nil, c, nb)
+	if noiseutil.EncryptLockNeeded {
+		via.ConnectionState.writeLock.Unlock()
+	}
 	if err != nil {
 		via.logger(f.l).Info("Failed to EncryptDanger in sendVia", "error", err)
 		return
 	}
-	if err := f.writers[0].WriteTo(out, via.remote); err != nil {
+	err = f.writers[0].WriteTo(out, via.remote)
+	if err != nil {
 		via.logger(f.l).Info("Failed to WriteTo in sendVia", "error", err)
 	}
 	f.connectionManager.RelayUsed(relay.LocalIndex)
 }
 
-// sendNoMetrics encrypts and writes one outbound nebula packet (data, control,
+func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte, q int) {
-// lighthouse, etc) using buf as the per-call wire scratch. When the hostinfo
-// has no direct remote we encrypt into the relay-reserved slot via
-// SealForRelay so sendViaInPlace can wrap it without an extra copy.
-func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p []byte, buf *WireBuffer, q int) {
 	if ci.eKey == nil {
 		return
 	}
 	useRelay := !remote.IsValid() && !hostinfo.remote.IsValid()
+	fullOut := out
 
+	if useRelay {
+		if len(out) < header.Len {
+			// out always has a capacity of mtu, but not always a length greater than the header.Len.
+			// Grow it to make sure the next operation works.
+			out = out[:header.Len]
+		}
+		// Save a header's worth of data at the front of the 'out' buffer.
+		out = out[header.Len:]
+	}
+
+	if noiseutil.EncryptLockNeeded {
+		// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
+		ci.writeLock.Lock()
+	}
+	c := ci.messageCounter.Add(1)
+
+	//l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p)
+	out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
 	f.connectionManager.Out(hostinfo)
 
 	// Query our LH if we haven't since the last time we've been rebound, this will cause the remote to punch against
@@ -342,42 +381,50 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 		}
 	}
 
-	var out []byte
 	var err error
-	if useRelay {
+	out, err = ci.eKey.EncryptDanger(out, out, p, c, nb)
-		out, err = buf.SealForRelay(ci, t, st, hostinfo.remoteIndexId, p)
+	if noiseutil.EncryptLockNeeded {
-	} else {
+		ci.writeLock.Unlock()
-		out, err = buf.Seal(ci, t, st, hostinfo.remoteIndexId, p)
 	}
 	if err != nil {
 		hostinfo.logger(f.l).Error("Failed to encrypt outgoing packet",
 			"error", err,
 			"udpAddr", remote,
+			"counter", c,
+			"attemptedCounter", c,
 		)
 		return
 	}
 
-	switch {
+	if remote.IsValid() {
-	case remote.IsValid():
+		err = f.writers[q].WriteTo(out, remote)
-		if err := f.writers[q].WriteTo(out, remote); err != nil {
+		if err != nil {
-			hostinfo.logger(f.l).Error("Failed to write outgoing packet", "error", err, "udpAddr", remote)
+			hostinfo.logger(f.l).Error("Failed to write outgoing packet",
+				"error", err,
+				"udpAddr", remote,
+			)
 		}
-	case hostinfo.remote.IsValid():
+	} else if hostinfo.remote.IsValid() {
-		if err := f.writers[q].WriteTo(out, hostinfo.remote); err != nil {
+		err = f.writers[q].WriteTo(out, hostinfo.remote)
-			hostinfo.logger(f.l).Error("Failed to write outgoing packet", "error", err, "udpAddr", hostinfo.remote)
+		if err != nil {
+			hostinfo.logger(f.l).Error("Failed to write outgoing packet",
+				"error", err,
+				"udpAddr", remote,
+			)
 		}
-	default:
+	} else {
-		// SealForRelay placed the inner ciphertext at buf.Out[header.Len:],
+		// Try to send via a relay
-		// so sendViaInPlace can wrap it with the outer relay header without
-		// an extra copy.
 		for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
 			relayHostInfo, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
 			if err != nil {
 				hostinfo.relayState.DeleteRelay(relayIP)
-				hostinfo.logger(f.l).Info("sendNoMetrics failed to find HostInfo", "relay", relayIP, "error", err)
+				hostinfo.logger(f.l).Info("sendNoMetrics failed to find HostInfo",
+					"relay", relayIP,
+					"error", err,
+				)
 				continue
 			}
-			f.sendViaInPlace(relayHostInfo, relay, len(out), buf)
+			f.SendVia(relayHostInfo, relay, out, nb, fullOut[:header.Len+len(out)], true)
 			break
 		}
 	}

interface.go

@@ -34,6 +34,7 @@ type InterfaceConfig struct {
 	HandshakeManager   *HandshakeManager
 	lightHouse         *LightHouse
 	connectionManager  *connectionManager
+	pmtudManager       *pmtudManager
 	DropLocalBroadcast bool
 	DropMulticast      bool
 	routines           int
@@ -57,6 +58,7 @@ type Interface struct {
 	pki                   *PKI
 	firewall              *Firewall
 	connectionManager     *connectionManager
+	pmtudManager          *pmtudManager
 	handshakeManager      *HandshakeManager
 	dnsServer             *dnsServer
 	createTime            time.Time
@@ -101,19 +103,19 @@ type Interface struct {
 	messageMetrics      *MessageMetrics
 	cachedPacketMetrics *cachedPacketMetrics
 
-	// bufAlloc hands out reusable WireBuffers sized for this interface's
-	// inside Device. All buf consumers (hot-path data-plane goroutines,
-	// long-lived workers, and cold callers) acquire from here so sizing
-	// is centralized and consistent. Long-lived owners just don't release.
-	bufAlloc WireBufferAllocator
-
 	l *slog.Logger
 }
 
 type EncWriter interface {
-	SendVia(via *HostInfo, relay *Relay, ad []byte, buf *WireBuffer)
+	SendVia(via *HostInfo,
-	SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p []byte, buf *WireBuffer)
+		relay *Relay,
-	SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p []byte, buf *WireBuffer)
+		ad,
+		nb,
+		out []byte,
+		nocopy bool,
+	)
+	SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p, nb, out []byte)
+	SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte)
 	Handshake(vpnAddr netip.Addr)
 	GetHostInfo(vpnAddr netip.Addr) *HostInfo
 	GetCertState() *CertState
@@ -195,6 +197,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 		myBroadcastAddrsTable: cs.myVpnBroadcastAddrsTable,
 		relayManager:          c.relayManager,
 		connectionManager:     c.connectionManager,
+		pmtudManager:          c.pmtudManager,
 		conntrackCacheTimeout: c.ConntrackCacheTimeout,
 
 		metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)),
@@ -204,8 +207,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 			dropped: metrics.GetOrRegisterCounter("hostinfo.cached_packets.dropped", nil),
 		},
 
-		bufAlloc: NewWireBufferPool(mtu, c.Inside.TunPrefixLen()),
-
 		l: c.l,
 	}
 
@@ -214,6 +215,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 	ifce.reQueryWait.Store(int64(c.reQueryWait))
 
 	ifce.connectionManager.intf = ifce
+	ifce.pmtudManager.intf = ifce
 
 	return ifce, nil
 }
@@ -313,11 +315,13 @@ func (f *Interface) listenOut(i int) {
 
 	ctCache := firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout)
 	lhh := f.lightHouse.NewRequestHandler()
-	// Long-lived per-receive-goroutine buf; never released back to the pool.
+	plaintext := make([]byte, udp.MTU)
-	buf := f.bufAlloc.Acquire()
+	h := &header.H{}
+	fwPacket := &firewall.Packet{}
+	nb := make([]byte, 12, 12)
 
 	err := li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) {
-		f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, buf, payload, lhh, i, ctCache.Get())
+		f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, plaintext[:0], payload, h, fwPacket, lhh, nb, i, ctCache.Get())
 	})
 
 	if err != nil && !f.closed.Load() {
@@ -329,12 +333,15 @@ func (f *Interface) listenOut(i int) {
 }
 
 func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
-	// Long-lived per-tun-reader buf; never released back to the pool.
+	packet := make([]byte, mtu)
-	buf := f.bufAlloc.Acquire()
+	out := make([]byte, mtu)
+	fwPacket := &firewall.Packet{}
+	nb := make([]byte, 12, 12)
+
 	conntrackCache := firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout)
 
 	for {
-		_, err := buf.ReadIPFromTUN(reader)
+		n, err := reader.Read(packet)
 		if err != nil {
 			if !f.closed.Load() {
 				f.l.Error("Error while reading outbound packet, closing", "error", err, "reader", i)
@@ -343,7 +350,7 @@ func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
 			break
 		}
 
-		f.consumeInsidePacket(buf, i, conntrackCache.Get())
+		f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get())
 	}
 
 	f.l.Debug("overlay reader is done", "reader", i)

lighthouse.go

@@ -63,10 +63,6 @@ type LightHouse struct {
 	interval     atomic.Int64
 	updateCancel context.CancelFunc
 	ifce         EncWriter
-	// bufAlloc lets the lighthouse query/update workers, request handlers
-	// and punchback goroutines acquire correctly sized WireBuffers from
-	// the same pool as the data plane. Set by main.go alongside ifce.
-	bufAlloc   WireBufferAllocator
 	nebulaPort   uint32 // 32 bits because protobuf does not have a uint16
 
 	advertiseAddrs atomic.Pointer[[]netip.AddrPort]
@@ -113,10 +109,6 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
 		punchy:             p,
 		updateTrigger:      make(chan struct{}, 1),
 		queryChan:          make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)),
-		// Default to a no-prefix pool so the query/update workers and
-		// request handlers have a working WireBufferAllocator before
-		// main.go wires up the real one from the Interface.
-		bufAlloc: NewWireBufferPool(mtu, 0),
 		l:                  l,
 	}
 	lighthouses := make([]netip.Addr, 0)
@@ -766,22 +758,21 @@ func (lh *LightHouse) startQueryWorker() {
 	}
 
 	go func() {
-		// Long-lived per-worker WireBuffer; reused for every lighthouse query
+		nb := make([]byte, 12, 12)
-		// this worker issues for the life of the goroutine.
+		out := make([]byte, mtu)
-		buf := lh.bufAlloc.Acquire()
 
 		for {
 			select {
 			case <-lh.ctx.Done():
 				return
 			case addr := <-lh.queryChan:
-				lh.innerQueryServer(addr, buf)
+				lh.innerQueryServer(addr, nb, out)
 			}
 		}
 	}()
 }
 
-func (lh *LightHouse) innerQueryServer(addr netip.Addr, buf *WireBuffer) {
+func (lh *LightHouse) innerQueryServer(addr netip.Addr, nb, out []byte) {
 	if lh.IsLighthouseAddr(addr) {
 		return
 	}
@@ -830,7 +821,7 @@ func (lh *LightHouse) innerQueryServer(addr netip.Addr, buf *WireBuffer) {
 				}
 			}
 
-			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v1Query, buf)
+			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v1Query, nb, out)
 			queried++
 
 		} else if v == cert.Version2 {
@@ -849,7 +840,7 @@ func (lh *LightHouse) innerQueryServer(addr netip.Addr, buf *WireBuffer) {
 				}
 			}
 
-			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v2Query, buf)
+			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v2Query, nb, out)
 			queried++
 
 		} else {
@@ -878,12 +869,8 @@ func (lh *LightHouse) StartUpdateWorker() {
 	go func() {
 		defer clockSource.Stop()
 
-		// Long-lived per-worker WireBuffer; reused across every periodic
-		// update for the life of this goroutine.
-		buf := lh.bufAlloc.Acquire()
-
 		for {
-			lh.sendUpdate(buf)
+			lh.SendUpdate()
 
 			select {
 			case <-updateCtx.Done():
@@ -897,15 +884,6 @@ func (lh *LightHouse) StartUpdateWorker() {
 	}()
 }
 
-// SendUpdate is the public entry point that triggers a one-shot lighthouse
-// update outside the worker loop (e.g. tests or reload paths). It allocates
-// its own WireBuffer since callers don't already own one.
-func (lh *LightHouse) SendUpdate() {
-	buf := lh.bufAlloc.Acquire()
-	defer lh.bufAlloc.Release(buf)
-	lh.sendUpdate(buf)
-}
-
 // TriggerUpdate requests an immediate lighthouse update. This is a non-blocking
 // operation intended to be called after a handshake completes with a lighthouse,
 // so the lighthouse has our current addresses without waiting for the next
@@ -917,7 +895,7 @@ func (lh *LightHouse) TriggerUpdate() {
 	}
 }
 
-func (lh *LightHouse) sendUpdate(buf *WireBuffer) {
+func (lh *LightHouse) SendUpdate() {
 	var v4 []*V4AddrPort
 	var v6 []*V6AddrPort
 
@@ -943,6 +921,9 @@ func (lh *LightHouse) sendUpdate(buf *WireBuffer) {
 		}
 	}
 
+	nb := make([]byte, 12, 12)
+	out := make([]byte, mtu)
+
 	var v1Update, v2Update []byte
 	var err error
 	updated := 0
@@ -993,7 +974,7 @@ func (lh *LightHouse) sendUpdate(buf *WireBuffer) {
 				}
 			}
 
-			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v1Update, buf)
+			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v1Update, nb, out)
 			updated++
 
 		} else if v == cert.Version2 {
@@ -1022,7 +1003,7 @@ func (lh *LightHouse) sendUpdate(buf *WireBuffer) {
 				}
 			}
 
-			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v2Update, buf)
+			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v2Update, nb, out)
 			updated++
 
 		} else {
@@ -1039,10 +1020,8 @@ func (lh *LightHouse) sendUpdate(buf *WireBuffer) {
 
 type LightHouseHandler struct {
 	lh   *LightHouse
-	// buf is the long-lived per-handler wire scratch. NewRequestHandler is
+	nb   []byte
-	// called once per data-plane receive goroutine, so buf is owned by that
+	out  []byte
-	// goroutine and reused for every lighthouse send the handler issues.
-	buf  *WireBuffer
 	pb   []byte
 	meta *NebulaMeta
 	l    *slog.Logger
@@ -1051,7 +1030,8 @@ type LightHouseHandler struct {
 func (lh *LightHouse) NewRequestHandler() *LightHouseHandler {
 	lhh := &LightHouseHandler{
 		lh:  lh,
-		buf: lh.bufAlloc.Acquire(),
+		nb:  make([]byte, 12, 12),
+		out: make([]byte, mtu),
 		l:   lh.l,
 		pb:  make([]byte, mtu),
 
@@ -1188,7 +1168,7 @@ func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, fromVpnAddrs []neti
 	}
 
 	lhh.lh.metricTx(NebulaMeta_HostQueryReply, 1)
-	w.SendMessageToVpnAddr(header.LightHouse, 0, fromVpnAddrs[0], lhh.pb[:ln], lhh.buf)
+	w.SendMessageToVpnAddr(header.LightHouse, 0, fromVpnAddrs[0], lhh.pb[:ln], lhh.nb, lhh.out[:0])
 
 	lhh.sendHostPunchNotification(n, fromVpnAddrs, queryVpnAddr, w)
 }
@@ -1248,7 +1228,7 @@ func (lhh *LightHouseHandler) sendHostPunchNotification(n *NebulaMeta, fromVpnAd
 	}
 
 	lhh.lh.metricTx(NebulaMeta_HostPunchNotification, 1)
-	w.SendMessageToVpnAddr(header.LightHouse, 0, punchNotifDest, lhh.pb[:ln], lhh.buf)
+	w.SendMessageToVpnAddr(header.LightHouse, 0, punchNotifDest, lhh.pb[:ln], lhh.nb, lhh.out[:0])
 }
 
 func (lhh *LightHouseHandler) coalesceAnswers(v cert.Version, c *cache, n *NebulaMeta) {
@@ -1405,7 +1385,7 @@ func (lhh *LightHouseHandler) handleHostUpdateNotification(n *NebulaMeta, fromVp
 	}
 
 	lhh.lh.metricTx(NebulaMeta_HostUpdateNotificationAck, 1)
-	w.SendMessageToVpnAddr(header.LightHouse, 0, fromVpnAddrs[0], lhh.pb[:ln], lhh.buf)
+	w.SendMessageToVpnAddr(header.LightHouse, 0, fromVpnAddrs[0], lhh.pb[:ln], lhh.nb, lhh.out[:0])
 }
 
 func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, w EncWriter) {
@@ -1472,13 +1452,10 @@ func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, fromVpn
 					"vpnAddr", detailsVpnAddr,
 				)
 			}
-			// We acquire and release a fresh buf within this goroutine so it
+			//NOTE: we have to allocate a new output buffer here since we are spawning a new goroutine
-			// returns to the pool once the punchback send completes. We
+			// for each punchBack packet. We should move this into a timerwheel or a single goroutine
-			// should move this into a timerwheel or a single goroutine
 			// managed by a channel.
-			pbuf := lhh.lh.bufAlloc.Acquire()
+			w.SendMessageToVpnAddr(header.Test, header.TestRequest, detailsVpnAddr, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
-			defer lhh.lh.bufAlloc.Release(pbuf)
-			w.SendMessageToVpnAddr(header.Test, header.TestRequest, detailsVpnAddr, []byte(""), pbuf)
 		}()
 	}
 }

lighthouse_test.go

@@ -372,12 +372,12 @@ type testEncWriter struct {
 	protocolVersion cert.Version
 }
 
-func (tw *testEncWriter) SendVia(via *HostInfo, relay *Relay, ad []byte, buf *WireBuffer) {
+func (tw *testEncWriter) SendVia(via *HostInfo, relay *Relay, ad, nb, out []byte, nocopy bool) {
 }
 func (tw *testEncWriter) Handshake(vpnIp netip.Addr) {
 }
 
-func (tw *testEncWriter) SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p []byte, _ *WireBuffer) {
+func (tw *testEncWriter) SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, _, _ []byte) {
 	msg := &NebulaMeta{}
 	err := msg.Unmarshal(p)
 	if tw.metaFilter == nil || msg.Type == *tw.metaFilter {
@@ -394,7 +394,7 @@ func (tw *testEncWriter) SendMessageToHostInfo(t header.MessageType, st header.M
 	}
 }
 
-func (tw *testEncWriter) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnIp netip.Addr, p []byte, _ *WireBuffer) {
+func (tw *testEncWriter) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnIp netip.Addr, p, _, _ []byte) {
 	msg := &NebulaMeta{}
 	err := msg.Unmarshal(p)
 	if tw.metaFilter == nil || msg.Type == *tw.metaFilter {

main.go

@@ -172,6 +172,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 	hostMap := NewHostMapFromConfig(l, c)
 	punchy := NewPunchyFromConfig(l, c)
 	connManager := newConnectionManagerFromConfig(l, c, hostMap, punchy)
+	pmtudMgr := newPMTUDManagerFromConfig(l, c, tun)
 	lightHouse, err := NewLightHouseFromConfig(ctx, l, c, pki.getCertState(), udpConns[0], punchy)
 	if err != nil {
 		return nil, util.ContextualizeIfNeeded("Failed to initialize lighthouse handler", err)
@@ -208,6 +209,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		DnsServer:             ds,
 		HandshakeManager:      handshakeManager,
 		connectionManager:     connManager,
+		pmtudManager:          pmtudMgr,
 		lightHouse:            lightHouse,
 		tryPromoteEvery:       c.GetUint32("counters.try_promote", defaultPromoteEvery),
 		reQueryEvery:          c.GetUint32("counters.requery_every_packets", defaultReQueryEvery),
@@ -232,7 +234,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 
 		ifce.writers = udpConns
 		lightHouse.ifce = ifce
-		lightHouse.bufAlloc = ifce.bufAlloc
 
 		ifce.RegisterConfigChangeCallbacks(c)
 		ifce.reloadDisconnectInvalid(c)
@@ -267,6 +268,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		dnsStart:               ds.Start,
 		lighthouseStart:        lightHouse.StartUpdateWorker,
 		connectionManagerStart: connManager.Start,
+		pmtudManagerStart:      pmtudMgr.Start,
 	}, nil
 }
 

noise.go

@@ -14,19 +14,6 @@ type endianness interface {
 
 var noiseEndianness endianness = binary.BigEndian
 
-// NonceSize is the AEAD nonce length used by all ciphers nebula supports
-// today (AES-GCM and ChaCha20-Poly1305 both use 96-bit nonces). Encrypt-
-// and DecryptDanger lay out the nonce as 4 zero bytes followed by an 8-byte
-// big-endian counter; if a future cipher with a different nonce size is
-// added, this constant and those layouts must change together.
-const NonceSize = 12
-
-// AEADOverhead is the AEAD authentication tag length the ciphers nebula
-// supports append to ciphertext. Both AES-GCM and ChaCha20-Poly1305 use
-// 128-bit tags. NebulaCipherState.Overhead() returns this dynamically from
-// the cipher; the constant is for sizing buffers at construction time.
-const AEADOverhead = 16
-
 type NebulaCipherState struct {
 	c cipher.AEAD
 }

outside.go

@@ -20,8 +20,7 @@ const (
 	minFwPacketLen = 4
 )
 
-func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []byte, lhf *LightHouseHandler, q int, localCache firewall.ConntrackCache) {
+func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
-	h := buf.H
 	err := h.Parse(packet)
 	if err != nil {
 		// Hole punch packets are 0 or 1 byte big, so lets ignore printing those errors
@@ -66,7 +65,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 
 		switch h.Subtype {
 		case header.MessageNone:
-			if !f.decryptToTun(hostinfo, h.MessageCounter, buf, packet, q, localCache) {
+			if !f.decryptToTun(hostinfo, h.MessageCounter, out, packet, fwPacket, nb, q, localCache) {
 				return
 			}
 		case header.MessageRelay:
@@ -77,9 +76,8 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 			// which will gracefully fail in the DecryptDanger call.
 			signedPayload := packet[:len(packet)-hostinfo.ConnectionState.dKey.Overhead()]
 			signatureValue := packet[len(packet)-hostinfo.ConnectionState.dKey.Overhead():]
-			// AAD-only validation: passing dst=nil since there's no plaintext
+			out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, signedPayload, signatureValue, h.MessageCounter, nb)
-			// to recover (ciphertext is just the trailing AEAD tag).
+			if err != nil {
-			if _, err = hostinfo.ConnectionState.dKey.DecryptDanger(nil, signedPayload, signatureValue, h.MessageCounter, buf.NB); err != nil {
 				return
 			}
 			// Successfully validated the thing. Get rid of the Relay header.
@@ -112,8 +110,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 					relay:     relay,
 					IsRelayed: true,
 				}
-				buf.Reset()
+				f.readOutsidePackets(via, out[:0], signedPayload, h, fwPacket, lhf, nb, q, localCache)
-				f.readOutsidePackets(via, buf, signedPayload, lhf, q, localCache)
 				return
 			case ForwardingType:
 				// Find the target HostInfo relay object
@@ -133,7 +130,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 					case ForwardingType:
 						// Forward this packet through the relay tunnel
 						// Find the target HostInfo
-						f.SendVia(targetHI, targetRelay, signedPayload, buf)
+						f.SendVia(targetHI, targetRelay, signedPayload, nb, out, false)
 						return
 					case TerminalType:
 						hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
@@ -155,7 +152,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 			return
 		}
 
-		d, err := f.decrypt(hostinfo, h.MessageCounter, buf, packet, h)
+		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
 			hostinfo.logger(f.l).Error("Failed to decrypt lighthouse packet",
 				"error", err,
@@ -176,7 +173,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 			return
 		}
 
-		d, err := f.decrypt(hostinfo, h.MessageCounter, buf, packet, h)
+		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
 			hostinfo.logger(f.l).Error("Failed to decrypt test packet",
 				"error", err,
@@ -186,11 +183,20 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 			return
 		}
 
-		if h.Subtype == header.TestRequest {
+		switch h.Subtype {
+		case header.TestRequest:
 			// This testRequest might be from TryPromoteBest, so we should roam
-			// to the new IP address before responding.
+			// to the new IP address before responding
 			f.handleHostRoaming(hostinfo, via)
-			f.send(header.Test, header.TestReply, ci, hostinfo, d, buf)
+			f.send(header.Test, header.TestReply, ci, hostinfo, d, nb, out)
+		case header.MTUDProbeRequest:
+			// Reply with just the 8-byte ack header so the reverse path doesn't have to
+			// carry the full probe size; we only verify the forward direction.
+			if len(d) >= 8 {
+				f.send(header.Test, header.MTUDProbeReply, ci, hostinfo, d[:8], nb, out)
+			}
+		case header.MTUDProbeReply:
+			f.pmtudManager.HandleReply(hostinfo.localIndexId, d)
 		}
 
 		// Fallthrough to the bottom to record incoming traffic
@@ -213,7 +219,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 		if !f.handleEncrypted(ci, via, h) {
 			return
 		}
-		_, err = f.decrypt(hostinfo, h.MessageCounter, buf, packet, h)
+		_, err = f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
 			hostinfo.logger(f.l).Error("Failed to decrypt CloseTunnel packet",
 				"error", err,
@@ -233,7 +239,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 			return
 		}
 
-		d, err := f.decrypt(hostinfo, h.MessageCounter, buf, packet, h)
+		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
 			hostinfo.logger(f.l).Error("Failed to decrypt Control packet",
 				"error", err,
@@ -260,6 +266,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []
 
 // closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote
 func (f *Interface) closeTunnel(hostInfo *HostInfo) {
+	f.pmtudManager.OnTunnelDown(hostInfo)
 	final := f.hostMap.DeleteHostInfo(hostInfo)
 	if final {
 		// We no longer have any tunnels with this vpn addr, clear learned lighthouse state to lower memory usage
@@ -269,9 +276,7 @@ func (f *Interface) closeTunnel(hostInfo *HostInfo) {
 
 // sendCloseTunnel is a helper function to send a proper close tunnel packet to a remote
 func (f *Interface) sendCloseTunnel(h *HostInfo) {
-	buf := f.bufAlloc.Acquire()
+	f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, make([]byte, 12, 12), make([]byte, mtu))
-	defer f.bufAlloc.Release(buf)
-	f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, buf)
 }
 
 func (f *Interface) handleHostRoaming(hostinfo *HostInfo, via ViaSender) {
@@ -301,6 +306,7 @@ func (f *Interface) handleHostRoaming(hostinfo *HostInfo, via ViaSender) {
 		hostinfo.lastRoam = time.Now()
 		hostinfo.lastRoamRemote = hostinfo.remote
 		hostinfo.SetRemote(via.UdpAddr)
+		f.pmtudManager.OnRoam(hostinfo)
 	}
 
 }
@@ -520,8 +526,9 @@ func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
 	return nil
 }
 
-func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, buf *WireBuffer, packet []byte, h *header.H) ([]byte, error) {
+func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, out []byte, packet []byte, h *header.H, nb []byte) ([]byte, error) {
-	plaintext, err := buf.DecryptForHandler(hostinfo.ConnectionState, packet, mc)
+	var err error
+	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], mc, nb)
 	if err != nil {
 		return nil, err
 	}
@@ -533,41 +540,42 @@ func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, buf *WireBuffer, pack
 		return nil, errors.New("out of window packet")
 	}
 
-	return plaintext, nil
+	return out, nil
 }
 
-func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, buf *WireBuffer, packet []byte, q int, localCache firewall.ConntrackCache) bool {
+func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache) bool {
-	if err := buf.DecryptDatagram(hostinfo.ConnectionState, packet, messageCounter); err != nil {
+	var err error
+
+	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
+	if err != nil {
 		hostinfo.logger(f.l).Error("Failed to decrypt packet", "error", err)
 		return false
 	}
 
-	ipPacket := buf.IPPacket()
+	err = newPacket(out, true, fwPacket)
-	if err := newPacket(ipPacket, true, buf.FwPacket); err != nil {
+	if err != nil {
 		hostinfo.logger(f.l).Warn("Error while validating inbound packet",
 			"error", err,
-			"packet", ipPacket,
+			"packet", out,
 		)
 		return false
 	}
 
 	if !hostinfo.ConnectionState.window.Update(f.l, messageCounter) {
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			hostinfo.logger(f.l).Debug("dropping out of window packet", "fwPacket", buf.FwPacket)
+			hostinfo.logger(f.l).Debug("dropping out of window packet", "fwPacket", fwPacket)
 		}
 		return false
 	}
 
-	dropReason := f.firewall.Drop(*buf.FwPacket, true, hostinfo, f.pki.GetCAPool(), localCache)
+	dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason != nil {
-		// NOTE: We hand `packet` (the original UDP ciphertext we already
+		// NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore
-		// decrypted from) as the reject-IP scratch since we no longer
+		// This gives us a buffer to build the reject packet in
-		// need its ciphertext, and it's disjoint from buf.Out where
+		f.rejectOutside(out, hostinfo.ConnectionState, hostinfo, nb, packet, q)
-		// sendNoMetrics will encrypt the wire packet.
-		f.rejectOutside(ipPacket, hostinfo.ConnectionState, hostinfo, packet, buf, q)
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			hostinfo.logger(f.l).Debug("dropping inbound packet",
-				"fwPacket", buf.FwPacket,
+				"fwPacket", fwPacket,
 				"reason", dropReason,
 			)
 		}
@@ -575,7 +583,8 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, buf
 	}
 
 	f.connectionManager.In(hostinfo)
-	if _, err := buf.WriteIPToTUN(f.readers[q]); err != nil {
+	_, err = f.readers[q].Write(out)
+	if err != nil {
 		f.l.Error("Failed to write to tun", "error", err)
 	}
 	return true

overlay/device.go

@@ -15,7 +15,13 @@ type Device interface {
 	RoutesFor(netip.Addr) routing.Gateways
 	SupportsMultiqueue() bool
 	NewMultiQueueReader() (io.ReadWriteCloser, error)
-	// TunPrefixLen reports the number of bytes the device prepends to every IP packet on the wire.
+	// SupportsPerPeerMTU reports whether SetPeerMTU is implemented for real on
-	// Currently only non zero for the BSD tun devices.
+	// this platform. PMTUD requires this; the manager will refuse to enable when
-	TunPrefixLen() int
+	// false even if the operator set tun.max_mtu, because a discovered MTU we
+	// can't actually install does the operator no good.
+	SupportsPerPeerMTU() bool
+	// SetPeerMTU installs a per-peer MTU on the routing table so the kernel will
+	// surface PTB / EMSGSIZE for inside packets to that peer that would exceed mtu.
+	// Pass mtu=0 to remove the override and let the device default apply.
+	SetPeerMTU(addr netip.Addr, mtu int) error
 }

overlay/overlaytest/noop.go

@@ -39,6 +39,14 @@ func (NoopTun) Write([]byte) (int, error) {
 	return 0, nil
 }
 
+func (NoopTun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (NoopTun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (NoopTun) SupportsMultiqueue() bool {
 	return false
 }
@@ -50,5 +58,3 @@ func (NoopTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 func (NoopTun) Close() error {
 	return nil
 }
-
-func (NoopTun) TunPrefixLen() int { return 0 }

overlay/tun_android.go

@@ -95,6 +95,14 @@ func (t *tun) Name() string {
 	return "android"
 }
 
+func (t *tun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *tun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *tun) SupportsMultiqueue() bool {
 	return false
 }
@@ -102,5 +110,3 @@ func (t *tun) SupportsMultiqueue() bool {
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for android")
 }
-
-func (t *tun) TunPrefixLen() int { return 0 }

overlay/tun_bsd.go

@@ -1,29 +0,0 @@
-//go:build (darwin || ios || freebsd || openbsd || netbsd) && !e2e_testing
-
-package overlay
-
-import (
-	"fmt"
-	"syscall"
-)
-
-// StampTunPrefix writes the 4-byte AF_INET / AF_INET6 protocol-family marker into buf[0:4] in place,
-// picking the family from the first byte of the IP packet at buf[4].
-func StampTunPrefix(buf []byte) error {
-	if len(buf) < 5 {
-		return fmt.Errorf("tun write buffer too small for prefix")
-	}
-	ipVer := buf[4] >> 4
-	buf[0] = 0
-	buf[1] = 0
-	buf[2] = 0
-	switch ipVer {
-	case 4:
-		buf[3] = syscall.AF_INET
-	case 6:
-		buf[3] = syscall.AF_INET6
-	default:
-		return fmt.Errorf("unable to determine IP version from packet")
-	}
-	return nil
-}

overlay/tun_darwin.go

@@ -11,6 +11,7 @@ import (
 	"net/netip"
 	"os"
 	"sync/atomic"
+	"syscall"
 	"unsafe"
 
 	"github.com/gaissmai/bart"
@@ -30,6 +31,9 @@ type tun struct {
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
 	linkAddr    *netroute.LinkAddr
 	l           *slog.Logger
+
+	// cache out buffer since we need to prepend 4 bytes for tun metadata
+	out []byte
 }
 
 type ifReq struct {
@@ -498,6 +502,44 @@ func delRoute(prefix netip.Prefix, gateway netroute.Addr) error {
 	return nil
 }
 
+func (t *tun) Read(to []byte) (int, error) {
+	buf := make([]byte, len(to)+4)
+
+	n, err := t.ReadWriteCloser.Read(buf)
+
+	copy(to, buf[4:])
+	return n - 4, err
+}
+
+// Write is only valid for single threaded use
+func (t *tun) Write(from []byte) (int, error) {
+	buf := t.out
+	if cap(buf) < len(from)+4 {
+		buf = make([]byte, len(from)+4)
+		t.out = buf
+	}
+	buf = buf[:len(from)+4]
+
+	if len(from) == 0 {
+		return 0, syscall.EIO
+	}
+
+	// Determine the IP Family for the NULL L2 Header
+	ipVer := from[0] >> 4
+	if ipVer == 4 {
+		buf[3] = syscall.AF_INET
+	} else if ipVer == 6 {
+		buf[3] = syscall.AF_INET6
+	} else {
+		return 0, fmt.Errorf("unable to determine IP version from packet")
+	}
+
+	copy(buf[4:], from)
+
+	n, err := t.ReadWriteCloser.Write(buf)
+	return n - 4, err
+}
+
 func (t *tun) Networks() []netip.Prefix {
 	return t.vpnNetworks
 }
@@ -506,6 +548,14 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
+func (t *tun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *tun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *tun) SupportsMultiqueue() bool {
 	return false
 }
@@ -513,7 +563,3 @@ func (t *tun) SupportsMultiqueue() bool {
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
 }
-
-// TunPrefixLen reports the 4-byte BSD AF_INET / AF_INET6 protocol-family
-// marker the kernel prepends on read and expects on write.
-func (t *tun) TunPrefixLen() int { return 4 }

overlay/tun_disabled.go

@@ -106,6 +106,14 @@ func (t *disabledTun) Write(b []byte) (int, error) {
 	return len(b), nil
 }
 
+func (t *disabledTun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *disabledTun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *disabledTun) SupportsMultiqueue() bool {
 	return true
 }
@@ -136,5 +144,3 @@ func (p prettyPacket) String() string {
 
 	return s.String()
 }
-
-func (t *disabledTun) TunPrefixLen() int { return 0 }

overlay/tun_freebsd.go

@@ -158,43 +158,74 @@ func (t *tun) blockOnWrite() error {
 }
 
 func (t *tun) Read(to []byte) (int, error) {
-	for {
+	// first 4 bytes is protocol family, in network byte order
-		n, err := unix.Read(t.fd, to)
+	var head [4]byte
-		if err == nil {
+	iovecs := [2]syscall.Iovec{
-			return n, nil
+		{&head[0], 4},
+		{&to[0], uint64(len(to))},
 	}
-		switch err {
+	for {
+		n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
+		if errno == 0 {
+			bytesRead := int(n)
+			if bytesRead < 4 {
+				return 0, nil
+			}
+			return bytesRead - 4, nil
+		}
+		switch errno {
 		case unix.EAGAIN:
-			if berr := t.blockOnRead(); berr != nil {
+			if err := t.blockOnRead(); err != nil {
-				return 0, berr
+				return 0, err
 			}
 		case unix.EINTR:
 			// retry
 		case unix.EBADF:
 			return 0, os.ErrClosed
 		default:
-			return 0, err
+			return 0, errno
 		}
 	}
 }
 
+// Write is only valid for single threaded use
 func (t *tun) Write(from []byte) (int, error) {
-	for {
+	if len(from) <= 1 {
-		n, err := unix.Write(t.fd, from)
+		return 0, syscall.EIO
-		if err == nil {
-			return n, nil
 	}
-		switch err {
+
+	ipVer := from[0] >> 4
+	var head [4]byte
+	// first 4 bytes is protocol family, in network byte order
+	switch ipVer {
+	case 4:
+		head[3] = syscall.AF_INET
+	case 6:
+		head[3] = syscall.AF_INET6
+	default:
+		return 0, fmt.Errorf("unable to determine IP version from packet")
+	}
+
+	iovecs := [2]syscall.Iovec{
+		{&head[0], 4},
+		{&from[0], uint64(len(from))},
+	}
+	for {
+		n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
+		if errno == 0 {
+			return int(n) - 4, nil
+		}
+		switch errno {
 		case unix.EAGAIN:
-			if berr := t.blockOnWrite(); berr != nil {
+			if err := t.blockOnWrite(); err != nil {
-				return 0, berr
+				return 0, err
 			}
 		case unix.EINTR:
 			// retry
 		case unix.EBADF:
 			return 0, os.ErrClosed
 		default:
-			return 0, err
+			return 0, errno
 		}
 	}
 }
@@ -530,6 +561,14 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
+func (t *tun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *tun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *tun) SupportsMultiqueue() bool {
 	return false
 }
@@ -701,7 +740,3 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
 
 	return nil, nil
 }
-
-// TunPrefixLen reports the 4-byte BSD AF_INET / AF_INET6 protocol-family
-// marker the kernel prepends on read and expects on write.
-func (t *tun) TunPrefixLen() int { return 4 }

overlay/tun_ios.go

@@ -4,12 +4,15 @@
 package overlay
 
 import (
+	"errors"
 	"fmt"
 	"io"
 	"log/slog"
 	"net/netip"
 	"os"
+	"sync"
 	"sync/atomic"
+	"syscall"
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
@@ -33,7 +36,7 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
 	file := os.NewFile(uintptr(deviceFd), "/dev/tun")
 	t := &tun{
 		vpnNetworks:     vpnNetworks,
-		ReadWriteCloser: file,
+		ReadWriteCloser: &tunReadCloser{f: file},
 		l:               l,
 	}
 
@@ -82,6 +85,64 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
 	return r
 }
 
+// The following is hoisted up from water, we do this so we can inject our own fd on iOS
+type tunReadCloser struct {
+	f io.ReadWriteCloser
+
+	rMu  sync.Mutex
+	rBuf []byte
+
+	wMu  sync.Mutex
+	wBuf []byte
+}
+
+func (tr *tunReadCloser) Read(to []byte) (int, error) {
+	tr.rMu.Lock()
+	defer tr.rMu.Unlock()
+
+	if cap(tr.rBuf) < len(to)+4 {
+		tr.rBuf = make([]byte, len(to)+4)
+	}
+	tr.rBuf = tr.rBuf[:len(to)+4]
+
+	n, err := tr.f.Read(tr.rBuf)
+	copy(to, tr.rBuf[4:])
+	return n - 4, err
+}
+
+func (tr *tunReadCloser) Write(from []byte) (int, error) {
+	if len(from) == 0 {
+		return 0, syscall.EIO
+	}
+
+	tr.wMu.Lock()
+	defer tr.wMu.Unlock()
+
+	if cap(tr.wBuf) < len(from)+4 {
+		tr.wBuf = make([]byte, len(from)+4)
+	}
+	tr.wBuf = tr.wBuf[:len(from)+4]
+
+	// Determine the IP Family for the NULL L2 Header
+	ipVer := from[0] >> 4
+	if ipVer == 4 {
+		tr.wBuf[3] = syscall.AF_INET
+	} else if ipVer == 6 {
+		tr.wBuf[3] = syscall.AF_INET6
+	} else {
+		return 0, errors.New("unable to determine IP version from packet")
+	}
+
+	copy(tr.wBuf[4:], from)
+
+	n, err := tr.f.Write(tr.wBuf)
+	return n - 4, err
+}
+
+func (tr *tunReadCloser) Close() error {
+	return tr.f.Close()
+}
+
 func (t *tun) Networks() []netip.Prefix {
 	return t.vpnNetworks
 }
@@ -90,6 +151,14 @@ func (t *tun) Name() string {
 	return "iOS"
 }
 
+func (t *tun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *tun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *tun) SupportsMultiqueue() bool {
 	return false
 }
@@ -97,7 +166,3 @@ func (t *tun) SupportsMultiqueue() bool {
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for ios")
 }
-
-// TunPrefixLen reports the 4-byte BSD AF_INET / AF_INET6 protocol-family
-// marker the kernel prepends on read and expects on write.
-func (t *tun) TunPrefixLen() int { return 4 }

overlay/tun_linux.go

@@ -368,6 +368,13 @@ func (t *tun) reload(c *config.C, initial bool) error {
 		}
 	}
 
+	// tun.max_mtu raises the device MTU above tun.mtu so PMTUD has headroom to
+	// install per-peer routes between tun.mtu (floor) and tun.max_mtu (ceiling).
+	// When unset (default 0) the device MTU is unchanged from existing behavior.
+	if pmtudCeiling := c.GetInt("tun.max_mtu", 0); pmtudCeiling > newMaxMTU {
+		newMaxMTU = pmtudCeiling
+	}
+
 	t.MaxMTU = newMaxMTU
 	t.DefaultMTU = newDefaultMTU
 
@@ -596,7 +603,7 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
 		LinkIndex: t.deviceIndex,
 		Dst:       dr,
 		MTU:       t.DefaultMTU,
-		AdvMSS:    t.advMSS(Route{}),
+		AdvMSS:    t.advMSS(Route{Cidr: cidr}),
 		Scope:     unix.RT_SCOPE_LINK,
 		Src:       net.IP(cidr.Addr().AsSlice()),
 		Protocol:  unix.RTPROT_KERNEL,
@@ -705,17 +712,68 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
+func (t *tun) SupportsPerPeerMTU() bool {
+	return true
+}
+
+// SetPeerMTU installs a host route (/32 for an IPv4 vpn address, /128 for an IPv6
+// vpn address) to addr through this tun device with the given MTU. This causes
+// the kernel to reject (or surface PTB to apps for) inside packets to addr that
+// would exceed mtu. Pass mtu=0 to remove the override and let the per-vpn-network
+// route apply again. PoC: assumes addr is reachable directly via this device.
+func (t *tun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	bits := addr.BitLen()
+	prefix := netip.PrefixFrom(addr, bits)
+
+	dr := &net.IPNet{
+		IP:   addr.AsSlice(),
+		Mask: net.CIDRMask(bits, bits),
+	}
+
+	if mtu == 0 {
+		nr := netlink.Route{
+			LinkIndex: t.deviceIndex,
+			Dst:       dr,
+			Scope:     unix.RT_SCOPE_LINK,
+		}
+		if err := netlink.RouteDel(&nr); err != nil {
+			return fmt.Errorf("failed to remove per-peer mtu route %v: %w", prefix, err)
+		}
+		return nil
+	}
+
+	nr := netlink.Route{
+		LinkIndex: t.deviceIndex,
+		Dst:       dr,
+		MTU:       mtu,
+		AdvMSS:    t.advMSS(Route{Cidr: prefix, MTU: mtu}),
+		Scope:     unix.RT_SCOPE_LINK,
+	}
+	if err := netlink.RouteReplace(&nr); err != nil {
+		return fmt.Errorf("failed to set per-peer mtu route %v mtu=%d: %w", prefix, mtu, err)
+	}
+	return nil
+}
+
 func (t *tun) advMSS(r Route) int {
 	mtu := r.MTU
 	if r.MTU == 0 {
 		mtu = t.DefaultMTU
 	}
 
-	// We only need to set advmss if the route MTU does not match the device MTU
+	// We only need to set advmss if the route MTU does not match the device MTU.
-	if mtu != t.MaxMTU {
+	if mtu == t.MaxMTU {
-		return mtu - 40
-	}
 		return 0
+	}
+
+	// MSS = MTU - (IP header + TCP header). TCP is always 20 bytes; IP is 20 for
+	// v4 and 40 for v6. r.Cidr is the route destination so it tells us which
+	// family this route is in. If Cidr is unset (empty Route) we default to v4.
+	addr := r.Cidr.Addr()
+	if addr.Is6() && !addr.Is4In6() {
+		return mtu - 60
+	}
+	return mtu - 40
 }
 
 func (t *tun) watchRoutes() {
@@ -907,5 +965,3 @@ func (t *tun) Close() error {
 	}
 	return err
 }
-
-func (t *tun) TunPrefixLen() int { return 0 }

overlay/tun_netbsd.go

@@ -58,13 +58,13 @@ type addrLifetime struct {
 }
 
 type tun struct {
-	io.ReadWriteCloser
 	Device      string
 	vpnNetworks []netip.Prefix
 	MTU         int
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
 	l           *slog.Logger
+	f           *os.File
 	fd          int
 }
 
@@ -96,7 +96,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 	}
 
 	t := &tun{
-		ReadWriteCloser: os.NewFile(uintptr(fd), ""),
+		f:           os.NewFile(uintptr(fd), ""),
 		fd:          fd,
 		Device:      deviceName,
 		vpnNetworks: vpnNetworks,
@@ -120,12 +120,12 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 }
 
 func (t *tun) Close() error {
-	if t.ReadWriteCloser != nil {
+	if t.f != nil {
-		if err := t.ReadWriteCloser.Close(); err != nil {
+		if err := t.f.Close(); err != nil {
 			return fmt.Errorf("error closing tun file: %w", err)
 		}
 
-		// Close on the os.File should have handled the fd for us but let's be extra sure
+		// t.f.Close should have handled it for us but let's be extra sure
 		_ = unix.Close(t.fd)
 
 		s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
@@ -141,6 +141,99 @@ func (t *tun) Close() error {
 	return nil
 }
 
+func (t *tun) Read(to []byte) (int, error) {
+	rc, err := t.f.SyscallConn()
+	if err != nil {
+		return 0, fmt.Errorf("failed to get syscall conn for tun: %w", err)
+	}
+
+	var errno syscall.Errno
+	var n uintptr
+	err = rc.Read(func(fd uintptr) bool {
+		// first 4 bytes is protocol family, in network byte order
+		head := [4]byte{}
+		iovecs := []syscall.Iovec{
+			{&head[0], 4},
+			{&to[0], uint64(len(to))},
+		}
+
+		n, _, errno = syscall.Syscall(syscall.SYS_READV, fd, uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
+		if errno.Temporary() {
+			// We got an EAGAIN, EINTR, or EWOULDBLOCK, go again
+			return false
+		}
+		return true
+	})
+	if err != nil {
+		if err == syscall.EBADF || err.Error() == "use of closed file" {
+			// Go doesn't export poll.ErrFileClosing but happily reports it to us so here we are
+			// https://github.com/golang/go/blob/master/src/internal/poll/fd_poll_runtime.go#L121
+			return 0, os.ErrClosed
+		}
+		return 0, fmt.Errorf("failed to make read call for tun: %w", err)
+	}
+
+	if errno != 0 {
+		return 0, fmt.Errorf("failed to make inner read call for tun: %w", errno)
+	}
+
+	// fix bytes read number to exclude header
+	bytesRead := int(n)
+	if bytesRead < 0 {
+		return bytesRead, nil
+	} else if bytesRead < 4 {
+		return 0, nil
+	} else {
+		return bytesRead - 4, nil
+	}
+}
+
+// Write is only valid for single threaded use
+func (t *tun) Write(from []byte) (int, error) {
+	if len(from) <= 1 {
+		return 0, syscall.EIO
+	}
+
+	ipVer := from[0] >> 4
+	var head [4]byte
+	// first 4 bytes is protocol family, in network byte order
+	if ipVer == 4 {
+		head[3] = syscall.AF_INET
+	} else if ipVer == 6 {
+		head[3] = syscall.AF_INET6
+	} else {
+		return 0, fmt.Errorf("unable to determine IP version from packet")
+	}
+
+	rc, err := t.f.SyscallConn()
+	if err != nil {
+		return 0, err
+	}
+
+	var errno syscall.Errno
+	var n uintptr
+	err = rc.Write(func(fd uintptr) bool {
+		iovecs := []syscall.Iovec{
+			{&head[0], 4},
+			{&from[0], uint64(len(from))},
+		}
+
+		n, _, errno = syscall.Syscall(syscall.SYS_WRITEV, fd, uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
+		// According to NetBSD documentation for TUN, writes will only return errors in which
+		// this packet will never be delivered so just go on living life.
+		return true
+	})
+	if err != nil {
+		return 0, err
+	}
+
+	if errno != 0 {
+		return 0, errno
+	}
+
+	return int(n) - 4, err
+}
+
 func (t *tun) addIp(cidr netip.Prefix) error {
 	if cidr.Addr().Is4() {
 		var req ifreqAlias4
@@ -297,6 +390,14 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
+func (t *tun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *tun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *tun) SupportsMultiqueue() bool {
 	return false
 }
@@ -458,7 +559,3 @@ func delRoute(prefix netip.Prefix, gateways []netip.Prefix) error {
 
 	return nil
 }
-
-// TunPrefixLen reports the 4-byte BSD AF_INET / AF_INET6 protocol-family
-// marker the kernel prepends on read and expects on write.
-func (t *tun) TunPrefixLen() int { return 4 }

overlay/tun_no_prefix.go

@@ -1,10 +0,0 @@
-//go:build (!darwin && !ios && !freebsd && !openbsd && !netbsd) || e2e_testing
-
-package overlay
-
-// StampTunPrefix is a no-op on platforms whose tun devices have no
-// protocol-family marker. WireBuffer only invokes it when its prefixLen
-// is non-zero, so this should never be reached on these platforms.
-func StampTunPrefix(buf []byte) error {
-	return nil
-}

overlay/tun_openbsd.go

@@ -49,14 +49,16 @@ type ifreq struct {
 }
 
 type tun struct {
-	io.ReadWriteCloser
 	Device      string
 	vpnNetworks []netip.Prefix
 	MTU         int
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
 	l           *slog.Logger
+	f           *os.File
 	fd          int
+	// cache out buffer since we need to prepend 4 bytes for tun metadata
+	out []byte
 }
 
 var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
@@ -87,7 +89,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 	}
 
 	t := &tun{
-		ReadWriteCloser: os.NewFile(uintptr(fd), ""),
+		f:           os.NewFile(uintptr(fd), ""),
 		fd:          fd,
 		Device:      deviceName,
 		vpnNetworks: vpnNetworks,
@@ -111,17 +113,55 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 }
 
 func (t *tun) Close() error {
-	if t.ReadWriteCloser != nil {
+	if t.f != nil {
-		if err := t.ReadWriteCloser.Close(); err != nil {
+		if err := t.f.Close(); err != nil {
 			return fmt.Errorf("error closing tun file: %w", err)
 		}
 
-		// Close on the os.File should have handled the fd for us but let's be extra sure
+		// t.f.Close should have handled it for us but let's be extra sure
 		_ = unix.Close(t.fd)
 	}
 	return nil
 }
 
+func (t *tun) Read(to []byte) (int, error) {
+	buf := make([]byte, len(to)+4)
+
+	n, err := t.f.Read(buf)
+
+	copy(to, buf[4:])
+	return n - 4, err
+}
+
+// Write is only valid for single threaded use
+func (t *tun) Write(from []byte) (int, error) {
+	buf := t.out
+	if cap(buf) < len(from)+4 {
+		buf = make([]byte, len(from)+4)
+		t.out = buf
+	}
+	buf = buf[:len(from)+4]
+
+	if len(from) == 0 {
+		return 0, syscall.EIO
+	}
+
+	// Determine the IP Family for the NULL L2 Header
+	ipVer := from[0] >> 4
+	if ipVer == 4 {
+		buf[3] = syscall.AF_INET
+	} else if ipVer == 6 {
+		buf[3] = syscall.AF_INET6
+	} else {
+		return 0, fmt.Errorf("unable to determine IP version from packet")
+	}
+
+	copy(buf[4:], from)
+
+	n, err := t.f.Write(buf)
+	return n - 4, err
+}
+
 func (t *tun) addIp(cidr netip.Prefix) error {
 	if cidr.Addr().Is4() {
 		var req ifreqAlias4
@@ -270,6 +310,14 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
+func (t *tun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *tun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *tun) SupportsMultiqueue() bool {
 	return false
 }
@@ -431,7 +479,3 @@ func delRoute(prefix netip.Prefix, gateways []netip.Prefix) error {
 
 	return nil
 }
-
-// TunPrefixLen reports the 4-byte BSD AF_INET / AF_INET6 protocol-family
-// marker the kernel prepends on read and expects on write.
-func (t *tun) TunPrefixLen() int { return 4 }

overlay/tun_tester.go

@@ -15,7 +15,6 @@ import (
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/routing"
-	"github.com/slackhq/nebula/udp"
 )
 
 type TestTun struct {
@@ -55,12 +54,9 @@ func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*TestTu
 	return nil, fmt.Errorf("newTunFromFd not supported")
 }
 
-// Send will place a byte array onto the receive queue for nebula to consume.
+// Send will place a byte array onto the receive queue for nebula to consume
 // These are unencrypted ip layer frames destined for another nebula node.
-// packets should exit the udp side, capture them with udpConn.Get.
+// packets should exit the udp side, capture them with udpConn.Get
-//
-// Send copies the input via the freelist, so the caller is free to mutate
-// or reuse it after the call returns.
 func (t *TestTun) Send(packet []byte) {
 	if t.closed.Load() {
 		return
@@ -69,9 +65,7 @@ func (t *TestTun) Send(packet []byte) {
 	if t.l.Enabled(context.Background(), slog.LevelDebug) {
 		t.l.Debug("Tun receiving injected packet", "dataLen", len(packet))
 	}
-	buf := acquireTunBuf(len(packet))
+	t.rxPackets <- packet
-	copy(buf, packet)
-	t.rxPackets <- buf
 }
 
 // Get will pull an unencrypted ip layer frame from the transmit queue
@@ -111,49 +105,25 @@ func (t *TestTun) Name() string {
 	return t.Device
 }
 
+func (t *TestTun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *TestTun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *TestTun) Write(b []byte) (n int, err error) {
 	if t.closed.Load() {
 		return 0, io.ErrClosedPipe
 	}
 
-	packet := acquireTunBuf(len(b))
+	packet := make([]byte, len(b), len(b))
 	copy(packet, b)
 	t.TxPackets <- packet
 	return len(b), nil
 }
 
-// ReleaseTunBuf returns a slice from TxPackets to the harness freelist, don't use the bytes after the call.
-// Channel-backed instead of sync.Pool because putting a []byte in a sync.Pool escapes the slice header to heap.
-func ReleaseTunBuf(b []byte) {
-	if b == nil {
-		return
-	}
-	select {
-	case tunBufFreelist <- b:
-	default:
-		// Freelist full; drop the buffer for the GC.
-	}
-}
-
-// tunBufFreelist retains the backing arrays for TestTun.Write so steady-state allocation drops to zero once the
-// freelist has saturated for the current MTU.
-var tunBufFreelist = make(chan []byte, 64)
-
-func acquireTunBuf(n int) []byte {
-	var b []byte
-	select {
-	case b = <-tunBufFreelist:
-	default:
-		b = make([]byte, 0, udp.MTU)
-	}
-	if cap(b) < n {
-		b = make([]byte, n)
-	} else {
-		b = b[:n]
-	}
-	return b
-}
-
 func (t *TestTun) Close() error {
 	if t.closed.CompareAndSwap(false, true) {
 		close(t.rxPackets)
@@ -167,14 +137,8 @@ func (t *TestTun) Read(b []byte) (int, error) {
 	if !ok {
 		return 0, os.ErrClosed
 	}
-	n := len(p)
 	copy(b, p)
-	// Send always pushes a freelist-acquired slice, return it once we've copied the bytes into the caller's buffer.
+	return len(p), nil
-	select {
-	case tunBufFreelist <- p:
-	default:
-	}
-	return n, nil
 }
 
 func (t *TestTun) SupportsMultiqueue() bool {
@@ -184,5 +148,3 @@ func (t *TestTun) SupportsMultiqueue() bool {
 func (t *TestTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented")
 }
-
-func (t *TestTun) TunPrefixLen() int { return 0 }

overlay/tun_windows.go

@@ -229,6 +229,14 @@ func (t *winTun) Name() string {
 	return t.Device
 }
 
+func (t *winTun) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (t *winTun) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (t *winTun) Read(b []byte) (int, error) {
 	return t.tun.Read(b, 0)
 }
@@ -296,5 +304,3 @@ func checkWinTunExists() error {
 	_, err = syscall.LoadDLL(filepath.Join(filepath.Dir(myPath), "dist", "windows", "wintun", "bin", arch, "wintun.dll"))
 	return err
 }
-
-func (t *winTun) TunPrefixLen() int { return 0 }

overlay/user.go

@@ -46,6 +46,14 @@ func (d *UserDevice) RoutesFor(ip netip.Addr) routing.Gateways {
 	return routing.Gateways{routing.NewGateway(ip, 1)}
 }
 
+func (d *UserDevice) SupportsPerPeerMTU() bool {
+	return false
+}
+
+func (d *UserDevice) SetPeerMTU(addr netip.Addr, mtu int) error {
+	return nil
+}
+
 func (d *UserDevice) SupportsMultiqueue() bool {
 	return true
 }
@@ -69,5 +77,3 @@ func (d *UserDevice) Close() error {
 	d.outboundWriter.Close()
 	return nil
 }
-
-func (d *UserDevice) TunPrefixLen() int { return 0 }

pmtud_manager.go

@@ -0,0 +1,623 @@
+package nebula
+
+import (
+	"context"
+	"encoding/binary"
+	"log/slog"
+	"math/rand/v2"
+	"net/netip"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/overlay"
+)
+
+// PMTUD PoC: discover the path MTU per-tunnel via authenticated probes that ride
+// the existing crypto session. We follow RFC 8899 PLPMTUD: a binary search
+// between a known-good floor and a configured ceiling, with N consecutive probe
+// losses at a size treated as "doesn't fit." Confirmed PMTU is pushed to the
+// overlay device, which on Linux installs a per-host route with the discovered
+// MTU. The kernel then surfaces EMSGSIZE / PTB to apps writing to the tun.
+//
+// Probe payload format (request):
+//
+//	[magic uint32 BE][probeID uint32 BE][padding 0x00...]
+//
+// Reply is a small ack with the same magic and probeID and no padding. We do not
+// verify the reverse-path MTU; only the forward direction matters for the
+// receiver's MTU on the inside.
+
+const (
+	pmtudMagic uint32 = 0x504D5544 // 'P' 'M' 'U' 'D'
+	pmtudFloor        = 1280       // IPv6 minimum payload, also a safe internet MTU floor
+
+	// pmtudConverged is the bytes-tolerance for stopping the search.
+	pmtudConverged = 8
+
+	// pmtudMaxLoss matches RFC 8899 MAX_PROBES (default 3).
+	pmtudMaxLoss = 3
+
+	// pmtudProbeInterval is the time between probe ticks during the search phase.
+	// Once a peer converges the wheel stops ticking it; re-validation is driven
+	// by connection_manager via MaybeProbeAsTest at its natural test cadence.
+	pmtudProbeInterval = 500 * time.Millisecond
+
+	// pmtudWheelMax is the wheel's maximum supported scheduling duration. We
+	// only ever schedule at pmtudProbeInterval today, but the wheel needs a
+	// max greater than its tick to allocate its slot ring sensibly.
+	pmtudWheelMax = 5 * time.Second
+
+	// pmtudOverheadPessimistic assumes IPv6 underlay + relay framing:
+	//   IPv6(40) + UDP(8) + outer nebula(16) + outer AEAD tag(16)
+	// + inner nebula(16) + inner AEAD tag(16) = 112 bytes.
+	// TODO: track underlay address family and per-peer relay state on the HostInfo
+	// so the manager can use the actual overhead for that tunnel and recover the
+	// 32 bytes we pessimistically give up on direct IPv6 paths and the 52 bytes on
+	// direct IPv4 paths.
+	pmtudOverheadPessimistic = 112
+
+	// pmtudUnsupportedAfter is the number of consecutive lost probes (across any
+	// sizes) without ever receiving a reply that we treat as evidence the peer
+	// does not understand the MTUDProbeRequest subtype (i.e. it's running an
+	// older nebula). After this many failures with everReplied=false we mark the
+	// peer pmtud-unsupported and stop scheduling probes. K is small enough that
+	// it fires before the binary search would naturally converge to floor (which
+	// would otherwise be ~30 wasted probes), but large enough to absorb a few
+	// transient probe losses on a path that's just starting to settle.
+	pmtudUnsupportedAfter = 5
+)
+
+// pmtudPeer tracks the binary-search state for one tunnel.
+type pmtudPeer struct {
+	mu       sync.Mutex
+	addr     netip.Addr
+	localIdx uint32
+
+	// low is the largest outer IP packet size we have a confirmed ack for.
+	// high is the smallest size we believe fails (the search ceiling to start).
+	low, high int
+
+	// inFlightSize is the outer IP packet size of the probe currently awaiting
+	// an ack. 0 means no probe in flight.
+	inFlightSize int
+	// inFlightID matches the probeID echoed in the reply.
+	inFlightID uint32
+	// losses counts consecutive failures at inFlightSize.
+	losses int
+
+	// firstProbe is true until we have sent the first probe of a search. The
+	// first probe targets the ceiling directly (RFC 8899 permits this Search
+	// Algorithm choice); operators who set tun.max_mtu typically have a path
+	// that supports it, so we converge in one probe in the common case.
+	firstProbe bool
+	// everReplied is true once we have ever received any MTUDProbeReply from
+	// this peer. Combined with consecutiveFailures, this lets us detect peers
+	// that don't understand the new subtype and stop probing them.
+	everReplied bool
+	// consecutiveFailures counts probes lost without an intervening reply.
+	// Resets to 0 on any successful reply.
+	consecutiveFailures int
+	// unsupported is set true once we conclude the peer doesn't speak PMTUD.
+	// The manager skips probes for unsupported peers.
+	unsupported bool
+
+	// converged means we have a confirmed PMTU and are in the slow re-validation phase.
+	converged bool
+	// applied is the inner MTU we last pushed to the overlay device (0 if never).
+	applied int
+}
+
+func (p *pmtudPeer) overhead() int {
+	// TODO: branch on actual underlay family + relay state for this peer.
+	return pmtudOverheadPessimistic
+}
+
+func (p *pmtudPeer) midpoint() int {
+	return (p.low + p.high) / 2
+}
+
+type pmtudManager struct {
+	intf   *Interface
+	device overlay.Device
+
+	// peers is keyed by HostInfo.localIndexId.
+	peers sync.Map // map[uint32]*pmtudPeer
+
+	wheel *LockingTimerWheel[uint32]
+
+	// floor is the always-safe inner MTU (= tun.mtu). Per-peer routes start here
+	// on tunnel-up so unprobed traffic is always small enough to fit. Stored as
+	// atomic int64 so reload can update it without coordinating with the readers
+	// in tick/HandleReply/OnTunnelUp.
+	floor atomic.Int64
+	// ceiling is the search ceiling expressed as an outer IP packet size, derived
+	// from tun.max_mtu (which is the kernel's device MTU on the tun) plus our
+	// pessimistic overhead. PMTUD will not probe larger than this.
+	ceiling atomic.Int64
+
+	enabled atomic.Bool
+
+	l *slog.Logger
+}
+
+func newPMTUDManagerFromConfig(l *slog.Logger, c *config.C, device overlay.Device) *pmtudManager {
+	m := &pmtudManager{
+		device: device,
+		wheel:  NewLockingTimerWheel[uint32](pmtudProbeInterval, pmtudWheelMax),
+		l:      l,
+	}
+	c.RegisterReloadCallback(func(c *config.C) { m.reload(c, false) })
+	m.reload(c, true)
+	return m
+}
+
+// reload applies tun.mtu / tun.max_mtu changes to the manager. On the initial
+// call (during construction) it just snapshots state; on a live reload it also
+// transitions in-flight peers to match the new bounds: clearing per-peer routes
+// when newly disabled, seeding peers from the hostmap and flipping DF on
+// outside sockets when newly enabled, and rebounding existing searches in
+// place when only the ceiling moved.
+func (m *pmtudManager) reload(c *config.C, initial bool) {
+	if !initial && !c.HasChanged("tun.mtu") && !c.HasChanged("tun.max_mtu") {
+		return
+	}
+
+	floor := c.GetInt("tun.mtu", overlay.DefaultMTU)
+	maxMTU := c.GetInt("tun.max_mtu", 0)
+
+	enable := maxMTU > floor && m.device.SupportsPerPeerMTU()
+	var ceiling int
+	if enable {
+		ceiling = maxMTU + pmtudOverheadPessimistic
+	}
+
+	if initial {
+		m.floor.Store(int64(floor))
+		m.ceiling.Store(int64(ceiling))
+		m.enabled.Store(enable)
+		switch {
+		case enable:
+			m.l.Info("pmtud enabled", "floor", floor, "ceiling", ceiling, "tun.max_mtu", maxMTU)
+		case maxMTU > floor:
+			m.l.Warn("pmtud disabled: this platform does not yet support per-peer MTU routes",
+				"tun.max_mtu", maxMTU)
+		}
+		return
+	}
+
+	wasEnabled := m.enabled.Load()
+	m.floor.Store(int64(floor))
+	m.ceiling.Store(int64(ceiling))
+	m.enabled.Store(enable)
+
+	switch {
+	case wasEnabled && !enable:
+		m.disableLive(floor, maxMTU)
+	case !wasEnabled && enable:
+		m.enableLive(floor, ceiling, maxMTU)
+	case wasEnabled && enable:
+		m.reboundLive(floor, ceiling, maxMTU)
+	}
+}
+
+// disableLive clears per-peer routes and drops all peer state. We do not
+// disable DF on the outside sockets; once on, it stays on for the life of the
+// process. Operators flipping pmtud off live get correct routing behavior; if
+// they want the historical no-DF behavior back they need to restart.
+func (m *pmtudManager) disableLive(floor, maxMTU int) {
+	m.peers.Range(func(k, v any) bool {
+		p := v.(*pmtudPeer)
+		p.mu.Lock()
+		applied := p.applied
+		addr := p.addr
+		p.applied = 0
+		p.mu.Unlock()
+		if applied != 0 {
+			if err := m.device.SetPeerMTU(addr, 0); err != nil {
+				m.l.Warn("pmtud: failed to clear per-peer mtu on disable", "addr", addr, "error", err)
+			}
+		}
+		m.peers.Delete(k)
+		return true
+	})
+	m.l.Info("pmtud disabled (tun.max_mtu <= tun.mtu)", "tun.mtu", floor, "tun.max_mtu", maxMTU)
+}
+
+// enableLive flips DF on every outside socket. We don't pre-seed existing
+// tunnels here; connection_manager's normal test cadence will eventually call
+// MaybeProbeAsTest for each peer, which seeds on miss and lets the wheel pick
+// up the search from there. New tunnels established after this point still
+// take the OnTunnelUp fast path.
+func (m *pmtudManager) enableLive(floor, ceiling, maxMTU int) {
+	m.enableDF()
+	m.l.Info("pmtud enabled", "floor", floor, "ceiling", ceiling, "tun.max_mtu", maxMTU)
+}
+
+// reboundLive resets each peer's search state to the new bounds. Peers whose
+// confirmed PMTU still fits under the new ceiling keep their applied route in
+// place during the new search; peers whose confirmed PMTU exceeds the new
+// ceiling get cleared back to floor and re-search from scratch. The unsupported
+// flag is preserved because peer software version doesn't change on reload.
+func (m *pmtudManager) reboundLive(floor, ceiling, maxMTU int) {
+	overhead := pmtudOverheadPessimistic
+	m.peers.Range(func(k, v any) bool {
+		p := v.(*pmtudPeer)
+		p.mu.Lock()
+		if p.applied > 0 && p.applied+overhead > ceiling {
+			if err := m.device.SetPeerMTU(p.addr, 0); err != nil {
+				m.l.Warn("pmtud: failed to clear per-peer mtu on rebound", "addr", p.addr, "error", err)
+			} else {
+				p.applied = 0
+			}
+		}
+		p.low = floor + overhead
+		p.high = ceiling
+		p.inFlightSize = 0
+		p.inFlightID = 0
+		p.losses = 0
+		p.firstProbe = !p.unsupported
+		p.converged = false
+		idx := p.localIdx
+		unsupported := p.unsupported
+		p.mu.Unlock()
+		if !unsupported {
+			m.wheel.Add(idx, pmtudProbeInterval)
+		}
+		return true
+	})
+	m.l.Info("pmtud reloaded", "floor", floor, "ceiling", ceiling, "tun.max_mtu", maxMTU)
+}
+
+// enableDF asks every outside socket to set the don't-fragment bit on outbound
+// packets. Idempotent: safe to call from both Start (initial enable) and from a
+// live reload that flips pmtud on.
+func (m *pmtudManager) enableDF() {
+	for i, w := range m.intf.writers {
+		if err := w.EnablePathMTUDiscovery(); err != nil {
+			m.l.Warn("pmtud: failed to enable path mtu discovery on outside socket; pmtud will not work correctly",
+				"writer", i, "error", err)
+		}
+	}
+}
+
+// Start runs the probe scheduler until ctx is done. The loop runs even when PMTUD
+// is disabled at startup so a hot reload can turn it on without restarting nebula.
+//
+// When PMTUD is enabled at startup we ask each outside socket to enable
+// path-MTU discovery (DF on every send). This is intentionally gated on the
+// feature being on so that operators who haven't opted in keep the historical
+// behavior where the kernel may fragment outbound nebula UDP packets. A live
+// reload from disabled to enabled will also flip DF on via enableLive; the
+// reverse direction does not turn DF off, so flipping pmtud back off live
+// keeps DF on until restart.
+func (m *pmtudManager) Start(ctx context.Context) {
+	if m.enabled.Load() {
+		m.enableDF()
+	}
+
+	ticker := time.NewTicker(m.wheel.t.tickDuration)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case now := <-ticker.C:
+			m.wheel.Advance(now)
+			for {
+				idx, has := m.wheel.Purge()
+				if !has {
+					break
+				}
+				m.tick(idx)
+			}
+		}
+	}
+}
+
+// OnTunnelUp is called when a HostInfo becomes traffic-watched. The kernel
+// already routes packets to this peer through the per-vpn-network route (mtu =
+// tun.mtu), so the floor is in effect implicitly. We just kick off the search
+// here; HandleReply will install a per-host /32 (or /128) route once a larger
+// size is confirmed.
+func (m *pmtudManager) OnTunnelUp(hi *HostInfo) {
+	if !m.enabled.Load() {
+		return
+	}
+	m.seedPeer(hi)
+}
+
+// seedPeer is the shared body of OnTunnelUp and the live-reload enable path.
+// LoadOrStore protects against double-seeding the same localIndexId from a
+// race between OnTunnelUp and a reload-driven hostmap walk.
+func (m *pmtudManager) seedPeer(hi *HostInfo) {
+	if hi == nil || len(hi.vpnAddrs) == 0 {
+		return
+	}
+	floor := int(m.floor.Load())
+	ceiling := int(m.ceiling.Load())
+	p := &pmtudPeer{
+		addr:       hi.vpnAddrs[0],
+		localIdx:   hi.localIndexId,
+		low:        floor + pmtudOverheadPessimistic,
+		high:       ceiling,
+		firstProbe: true,
+	}
+	if _, loaded := m.peers.LoadOrStore(hi.localIndexId, p); loaded {
+		return
+	}
+	m.wheel.Add(hi.localIndexId, pmtudProbeInterval)
+}
+
+// OnTunnelDown is called when a HostInfo is being torn down. Removes any per-host
+// MTU override so the device default applies again.
+func (m *pmtudManager) OnTunnelDown(hi *HostInfo) {
+	if hi == nil {
+		return
+	}
+	v, ok := m.peers.LoadAndDelete(hi.localIndexId)
+	if !ok {
+		return
+	}
+	p := v.(*pmtudPeer)
+	p.mu.Lock()
+	applied := p.applied
+	addr := p.addr
+	p.applied = 0
+	p.mu.Unlock()
+	if applied != 0 {
+		if err := m.device.SetPeerMTU(addr, 0); err != nil {
+			m.l.Warn("pmtud: failed to clear per-peer mtu", "addr", addr, "error", err)
+		}
+	}
+}
+
+// OnRoam is called when a HostInfo's remote underlay address changes. The path
+// MTU may now be different; drop the per-host route so the kernel falls back to
+// the per-vpn-network route (mtu = tun.mtu floor), then restart the search.
+// We do not reset the unsupported flag: peer software version doesn't change on
+// roam, so once we've decided a peer doesn't speak PMTUD we stay decided.
+func (m *pmtudManager) OnRoam(hi *HostInfo) {
+	if !m.enabled.Load() || hi == nil {
+		return
+	}
+	v, ok := m.peers.Load(hi.localIndexId)
+	if !ok {
+		return
+	}
+	p := v.(*pmtudPeer)
+	p.mu.Lock()
+	if p.unsupported {
+		p.mu.Unlock()
+		return
+	}
+	p.low = int(m.floor.Load()) + pmtudOverheadPessimistic
+	p.high = int(m.ceiling.Load())
+	p.inFlightSize = 0
+	p.inFlightID = 0
+	p.losses = 0
+	p.consecutiveFailures = 0
+	p.firstProbe = true
+	p.converged = false
+	if p.applied != 0 {
+		if err := m.device.SetPeerMTU(p.addr, 0); err != nil {
+			m.l.Warn("pmtud: failed to clear per-peer mtu on roam", "addr", p.addr, "error", err)
+		} else {
+			p.applied = 0
+		}
+	}
+	p.mu.Unlock()
+	m.wheel.Add(hi.localIndexId, pmtudProbeInterval)
+}
+
+// MaybeProbeAsTest is called by connection_manager when it would otherwise send
+// a TestRequest because a tunnel has gone silent. If we have a confirmed PMTU
+// for this peer that's larger than the floor, we send a probe at that size
+// instead. The reply confirms both liveness (consumed by connection_manager via
+// the existing inbound traffic accounting fallthrough in outside.go) and that
+// the confirmed PMTU still fits (consumed by HandleReply here). One synthetic
+// packet does the work of two.
+//
+// Returns true if a probe was sent. False means the caller should send a
+// regular TestRequest at the floor.
+//
+// On probe failure, connection_manager's existing pendingDeletion timeout will
+// tear the tunnel down. Heavy hammer, but correct: a re-handshake re-runs PMTUD
+// discovery against the now-shrunken path. A future EMSGSIZE-capture followup
+// can replace this with a soft-drop-and-research flow.
+func (m *pmtudManager) MaybeProbeAsTest(hi *HostInfo) bool {
+	if !m.enabled.Load() || hi == nil {
+		return false
+	}
+	v, ok := m.peers.Load(hi.localIndexId)
+	if !ok {
+		// Tunnel pre-dates the manager being aware of it (e.g. pmtud was just
+		// enabled live, or AddTrafficWatch fired before this call). Seed the
+		// peer so the wheel picks up the search; let connection_manager send
+		// its regular TestRequest this cycle.
+		m.seedPeer(hi)
+		return false
+	}
+	p := v.(*pmtudPeer)
+	p.mu.Lock()
+	if p.unsupported || p.applied == 0 {
+		p.mu.Unlock()
+		return false
+	}
+	overhead := p.overhead()
+	size := p.applied + overhead
+	id := rand.Uint32()
+	p.inFlightSize = size
+	p.inFlightID = id
+	p.mu.Unlock()
+
+	m.sendProbe(hi, size, id, overhead)
+	return true
+}
+
+// HandleReply consumes an MTUDProbeReply payload from the receive path.
+func (m *pmtudManager) HandleReply(localIdx uint32, payload []byte) {
+	if !m.enabled.Load() {
+		return
+	}
+	if len(payload) < 8 {
+		return
+	}
+	if binary.BigEndian.Uint32(payload[0:4]) != pmtudMagic {
+		return
+	}
+	id := binary.BigEndian.Uint32(payload[4:8])
+
+	v, ok := m.peers.Load(localIdx)
+	if !ok {
+		return
+	}
+	p := v.(*pmtudPeer)
+	p.mu.Lock()
+	defer p.mu.Unlock()
+
+	if p.inFlightSize == 0 || p.inFlightID != id {
+		return
+	}
+
+	confirmed := p.inFlightSize
+	p.low = confirmed
+	p.inFlightSize = 0
+	p.losses = 0
+	p.everReplied = true
+	p.consecutiveFailures = 0
+
+	innerMTU := confirmed - p.overhead()
+	// Only install a /32 override when it would actually raise the MTU above the
+	// per-vpn-network floor route. If the discovered MTU is <= floor, the /24
+	// already covers it; installing a /32 at floor would just create roam churn.
+	if innerMTU > int(m.floor.Load()) && p.applied != innerMTU {
+		if err := m.device.SetPeerMTU(p.addr, innerMTU); err != nil {
+			m.l.Warn("pmtud: failed to apply per-peer mtu", "addr", p.addr, "innerMTU", innerMTU, "error", err)
+		} else {
+			m.l.Info("pmtud probe confirmed",
+				"addr", p.addr,
+				"outerMTU", confirmed,
+				"innerMTU", innerMTU,
+				"low", p.low,
+				"high", p.high,
+			)
+			p.applied = innerMTU
+		}
+	}
+
+	if p.high-p.low <= pmtudConverged {
+		p.converged = true
+	} else {
+		p.converged = false
+	}
+}
+
+// tick handles one wheel firing for a single peer.
+func (m *pmtudManager) tick(localIdx uint32) {
+	v, ok := m.peers.Load(localIdx)
+	if !ok {
+		return
+	}
+	p := v.(*pmtudPeer)
+	p.mu.Lock()
+
+	if p.unsupported {
+		p.mu.Unlock()
+		return
+	}
+
+	// If a probe was outstanding, this tick is the loss timeout.
+	if p.inFlightSize != 0 {
+		p.losses++
+		p.consecutiveFailures++
+		if p.losses >= pmtudMaxLoss {
+			p.high = p.inFlightSize
+			p.inFlightSize = 0
+			p.losses = 0
+			if p.high-p.low <= pmtudConverged {
+				p.converged = true
+			}
+		}
+	}
+
+	// If we've never gotten a reply from this peer and we've burned through our
+	// failure budget, conclude the peer doesn't understand the MTUDProbeRequest
+	// subtype and stop scheduling probes for it.
+	if !p.everReplied && p.consecutiveFailures >= pmtudUnsupportedAfter {
+		p.unsupported = true
+		addr := p.addr
+		p.mu.Unlock()
+		m.l.Info("pmtud: peer not responding to probes, marking unsupported",
+			"addr", addr, "failures", pmtudUnsupportedAfter)
+		return
+	}
+
+	hi := m.intf.hostMap.QueryIndex(localIdx)
+	if hi == nil {
+		p.mu.Unlock()
+		m.peers.Delete(localIdx)
+		return
+	}
+
+	// Once a peer converges, the wheel stops scheduling for it. Re-validation
+	// (and the resulting black hole detection) is driven by connection_manager
+	// via MaybeProbeAsTest at its natural test cadence, so a converged peer
+	// has nothing for the wheel to do until OnRoam or a tunnel down/up cycle
+	// triggers a fresh search.
+	if p.converged {
+		p.mu.Unlock()
+		return
+	}
+
+	ceiling := int(m.ceiling.Load())
+	var size int
+	switch {
+	case p.firstProbe:
+		// Probe the ceiling directly. If the path supports it (the common case
+		// when an operator has explicitly configured tun.max_mtu), we converge
+		// in one round trip. If it fails, the standard binary search resumes
+		// on the next tick from the (low, ceiling) bounds.
+		size = ceiling
+		p.firstProbe = false
+	case p.losses > 0 && p.inFlightSize != 0:
+		size = p.inFlightSize
+	default:
+		size = p.midpoint()
+	}
+	if size < pmtudFloor {
+		size = pmtudFloor
+	}
+	if size > ceiling {
+		size = ceiling
+	}
+
+	id := rand.Uint32()
+	p.inFlightSize = size
+	p.inFlightID = id
+	overhead := p.overhead()
+	p.mu.Unlock()
+
+	m.sendProbe(hi, size, id, overhead)
+	m.wheel.Add(localIdx, pmtudProbeInterval)
+}
+
+// sendProbe builds an MTUDProbeRequest payload that will produce an outer IP
+// packet of approximately `outerSize` bytes, then sends it.
+func (m *pmtudManager) sendProbe(hi *HostInfo, outerSize int, id uint32, overhead int) {
+	payloadLen := outerSize - overhead
+	if payloadLen < 8 {
+		payloadLen = 8
+	}
+	p := make([]byte, payloadLen)
+	binary.BigEndian.PutUint32(p[0:4], pmtudMagic)
+	binary.BigEndian.PutUint32(p[4:8], id)
+	// remaining bytes are zero-padding
+
+	nb := make([]byte, 12)
+	out := make([]byte, outerSize+128) // headroom for header/tag/relay framing
+	m.intf.SendMessageToHostInfo(header.Test, header.MTUDProbeRequest, hi, p, nb, out)
+}

relay_manager.go

@@ -63,9 +63,6 @@ func (rm *relayManager) StartRelays(f *Interface, vpnIp netip.Addr, hostinfo *Ho
 	}
 
 	hostinfo.logger(rm.l).Info("Attempt to relay through hosts", "relays", hostinfo.remotes.relays)
-	// One WireBuffer for the whole relay-fanout loop.
-	buf := f.bufAlloc.Acquire()
-	defer f.bufAlloc.Release(buf)
 	// Send a RelayRequest to all known Relay IP's
 	for _, relay := range hostinfo.remotes.relays {
 		// Don't relay through the host I'm trying to connect to
@@ -127,7 +124,7 @@ func (rm *relayManager) StartRelays(f *Interface, vpnIp netip.Addr, hostinfo *Ho
 				if err != nil {
 					hostinfo.logger(rm.l).Error("Failed to marshal Control message to create relay", "error", err)
 				} else {
-					f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, buf)
+					f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, make([]byte, 12), make([]byte, mtu))
 					rm.l.Info("send CreateRelayRequest",
 						"relayFrom", f.myVpnAddrs[0],
 						"relayTo", vpnIp,
@@ -142,7 +139,7 @@ func (rm *relayManager) StartRelays(f *Interface, vpnIp netip.Addr, hostinfo *Ho
 		switch existingRelay.State {
 		case Established:
 			hostinfo.logger(rm.l).Info("Send handshake via relay", "relay", relay.String())
-			f.SendVia(relayHostInfo, existingRelay, stage0, buf)
+			f.SendVia(relayHostInfo, existingRelay, stage0, make([]byte, 12), make([]byte, mtu), false)
 		case Disestablished:
 			// Mark this relay as 'requested'
 			relayHostInfo.relayState.UpdateRelayForByIpState(vpnIp, Requested)
@@ -183,7 +180,7 @@ func (rm *relayManager) StartRelays(f *Interface, vpnIp netip.Addr, hostinfo *Ho
 				hostinfo.logger(rm.l).Error("Failed to marshal Control message to create relay", "error", err)
 			} else {
 				// This must send over the hostinfo, not over hm.Hosts[ip]
-				f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, buf)
+				f.SendMessageToHostInfo(header.Control, 0, relayHostInfo, msg, make([]byte, 12), make([]byte, mtu))
 				rm.l.Info("send CreateRelayRequest",
 					"relayFrom", f.myVpnAddrs[0],
 					"relayTo", vpnIp,
@@ -371,9 +368,7 @@ func (rm *relayManager) handleCreateRelayResponse(v cert.Version, h *HostInfo, f
 		if err != nil {
 			rm.l.Error("relayManager Failed to marshal Control CreateRelayResponse message to create relay", "error", err)
 		} else {
-			buf := f.bufAlloc.Acquire()
+			f.SendMessageToHostInfo(header.Control, 0, peerHostInfo, msg, make([]byte, 12), make([]byte, mtu))
-			f.SendMessageToHostInfo(header.Control, 0, peerHostInfo, msg, buf)
-			f.bufAlloc.Release(buf)
 			rm.l.Info("send CreateRelayResponse",
 				"relayFrom", resp.RelayFromAddr,
 				"relayTo", resp.RelayToAddr,
@@ -473,9 +468,7 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
 		if err != nil {
 			logMsg.Error("relayManager Failed to marshal Control CreateRelayResponse message to create relay", "error", err)
 		} else {
-			buf := f.bufAlloc.Acquire()
+			f.SendMessageToHostInfo(header.Control, 0, h, msg, make([]byte, 12), make([]byte, mtu))
-			f.SendMessageToHostInfo(header.Control, 0, h, msg, buf)
-			f.bufAlloc.Release(buf)
 			rm.l.Info("send CreateRelayResponse",
 				"relayFrom", from,
 				"relayTo", target,
@@ -545,9 +538,7 @@ func (rm *relayManager) handleCreateRelayRequest(v cert.Version, h *HostInfo, f
 		if err != nil {
 			logMsg.Error("relayManager Failed to marshal Control message to create relay", "error", err)
 		} else {
-			buf := f.bufAlloc.Acquire()
+			f.SendMessageToHostInfo(header.Control, 0, peer, msg, make([]byte, 12), make([]byte, mtu))
-			f.SendMessageToHostInfo(header.Control, 0, peer, msg, buf)
-			f.bufAlloc.Release(buf)
 			rm.l.Info("send CreateRelayRequest",
 				"relayFrom", h.vpnAddrs[0],
 				"relayTo", target,

ssh.go

@@ -632,9 +632,15 @@ func sshCloseTunnel(ifce *Interface, fs any, a []string, w sshd.StringWriter) er
 	}
 
 	if !flags.LocalOnly {
-		buf := ifce.bufAlloc.Acquire()
+		ifce.send(
-		ifce.send(header.CloseTunnel, 0, hostInfo.ConnectionState, hostInfo, []byte{}, buf)
+			header.CloseTunnel,
-		ifce.bufAlloc.Release(buf)
+			0,
+			hostInfo.ConnectionState,
+			hostInfo,
+			[]byte{},
+			make([]byte, 12, 12),
+			make([]byte, mtu),
+		)
 	}
 
 	ifce.closeTunnel(hostInfo)

udp/conn.go

@@ -20,6 +20,12 @@ type Conn interface {
 	WriteTo(b []byte, addr netip.AddrPort) error
 	ReloadConfig(c *config.C)
 	SupportsMultipleReaders() bool
+	// EnablePathMTUDiscovery sets the don't-fragment bit on outgoing packets for
+	// this socket. Called by the pmtud manager when PMTUD is enabled. A no-op on
+	// platforms that don't support it; nebula's default behavior (no DF, kernel
+	// fragmentation allowed) is preserved on those platforms and on this one when
+	// PMTUD is disabled.
+	EnablePathMTUDiscovery() error
 	Close() error
 }
 
@@ -43,6 +49,9 @@ func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error {
 func (NoopConn) ReloadConfig(_ *config.C) {
 	return
 }
+func (NoopConn) EnablePathMTUDiscovery() error {
+	return nil
+}
 func (NoopConn) Close() error {
 	return nil
 }

udp/udp_android.go

@@ -44,3 +44,17 @@ func NewListenConfig(multi bool) net.ListenConfig {
 func (u *GenericConn) Rebind() error {
 	return nil
 }
+
+// EnablePathMTUDiscovery sets the don't-fragment bit on outbound packets.
+// Android is Linux underneath, so we use IP_PMTUDISC_PROBE (kernel sets DF but
+// does not consume incoming ICMP frag-needed for its PMTU cache; the manager
+// drives discovery via authenticated probes).
+func (u *GenericConn) EnablePathMTUDiscovery() error {
+	v4 := u.isV4Socket()
+	return u.controlFD(func(fd uintptr) error {
+		if v4 {
+			return unix.SetsockoptInt(int(fd), unix.IPPROTO_IP, unix.IP_MTU_DISCOVER, unix.IP_PMTUDISC_PROBE)
+		}
+		return unix.SetsockoptInt(int(fd), unix.IPPROTO_IPV6, unix.IPV6_MTU_DISCOVER, unix.IPV6_PMTUDISC_PROBE)
+	})
+}

udp/udp_bsd.go

@@ -47,3 +47,8 @@ func NewListenConfig(multi bool) net.ListenConfig {
 func (u *GenericConn) Rebind() error {
 	return nil
 }
+
+// EnablePathMTUDiscovery is split into per-OS files: udp_freebsd.go handles
+// FreeBSD (which has both IP_DONTFRAG and IPV6_DONTFRAG in the unix package);
+// udp_openbsd.go handles OpenBSD (v6 only; the kernel doesn't expose a v4 DF
+// sockopt).

udp/udp_darwin.go

@@ -187,6 +187,17 @@ func (u *StdConn) SupportsMultipleReaders() bool {
 	return false
 }
 
+// EnablePathMTUDiscovery sets the don't-fragment bit on every outbound packet.
+// On darwin we use IP_DONTFRAG (v4) / IPV6_DONTFRAG (v6). The kernel will return
+// EMSGSIZE for sends that exceed the local interface MTU; ICMP-driven PMTU
+// updates from upstream routers are processed by the kernel as usual.
+func (u *StdConn) EnablePathMTUDiscovery() error {
+	if u.isV4 {
+		return syscall.SetsockoptInt(int(u.sysFd), syscall.IPPROTO_IP, unix.IP_DONTFRAG, 1)
+	}
+	return syscall.SetsockoptInt(int(u.sysFd), syscall.IPPROTO_IPV6, unix.IPV6_DONTFRAG, 1)
+}
+
 func (u *StdConn) Rebind() error {
 	var err error
 	if u.isV4 {

udp/udp_freebsd.go

@@ -0,0 +1,25 @@
+//go:build freebsd && !e2e_testing
+// +build freebsd,!e2e_testing
+
+package udp
+
+import (
+	"golang.org/x/sys/unix"
+)
+
+// EnablePathMTUDiscovery sets the don't-fragment bit on outbound packets.
+// FreeBSD exposes IP_DONTFRAG (v4) and IPV6_DONTFRAG (v6) in golang.org/x/sys/unix.
+// Unlike Linux, BSDs don't have an explicit "don't consume incoming ICMP
+// frag-needed" knob for unconnected UDP sockets; the kernel's PMTU cache will
+// be updated from ICMP, which is benign for our usage (the cache only affects
+// what EMSGSIZE gets surfaced for; the manager drives its own discovery via
+// authenticated probes).
+func (u *GenericConn) EnablePathMTUDiscovery() error {
+	v4 := u.isV4Socket()
+	return u.controlFD(func(fd uintptr) error {
+		if v4 {
+			return unix.SetsockoptInt(int(fd), unix.IPPROTO_IP, unix.IP_DONTFRAG, 1)
+		}
+		return unix.SetsockoptInt(int(fd), unix.IPPROTO_IPV6, unix.IPV6_DONTFRAG, 1)
+	})
+}

udp/udp_generic.go

@@ -100,3 +100,41 @@ func (u *GenericConn) ListenOut(r EncReader) error {
 func (u *GenericConn) SupportsMultipleReaders() bool {
 	return false
 }
+
+// EnablePathMTUDiscovery is implemented per-platform alongside Rebind, in
+// udp_android.go / udp_bsd.go / udp_netbsd.go / udp_windows.go.
+
+// controlFD invokes f with the underlying UDP socket file descriptor (or
+// handle, on Windows). Used by platform files for setsockopt calls that the
+// stdlib net.UDPConn does not expose directly.
+func (u *GenericConn) controlFD(f func(fd uintptr) error) error {
+	rc, err := u.UDPConn.SyscallConn()
+	if err != nil {
+		return err
+	}
+	var sockErr error
+	err = rc.Control(func(fd uintptr) {
+		sockErr = f(fd)
+	})
+	if err != nil {
+		return err
+	}
+	return sockErr
+}
+
+// isV4Socket reports whether the local bind address looks like an IPv4 socket.
+// Used by EnablePathMTUDiscovery to pick IPPROTO_IP vs IPPROTO_IPV6 socket
+// options. Assumes pure-v4 or pure-v6 sockets; a dual-stack v6 socket bound to
+// :: will be treated as v6 (correct: setting IPV6_DONTFRAG covers v4-mapped
+// traffic too on most stacks).
+func (u *GenericConn) isV4Socket() bool {
+	la := u.UDPConn.LocalAddr()
+	if la == nil {
+		return false
+	}
+	ua, ok := la.(*net.UDPAddr)
+	if !ok {
+		return false
+	}
+	return ua.IP.To4() != nil
+}

udp/udp_linux.go

@@ -73,6 +73,21 @@ func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int)
 	return out, nil
 }
 
+// EnablePathMTUDiscovery sets IP_MTU_DISCOVER=IP_PMTUDISC_PROBE (IPV6 equivalent
+// for v6 sockets). This sets the don't-fragment bit on every outbound packet but
+// tells the kernel not to consume incoming ICMP frag-needed for its own PMTU
+// cache; we drive PMTU discovery from the application via authenticated probes
+// (RFC 8899). Called by the pmtud manager when PMTUD is enabled. Without this
+// call the socket retains nebula's historical behavior (no DF, kernel may
+// fragment), preserving compatibility with deployments that depend on UDP
+// fragmentation.
+func (u *StdConn) EnablePathMTUDiscovery() error {
+	if u.isV4 {
+		return u.setSockOptIPInt(unix.IPPROTO_IP, unix.IP_MTU_DISCOVER, unix.IP_PMTUDISC_PROBE)
+	}
+	return u.setSockOptIPInt(unix.IPPROTO_IPV6, unix.IPV6_MTU_DISCOVER, unix.IPV6_PMTUDISC_PROBE)
+}
+
 func (u *StdConn) SupportsMultipleReaders() bool {
 	return true
 }
@@ -110,6 +125,21 @@ func (u *StdConn) setSockOptInt(opt int, n int) error {
 	return opErr
 }
 
+// setSockOptIPInt sets a socket option at a non-SOL_SOCKET level (e.g. IPPROTO_IP).
+func (u *StdConn) setSockOptIPInt(level, opt, n int) error {
+	if u.rawConn == nil {
+		return fmt.Errorf("no UDP connection")
+	}
+	var opErr error
+	err := u.rawConn.Control(func(fd uintptr) {
+		opErr = unix.SetsockoptInt(int(fd), level, opt, n)
+	})
+	if err != nil {
+		return err
+	}
+	return opErr
+}
+
 func (u *StdConn) SetRecvBuffer(n int) error {
 	return u.setSockOptInt(unix.SO_RCVBUFFORCE, n)
 }

udp/udp_netbsd.go

@@ -46,3 +46,18 @@ func NewListenConfig(multi bool) net.ListenConfig {
 func (u *GenericConn) Rebind() error {
 	return nil
 }
+
+// EnablePathMTUDiscovery sets the don't-fragment bit on outbound packets.
+// NetBSD exposes IPV6_DONTFRAG via golang.org/x/sys/unix but the kernel does
+// not provide a socket-level knob for setting DF on v4 UDP. The only IP-layer
+// constant exposed is IP_DF, which is the wire header flag, not a sockopt.
+// quic-go skips NetBSD for the same reason. So v4 sockets stay at nebula's
+// historical behavior (kernel may fragment); v6 gets DF.
+func (u *GenericConn) EnablePathMTUDiscovery() error {
+	if u.isV4Socket() {
+		return nil
+	}
+	return u.controlFD(func(fd uintptr) error {
+		return unix.SetsockoptInt(int(fd), unix.IPPROTO_IPV6, unix.IPV6_DONTFRAG, 1)
+	})
+}

udp/udp_openbsd.go

@@ -0,0 +1,23 @@
+//go:build openbsd && !e2e_testing
+// +build openbsd,!e2e_testing
+
+package udp
+
+import (
+	"golang.org/x/sys/unix"
+)
+
+// EnablePathMTUDiscovery sets the don't-fragment bit on outbound packets.
+// OpenBSD exposes IPV6_DONTFRAG via golang.org/x/sys/unix but the kernel does
+// not provide a socket-level knob for setting DF on v4 UDP. The only IP-layer
+// constant exposed is IP_DF, which is the wire header flag, not a sockopt.
+// quic-go skips OpenBSD for the same reason. So v4 sockets stay at nebula's
+// historical behavior (kernel may fragment); v6 gets DF.
+func (u *GenericConn) EnablePathMTUDiscovery() error {
+	if u.isV4Socket() {
+		return nil
+	}
+	return u.controlFD(func(fd uintptr) error {
+		return unix.SetsockoptInt(int(fd), unix.IPPROTO_IPV6, unix.IPV6_DONTFRAG, 1)
+	})
+}

udp/udp_rio_windows.go

@@ -335,6 +335,12 @@ func (u *RIOConn) Rebind() error {
 	return nil
 }
 
+// EnablePathMTUDiscovery is a no-op on Windows for now. PMTUD is Linux-only in
+// the initial PoC; Windows support would set IP_DONTFRAGMENT here.
+func (u *RIOConn) EnablePathMTUDiscovery() error {
+	return nil
+}
+
 func (u *RIOConn) ReloadConfig(*config.C) {}
 
 func (u *RIOConn) Close() error {

udp/udp_tester.go

@@ -21,48 +21,17 @@ type Packet struct {
 	Data []byte
 }
 
-// Copy returns a fresh *Packet (from the freelist) with a duplicate Data buffer.
 func (u *Packet) Copy() *Packet {
-	n := acquirePacket()
+	n := &Packet{
-	n.To = u.To
+		To:   u.To,
-	n.From = u.From
+		From: u.From,
-	if cap(n.Data) < len(u.Data) {
+		Data: make([]byte, len(u.Data)),
-		n.Data = make([]byte, len(u.Data))
-	} else {
-		n.Data = n.Data[:len(u.Data)]
 	}
+
 	copy(n.Data, u.Data)
 	return n
 }
 
-// Release returns p to the harness packet freelist.
-// Callers that pull a *Packet from Get / TxPackets must Release when done.
-// Channel-backed instead of sync.Pool because sync.Pool's per-P caches drain badly under cross-goroutine Get/Put,
-// and putting a []byte in a Pool escapes the slice header to heap.
-func (p *Packet) Release() {
-	if p == nil {
-		return
-	}
-	p.Data = p.Data[:0]
-	select {
-	case packetFreelist <- p:
-	default:
-		// Freelist full; drop the *Packet for the GC.
-	}
-}
-
-// packetFreelist retains *Packet structs (and their backing Data arrays) so steady-state allocation drops to zero.
-var packetFreelist = make(chan *Packet, 64)
-
-func acquirePacket() *Packet {
-	select {
-	case p := <-packetFreelist:
-		return p
-	default:
-		return &Packet{}
-	}
-}
-
 type TesterConn struct {
 	Addr netip.AddrPort
 
@@ -95,15 +64,13 @@ func NewListener(l *slog.Logger, ip netip.Addr, port int, _ bool, _ int) (Conn,
 // this is an encrypted packet or a handshake message in most cases
 // packets were transmitted from another nebula node, you can send them with Tun.Send
 func (u *TesterConn) Send(packet *Packet) {
-	if u.l.Enabled(context.Background(), slog.LevelDebug) {
+	h := &header.H{}
-		// Parse the header only under debug logging, otherwise the
-		// allocation would show up in every Send call.
-		var h header.H
 	if err := h.Parse(packet.Data); err != nil {
 		panic(err)
 	}
+	if u.l.Enabled(context.Background(), slog.LevelDebug) {
 		u.l.Debug("UDP receiving injected packet",
-			"header", &h,
+			"header", h,
 			"udpAddr", packet.From,
 			"dataLen", len(packet.Data),
 		)
@@ -140,18 +107,15 @@ func (u *TesterConn) Get(block bool) *Packet {
 //********************************************************************************************************************//
 
 func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
-	p := acquirePacket()
+	p := &Packet{
-	if cap(p.Data) < len(b) {
+		Data: make([]byte, len(b), len(b)),
-		p.Data = make([]byte, len(b))
+		From: u.Addr,
-	} else {
+		To:   addr,
-		p.Data = p.Data[:len(b)]
 	}
+
 	copy(p.Data, b)
-	p.From = u.Addr
-	p.To = addr
 	select {
 	case <-u.done:
-		p.Release()
 		return io.ErrClosedPipe
 	case u.TxPackets <- p:
 		return nil
@@ -165,7 +129,6 @@ func (u *TesterConn) ListenOut(r EncReader) error {
 			return os.ErrClosed
 		case p := <-u.RxPackets:
 			r(p.From, p.Data)
-			p.Release()
 		}
 	}
 }
@@ -189,6 +152,10 @@ func (u *TesterConn) Rebind() error {
 	return nil
 }
 
+func (u *TesterConn) EnablePathMTUDiscovery() error {
+	return nil
+}
+
 func (u *TesterConn) Close() error {
 	u.closeOnce.Do(func() {
 		close(u.done)

udp/udp_windows.go

@@ -9,6 +9,8 @@ import (
 	"net"
 	"net/netip"
 	"syscall"
+
+	"golang.org/x/sys/windows"
 )
 
 func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
@@ -44,3 +46,27 @@ func NewListenConfig(multi bool) net.ListenConfig {
 func (u *GenericConn) Rebind() error {
 	return nil
 }
+
+// Windows IP_DONTFRAGMENT and IPV6_DONTFRAG are not exposed in the
+// golang.org/x/sys/windows package. Defined locally per the values in
+// ws2ipdef.h / ws2tcpip.h. These are stable Win32 constants that have not
+// changed since at least Windows Vista.
+const (
+	winIPDontFragment = 14
+	winIPv6DontFrag   = 14
+)
+
+// EnablePathMTUDiscovery sets the don't-fragment bit on outbound packets.
+// Windows uses IP_DONTFRAGMENT (v4) and IPV6_DONTFRAG (v6) at IPPROTO_IP /
+// IPPROTO_IPV6 respectively. Note: this only enables DF on the GenericConn
+// fallback path. The RIO path (RIOConn) has its own EnablePathMTUDiscovery
+// in udp_rio_windows.go and is currently a no-op pending RIO-specific work.
+func (u *GenericConn) EnablePathMTUDiscovery() error {
+	v4 := u.isV4Socket()
+	return u.controlFD(func(fd uintptr) error {
+		if v4 {
+			return windows.SetsockoptInt(windows.Handle(fd), windows.IPPROTO_IP, winIPDontFragment, 1)
+		}
+		return windows.SetsockoptInt(windows.Handle(fd), windows.IPPROTO_IPV6, winIPv6DontFrag, 1)
+	})
+}

wire_buffer.go

@@ -1,255 +0,0 @@
-package nebula
-
-import (
-	"io"
-	"sync"
-
-	"github.com/slackhq/nebula/firewall"
-	"github.com/slackhq/nebula/header"
-	"github.com/slackhq/nebula/noiseutil"
-	"github.com/slackhq/nebula/overlay"
-)
-
-// WireBuffer is the per-goroutine working set for processing one IP packet
-// through the data plane. It owns:
-//
-//   - The IP-payload byte buffer used to hold the current inbound or
-//     outbound packet, with prefixLen bytes of slack at the front for
-//     the BSD AF_INET protocol-family marker.
-//   - The fwPacket scratch parsed by newPacket().
-//   - The 12-byte AEAD nonce scratch.
-//   - The header.H parse target used by the receive path.
-//   - An mtu-sized wire-output scratch for sendNoMetrics and for building
-//     reject packets.
-//
-// One WireBuffer is allocated per data-plane goroutine (listenIn for the
-// TUN-side, listenOut for the UDP-side) and reused for every packet. No
-// per-packet allocation. Future GRO/GSO/TSO and reliable-transport work
-// will likely extend this to carry batch state and fragment metadata.
-//
-// The TUN protocol-family prefix is handled here, not in the overlay
-// package. On BSDs the kernel writes the 4-byte marker into the slack on
-// read, and we stamp it into the slack before write. On linux/windows
-// /userspace devices prefixLen is 0 and the slack is empty.
-type WireBuffer struct {
-	// FwPacket is the parsed IP packet metadata (5-tuple, fragment flags,
-	// etc.) populated by newPacket().
-	FwPacket *firewall.Packet
-	// NB is a 12-byte scratch the AEAD uses for the nonce; reused so we
-	// don't allocate one per encrypt/decrypt.
-	NB []byte
-	// H is the parse target for inbound nebula headers. Receive path only.
-	H *header.H
-	// Out is an mtu-sized wire-output scratch passed to sendNoMetrics and
-	// rejectInside / rejectOutside. Sized to fit any single wire packet.
-	Out []byte
-
-	// ip is the IP-payload region: a slice of len 0, cap linkMTU sliced
-	// from raw at offset prefixLen. The current packet (if any) is
-	// ip[:bodyN]. The TUN prefix slack lives at raw[0:prefixLen] just
-	// before ip.
-	ip []byte
-	// raw is the backing slab. Layout:
-	//   [prefixLen bytes prefix slack | linkMTU bytes IP region | outSize bytes Out scratch]
-	// Holding it lets ReadIPFromTUN / WriteIPToTUN address the slack
-	// region directly.
-	raw       []byte
-	prefixLen int
-	bodyN     int
-}
-
-// NewWireBuffer returns a buffer sized to hold any single IP packet up to
-// linkMTU, plus a disjoint wire-output scratch sliced from the same backing
-// slab (the AEAD's Seal contract requires plaintext and dst not to partially
-// overlap, and keeping them in one slab gives a single allocation per
-// goroutine). Out is sized for the relay worst case
-// (linkMTU + 2*header.Len + 2*AEADOverhead).
-//
-// prefixLen is the number of bytes the destination tun device prepends/
-// expects on each IP packet (overlay.Device.TunPrefixLen). On BSDs this
-// is 4 (AF_INET marker); on linux/windows/userspace devices it is 0.
-func NewWireBuffer(linkMTU, prefixLen int) *WireBuffer {
-	outSize := linkMTU + 2*header.Len + 2*AEADOverhead
-	raw := make([]byte, prefixLen+linkMTU+outSize)
-	outStart := prefixLen + linkMTU
-	return &WireBuffer{
-		FwPacket:  &firewall.Packet{},
-		NB:        make([]byte, NonceSize),
-		H:         &header.H{},
-		Out:       raw[outStart : outStart : outStart+outSize],
-		ip:        raw[prefixLen:prefixLen:outStart],
-		raw:       raw,
-		prefixLen: prefixLen,
-	}
-}
-
-// Reset clears the body-length record so the buffer is ready for another
-// recv (e.g. relay-receive recursion before a nested decrypt).
-func (b *WireBuffer) Reset() { b.bodyN = 0 }
-
-// IPPacket returns the IP packet currently held in the payload region (after
-// a successful ReadIPFromTUN or DecryptDatagram). The slice aliases the
-// buffer; do not retain past the next operation.
-func (b *WireBuffer) IPPacket() []byte {
-	return b.ip[:b.bodyN]
-}
-
-// Seal stamps a nebula header at the front of buf.Out and AEAD-seals p as the
-// payload, treating the header as additional authenticated data. The lock
-// scope around counter increment + encrypt matches what goboring AESGCMTLS
-// requires; non-boring builds skip the lock.
-//
-// Returns the wire bytes (header || ciphertext || tag), aliased to buf.Out.
-// The slice is invalidated by the next Seal* call on this buffer.
-func (b *WireBuffer) Seal(ci *ConnectionState, t header.MessageType, st header.MessageSubType, remoteIndex uint32, p []byte) ([]byte, error) {
-	return b.sealInto(b.Out[:cap(b.Out)], ci, t, st, remoteIndex, p)
-}
-
-// SealForRelay is like Seal but reserves header.Len bytes of slack at the front
-// of buf.Out for an outer relay header. The inner header + ciphertext lands at
-// offset header.Len so a follow-up SealRelayInPlace can stamp the outer header
-// without copying. Use this when the caller may need to wrap the result in a
-// relay envelope after the fact.
-func (b *WireBuffer) SealForRelay(ci *ConnectionState, t header.MessageType, st header.MessageSubType, remoteIndex uint32, p []byte) ([]byte, error) {
-	return b.sealInto(b.Out[header.Len:cap(b.Out)], ci, t, st, remoteIndex, p)
-}
-
-func (b *WireBuffer) sealInto(out []byte, ci *ConnectionState, t header.MessageType, st header.MessageSubType, remoteIndex uint32, p []byte) ([]byte, error) {
-	if noiseutil.EncryptLockNeeded {
-		ci.writeLock.Lock()
-	}
-	c := ci.messageCounter.Add(1)
-	out = header.Encode(out, header.Version, t, st, remoteIndex, c)
-	out, err := ci.eKey.EncryptDanger(out, out, p, c, b.NB)
-	if noiseutil.EncryptLockNeeded {
-		ci.writeLock.Unlock()
-	}
-	return out, err
-}
-
-// SealRelayInPlace wraps an inner message that is already staged at
-// buf.Out[header.Len:header.Len+innerLen] (either from a SealForRelay encrypt
-// or from a copy via the SendVia entry point). It stamps the outer relay
-// header into buf.Out[:header.Len] and AAD-only seals over the entire region,
-// producing the wire bytes for the relay tunnel.
-//
-// Returns the wire bytes aliased to buf.Out; invalidated by the next Seal*
-// call on this buffer.
-func (b *WireBuffer) SealRelayInPlace(ci *ConnectionState, remoteIndex uint32, innerLen int) ([]byte, error) {
-	if noiseutil.EncryptLockNeeded {
-		ci.writeLock.Lock()
-	}
-	c := ci.messageCounter.Add(1)
-	out := b.Out[:cap(b.Out)]
-	out = header.Encode(out, header.Version, header.Message, header.MessageRelay, remoteIndex, c)
-	out = out[:header.Len+innerLen]
-	out, err := ci.eKey.EncryptDanger(out, out, nil, c, b.NB)
-	if noiseutil.EncryptLockNeeded {
-		ci.writeLock.Unlock()
-	}
-	return out, err
-}
-
-// StageRelayInner copies ad into the inner-payload slot at buf.Out[header.Len:]
-// so SealRelayInPlace can wrap it on the next call. Used by SendVia when ad
-// did not come from a prior SealForRelay (e.g. a handshake message being
-// forwarded through a relay tunnel without our own encryption).
-func (b *WireBuffer) StageRelayInner(ad []byte) int {
-	return copy(b.Out[header.Len:cap(b.Out)], ad)
-}
-
-// ReadIPFromTUN reads one IP packet from r into the payload region and
-// updates bodyN. On BSDs the kernel writes its 4-byte protocol-family
-// marker into the slack at raw[0:prefixLen] and the IP packet at
-// raw[prefixLen:prefixLen+n]; we hand it the slack-prefixed slice so
-// the kernel can do this in one syscall with no copy. On linux/windows/
-// userspace devices prefixLen is 0 and the slack is empty.
-func (b *WireBuffer) ReadIPFromTUN(r io.Reader) (int, error) {
-	n, err := r.Read(b.raw[:b.prefixLen+cap(b.ip)])
-	if err != nil {
-		b.bodyN = 0
-		return 0, err
-	}
-	if n < b.prefixLen {
-		b.bodyN = 0
-		return 0, nil
-	}
-	b.bodyN = n - b.prefixLen
-	return b.bodyN, nil
-}
-
-// WriteIPToTUN writes the IP packet currently in the payload region to w.
-// On BSDs we stamp the protocol-family marker into the slack at
-// raw[0:prefixLen] in place and write the entire slack+IP region in a
-// single syscall, so the kernel sees [marker][ip] back to back without a
-// userspace copy. On linux/windows/userspace devices the slack is empty
-// and we just write the IP region.
-func (b *WireBuffer) WriteIPToTUN(w io.Writer) (int, error) {
-	out := b.raw[:b.prefixLen+b.bodyN]
-	if b.prefixLen > 0 {
-		if err := overlay.StampTunPrefix(out); err != nil {
-			return 0, err
-		}
-	}
-	return w.Write(out)
-}
-
-// DecryptDatagram decrypts an inbound UDP packet into the payload region.
-func (b *WireBuffer) DecryptDatagram(ci *ConnectionState, packet []byte, mc uint64) error {
-	dst, err := ci.dKey.DecryptDanger(b.ip[:0], packet[:header.Len], packet[header.Len:], mc, b.NB)
-	if err != nil {
-		b.bodyN = 0
-		return err
-	}
-	b.bodyN = len(dst)
-	return nil
-}
-
-// DecryptForHandler decrypts an inbound UDP packet (lighthouse, test,
-// control, close-tunnel) into the payload region and returns the plaintext
-// slice for the in-process handler. Returned slice aliases the buffer.
-func (b *WireBuffer) DecryptForHandler(ci *ConnectionState, packet []byte, mc uint64) ([]byte, error) {
-	dst, err := ci.dKey.DecryptDanger(b.ip[:0], packet[:header.Len], packet[header.Len:], mc, b.NB)
-	if err != nil {
-		b.bodyN = 0
-		return nil, err
-	}
-	b.bodyN = len(dst)
-	return dst, nil
-}
-
-// WireBufferAllocator hands out reusable WireBuffers for cold callers that
-// don't own a long-lived per-goroutine buffer (control plane, relay manager,
-// connection manager teardown, etc.). Hot-path goroutines hold their own
-// buffer for the life of the goroutine and don't need to acquire one.
-type WireBufferAllocator interface {
-	Acquire() *WireBuffer
-	Release(*WireBuffer)
-}
-
-// wireBufferPool is a sync.Pool-backed WireBufferAllocator. The pool is
-// keyed off a single linkMTU and prefixLen; cold callers send across the
-// data-plane mtu and target the same Device, so we size the pool's
-// buffers the same way.
-type wireBufferPool struct {
-	pool sync.Pool
-}
-
-func NewWireBufferPool(linkMTU, prefixLen int) *wireBufferPool {
-	return &wireBufferPool{
-		pool: sync.Pool{
-			New: func() any {
-				return NewWireBuffer(linkMTU, prefixLen)
-			},
-		},
-	}
-}
-
-func (p *wireBufferPool) Acquire() *WireBuffer {
-	return p.pool.Get().(*WireBuffer)
-}
-
-func (p *wireBufferPool) Release(b *WireBuffer) {
-	b.Reset()
-	p.pool.Put(b)
-}