Do not warn about local address failures on every update

localAddrs runs inside every SendUpdate, which fires on lighthouse.interval
and again on every network change via RebindUDPServer. Logging the failure
there meant a persistent failure warned every interval for the life of the
process, once per failing interface.

collectLocalAddrs now returns its failures instead of logging them, which
keeps it stateless and lets the tests assert on errors rather than log
output. The lighthouse holds the previous error and warns only when it
changes, demoting repeats to Debug, matching how handshake_manager handles
repeated send failures.

.github/workflows/release.yml

@@ -12,9 +12,9 @@ jobs:
     steps:
       - uses: actions/checkout@v7
 
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v7
         with:
-          go-version: '1.25'
+          go-version: '1.26'
           check-latest: true
 
       - name: Build
@@ -38,9 +38,9 @@ jobs:
     steps:
       - uses: actions/checkout@v7
 
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v7
         with:
-          go-version: '1.25'
+          go-version: '1.26'
           check-latest: true
 
       - name: Build
@@ -78,9 +78,9 @@ jobs:
     steps:
       - uses: actions/checkout@v7
 
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v7
         with:
-          go-version: '1.25'
+          go-version: '1.26'
           check-latest: true
 
       - name: Import certificates

.github/workflows/smoke-extra.yml

@@ -32,9 +32,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     - name: add hashicorp source
@@ -64,9 +64,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     - name: add hashicorp source
@@ -90,9 +90,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     # WSL2 + Ubuntu so the smoke can run a real linux peer with its own

.github/workflows/smoke.yml

@@ -20,9 +20,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     - name: build

.github/workflows/test.yml

@@ -20,9 +20,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     - name: Install goimports
@@ -42,7 +42,7 @@ jobs:
     - name: golangci-lint
       uses: golangci/golangci-lint-action@v9
       with:
-        version: v2.5
+        version: v2.12
 
   test:
     name: Test ${{ matrix.name }}
@@ -80,9 +80,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     - name: Build
@@ -125,9 +125,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     - name: Build ${{ matrix.name }}

CHANGELOG.md

@@ -7,6 +7,88 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.11.0] - 2026-07-23
+
+See the [v1.11.0](https://github.com/slackhq/nebula/milestone/25?closed=1) milestone for a complete list of changes.
+
+### Breaking
+
+- Logging has switched from logrus to Go's structured `slog`. Log output changes: levels are upper case
+  (`level=INFO`), trace prints as `level=DEBUG-4`, timestamps are always RFC3339Nano and `logging.timestamp_format`
+  is ignored, and some messages were reworded. Review any log parsing before upgrading. This is also an API break
+  for embedders, as constructors now take a `*slog.Logger`. (#1672, #1734, #1621)
+- `firewall.inbound_action` and `firewall.outbound_action` (used to set reject vs. drop policy) were each being
+  applied to the opposite direction, that is now corrected. This only affects how blocked packets are answered, not
+  which packets the firewall allows or denies. If you set either of these you are getting the behavior of the other
+  one today and likely want to swap them before upgrading. (#1798)
+- On Windows, Nebula now installs WFP PERMIT filters for the nebula adapter and the listener port by default. WFP
+  sits below Windows Defender Firewall, so any WDF inbound rules you rely on for either will no longer apply. Set
+  `tun.windows_bypass_wdf` and `listen.windows_bypass_wdf` to false to leave WDF in charge. (#1710)
+- On Windows, the nebula device is now set to the `private` network category instead of whatever Windows decided,
+  which is usually `Public`. This makes the host firewall less restrictive on the overlay. Set
+  `tun.network_category` to `unset` to keep the old behavior. (#1710)
+- Reject packets for non-TCP now use ICMP code 13, communication administratively prohibited, instead of code 3,
+  port unreachable. Anything keying off the old code needs updating. (#1766, #1768)
+- The SSH debug server's profiling commands are now confined to `sshd.sandbox_dir`, which defaults to
+  `$TMP/nebula-debug`. Relative paths resolve inside it and absolute paths outside it are rejected, so anything
+  scripting `start-cpu-profile`, `save-heap-profile`, or `save-mutex-profile` with a path elsewhere needs the
+  directory set. The directory is not created for you. (#1622)
+
+### Added
+
+- Sign the Windows release binaries. (#1718)
+- Generate IPv6 reject packets, matching the existing IPv4 behavior. (#1766, #1767, #1768)
+- Accept `-` in `nebula-cert` to read from stdin or write to stdout. (#1714)
+- Search for both `config.yml` and `config.yaml` in service and command line modes. (#1717)
+- Add version labels to the Docker/OCI images. (#1772)
+- Rebind the listener and re-query lighthouses on macOS when the underlay network changes, so devices moving
+  between wifi and wired or between networks recover without waiting for dead tunnel detection. Controlled by
+  `listen.rebind_on_network_change` (default `true`, not reloadable). (#1816)
+
+### Changed
+
+- Reload the firewall when the unsafe networks in the certificate change. (#1719)
+- Reconfigure, start, and stop the stats listener on a config reload instead of requiring a restart. (#1670)
+- Update a static host's addresses when they change on reload. (#1713)
+- Don't require a port on ICMP firewall rules. (#1609)
+- Connection track ICMP traffic. (#1602)
+- Return `NODATA` instead of `NXDOMAIN` from the DNS server for a name that exists but has no record of the
+  requested type, so clients that query `AAAA` first (busybox/Alpine) fall through to `A`. (#1668)
+- Record the local host's details in the DNS server. (#1716)
+- Install Windows unsafe routes as link routes. (#1709)
+- Reduce relay handshake log spam, and only log a handshake send error at error level when the remote list
+  changes. (#1733, #1765, #1810)
+- Start, stop, and reload subsystems (DNS, stats, conntrack, ssh, punchy) cleanly without leaking goroutines. (#1640, #1654, #1661, #1667, #1669, #1708, #1806, #1815)
+- `Control` is now safe to stop and wait on from any lifecycle state, and a new `Control.Wait` blocks until nebula
+  has fully stopped and returns the first fatal reader error. Failed starts release the udp sockets and tun fd
+  instead of leaking them. (#1794)
+- Trigger an immediate lighthouse update when reconnecting to or adding a lighthouse instead of waiting for the next update tick. (#1645)
+- Bring the Darwin and OpenBSD tun implementations in line with the other BSDs. (#1703)
+- Update to build against go v1.26. (#1818)
+- Various dependency updates. (#1586, #1587, #1604, #1617, #1618, #1627, #1628, #1629, #1652, #1664, #1665, #1697, #1721, #1732, #1742, #1743, #1750, #1763, #1771, #1782, #1800, #1807)
+
+### Fixed
+
+- Fix a data race on a host's remote address that could send packets to the wrong address during a roam. (#1773)
+- Fix tunnels that could permanently escape connection manager monitoring. (#1752)
+- Fix a crash when reloading the SSH server's trusted keys. (#1787)
+- Fix hostmap corruption when a host has multiple overlay addresses. Each address now gets its own list instead of
+  a single shared chain, which also fixes two latent bugs on the add and makePrimary paths. (#1788, #1790)
+- Apply `remote_allow_list` IPv4 rules to 4-in-6 mapped addresses. (#1786)
+- Don't panic in the DNS server on a short or empty query name. (#1635)
+- Advance the replay window on relayed packets so a relay drops replayed frames instead of re-forwarding them. (#1751)
+- Fix a race in relay state handling. (#1753)
+- Lock replay window updates so concurrent readers can't corrupt it. (#1802)
+- Reject malformed handshakes more reliably, including invalid ed25519 key lengths. (#1601, #1756)
+- Properly handle `closetunnel` packets. (#1638)
+- Fix an IPv6 extension-header length overflow that could make the firewall parse the wrong protocol and ports. (#1789)
+- Fix relay re-establishment when a handshake arrives over a relay entry that a one-sided teardown left
+  `Disestablished`, which silently dropped every send until dead tunnel detection forced a re-handshake. (#1805)
+- Don't build new relay state on a tunnel that was just discarded. (#1796)
+- Don't delete the wrong pending hostinfo in the handshake manager. (#1811)
+- Don't call the packet reader after a UDP error on Darwin. (#1755)
+- Open the FreeBSD tun device non blocking. (#1666)
+
 ## [1.10.3] - 2026-02-06
 
 ### Security

Makefile

@@ -222,11 +222,14 @@ test-cov-html:
 	go test -coverprofile=coverage.out
 	go tool cover -html=coverage.out
 
+# The package builds only compile. The final line links an android binary so a linker-only failure,
+# such as the //go:linkname reference anet makes, cannot pass CI.
 build-test-mobile:
 	GOARCH=amd64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
 	GOARCH=arm64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
 	GOARCH=amd64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
 	GOARCH=arm64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
+	GOARCH=arm64 GOOS=android go build -ldflags=-checklinkname=0 -o /dev/null ${NEBULA_CMD_PATH}
 
 bench:
 	go test -bench=.

cmd/nebula/close_on_timer_test.go

@@ -0,0 +1,96 @@
+//go:build linux && !android && !e2e_testing
+
+package main
+
+import (
+	"fmt"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/cert"
+	cert_test "github.com/slackhq/nebula/cert_test"
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/test"
+	"github.com/stretchr/testify/require"
+)
+
+// TestControlStopClosesOnTimer reproduces the dnclient lifecycle: nebula runs as
+// a library, and on a config update dnclient calls Stop() in-process to tear the
+// old instance down before starting a new one. This boots a real nebula (real
+// blocking UDP sockets, tun disabled), lets it run, then Stop()s it on a timer
+// and asserts it actually closes. If the reader goroutines parked in recvmmsg
+// don't wake on Close(), Wait() blocks forever and this fails with a goroutine
+// dump instead of relying on a process signal to unstick them.
+func TestControlStopClosesOnTimer(t *testing.T) {
+	l := test.NewLogger()
+	dir := t.TempDir()
+
+	before := time.Now().Add(-time.Hour)
+	after := time.Now().Add(time.Hour)
+	ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, before, after, nil, nil, nil)
+	networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
+	_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "close-on-timer", before, after, networks, nil, nil)
+
+	caPath := filepath.Join(dir, "ca.pem")
+	certPath := filepath.Join(dir, "cert.pem")
+	keyPath := filepath.Join(dir, "key.pem")
+	require.NoError(t, os.WriteFile(caPath, caPEM, 0o600))
+	require.NoError(t, os.WriteFile(certPath, certPEM, 0o600))
+	require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600))
+
+	// tun disabled so no device/root is needed; routines: 2 so we exercise the
+	// multi-socket (SO_REUSEPORT) teardown, which is where dnclient runs.
+	configBody := fmt.Sprintf(`
+pki:
+  ca: %s
+  cert: %s
+  key: %s
+listen:
+  host: 127.0.0.1
+  port: 0
+tun:
+  disabled: true
+firewall:
+  outbound:
+    - port: any
+      proto: any
+      host: any
+  inbound:
+    - port: any
+      proto: any
+      host: any
+routines: 2
+`, caPath, certPath, keyPath)
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(configBody), 0o600))
+
+	c := config.NewC(l)
+	require.NoError(t, c.Load(dir))
+
+	ctrl, err := nebula.Main(c, false, "close-on-timer", l, nil)
+	require.NoError(t, err)
+	require.NoError(t, ctrl.Start())
+
+	// Run like a live nebula, then close on a timer, exactly as dnclient does.
+	<-time.NewTimer(5 * time.Second).C
+
+	stopped := make(chan struct{})
+	go func() {
+		ctrl.Stop() // closes the udp sockets (shutdown(2)) and the tun
+		ctrl.Wait() // blocks until every reader goroutine has returned
+		close(stopped)
+	}()
+
+	select {
+	case <-stopped:
+		t.Log("nebula closed cleanly on timer")
+	case <-time.After(10 * time.Second):
+		buf := make([]byte, 1<<20)
+		n := runtime.Stack(buf, true)
+		t.Fatalf("nebula did NOT close within 10s of Stop(): a blocking reader never woke\n%s", buf[:n])
+	}
+}

connection_manager_test.go

@@ -25,6 +25,7 @@ func newTestLighthouse() *LightHouse {
 	lighthouses := []netip.Addr{}
 	staticList := map[netip.Addr]struct{}{}
 
+	lh.localAddrsFn = func(*LocalAllowList) []netip.Addr { return nil }
 	lh.lighthouses.Store(&lighthouses)
 	lh.staticList.Store(&staticList)
 

connection_state.go

@@ -2,11 +2,13 @@ package nebula
 
 import (
 	"encoding/json"
+	"log/slog"
 	"sync"
 	"sync/atomic"
 
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/handshake"
+	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/noiseutil"
 )
 
@@ -20,6 +22,7 @@ type ConnectionState struct {
 	initiator      bool
 	messageCounter atomic.Uint64
 	window         *Bits
+	decryptLock    sync.Mutex
 	writeLock      sync.Mutex
 }
 
@@ -54,3 +57,52 @@ func (cs *ConnectionState) MarshalJSON() ([]byte, error) {
 func (cs *ConnectionState) Curve() cert.Curve {
 	return cs.myCert.Curve()
 }
+
+func (cs *ConnectionState) Decrypt(l *slog.Logger, messageCounter uint64, out []byte, packet []byte, nb []byte) ([]byte, error) {
+	var err error
+	cs.decryptLock.Lock()
+	result := cs.window.Check(l, messageCounter)
+	cs.decryptLock.Unlock()
+	if !result {
+		return nil, ErrAlreadySeen
+	}
+
+	out, err = cs.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
+	if err != nil {
+		return nil, err
+	}
+
+	cs.decryptLock.Lock()
+	result = cs.window.Update(l, messageCounter)
+	cs.decryptLock.Unlock()
+	if !result {
+		return nil, ErrAlreadySeen
+	}
+	return out, nil
+}
+
+// VerifyRelay verifies AEAD protected (but not encrypted) relay frames. packet must be length-checked by the caller.
+func (cs *ConnectionState) VerifyRelay(l *slog.Logger, messageCounter uint64, packet []byte, nb []byte) error {
+	cs.decryptLock.Lock()
+	result := cs.window.Check(l, messageCounter)
+	cs.decryptLock.Unlock()
+	if !result {
+		return ErrAlreadySeen
+	}
+
+	signedPayload := packet[:len(packet)-cs.dKey.Overhead()]
+	signatureValue := packet[len(packet)-cs.dKey.Overhead():]
+	_, err := cs.dKey.DecryptDanger(nil, signedPayload, signatureValue, messageCounter, nb)
+	if err != nil {
+		return err
+	}
+
+	cs.decryptLock.Lock()
+	result = cs.window.Update(l, messageCounter)
+	cs.decryptLock.Unlock()
+	if !result {
+		return ErrAlreadySeen
+	}
+
+	return nil
+}

control.go

@@ -53,6 +53,7 @@ type Control struct {
 	statsStart             func()
 	dnsStart               func()
 	lighthouseStart        func()
+	networkChangeStart     func(rebind func())
 	connectionManagerStart func(context.Context)
 }
 
@@ -104,6 +105,9 @@ func (c *Control) Start() error {
 	if c.dnsStart != nil {
 		go c.dnsStart()
 	}
+	if c.networkChangeStart != nil {
+		go c.networkChangeStart(c.RebindUDPServer)
+	}
 	if c.connectionManagerStart != nil {
 		go c.connectionManagerStart(c.ctx)
 	}
@@ -198,7 +202,11 @@ func (c *Control) RebindUDPServer() {
 		return
 	}
 
-	_ = c.f.outside.Rebind()
+	// A failure here means we are likely still pinned to the interface we came up on, so the rest of this is
+	// unlikely to help. Say so instead of silently carrying on as if we rebound.
+	if err := c.f.outside.Rebind(); err != nil {
+		c.l.Error("Failed to rebind udp socket", "error", err)
+	}
 
 	// Trigger a lighthouse update, useful for mobile clients that should have an update interval of 0
 	c.f.lightHouse.SendUpdate()

control_lifecycle_test.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/test"
 	"github.com/slackhq/nebula/udp"
@@ -31,9 +30,9 @@ func newFakeDevice() *fakeDevice {
 
 // Read blocks until Close like a real tun with no traffic, then reports EOF
 // the same way a closed device does
-func (d *fakeDevice) Read() ([]tio.Packet, error) {
+func (d *fakeDevice) Read(p []byte) (int, error) {
 	<-d.closedCh
-	return nil, io.EOF
+	return 0, io.EOF
 }
 
 func (d *fakeDevice) Write(p []byte) (int, error) { return len(p), nil }
@@ -50,8 +49,10 @@ func (d *fakeDevice) Activate() error                       { return nil }
 func (d *fakeDevice) Networks() []netip.Prefix              { return nil }
 func (d *fakeDevice) Name() string                          { return "fake" }
 func (d *fakeDevice) RoutesFor(netip.Addr) routing.Gateways { return nil }
-
+func (d *fakeDevice) SupportsMultiqueue() bool              { return false }
-func (d *fakeDevice) Queues(int) ([]tio.Queue, error) { return []tio.Queue{d}, nil }
+func (d *fakeDevice) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, errors.New("unsupported")
+}
 
 // newReadyControl hand-builds the minimum Control that Main would have
 // produced right before Start, including the construction token NewInterface
@@ -77,6 +78,7 @@ func newReadyControl(t *testing.T) (*Control, *fakeDevice, *fakeConn) {
 		inside:     dev,
 		outside:    conn,
 		writers:    []udp.Conn{conn},
+		readers:    make([]io.ReadWriteCloser, 1),
 		routines:   1,
 		hostMap:    newHostMap(l),
 		lightHouse: lh,
@@ -153,14 +155,7 @@ type multiqueueDevice struct {
 	*fakeDevice
 }
 
-// Queues claims multiqueue support but fails to open the second queue,
+func (d *multiqueueDevice) SupportsMultiqueue() bool { return true }
-// exercising the activation error path.
-func (d *multiqueueDevice) Queues(n int) ([]tio.Queue, error) {
-	if n > 1 {
-		return nil, errors.New("second queue failed to open")
-	}
-	return d.fakeDevice.Queues(n)
-}
 
 func TestControl_StartMultiqueueFailureReleases(t *testing.T) {
 	dev := &multiqueueDevice{fakeDevice: newFakeDevice()}
@@ -171,6 +166,7 @@ func TestControl_StartMultiqueueFailureReleases(t *testing.T) {
 		inside:   dev,
 		outside:  conn,
 		writers:  []udp.Conn{conn},
+		readers:  make([]io.ReadWriteCloser, 2),
 		routines: 2,
 		l:        test.NewLogger(),
 	}

control_tester.go

@@ -108,7 +108,19 @@ func (c *Control) GetVpnAddrs() []netip.Addr {
 }
 
 func (c *Control) GetUDPAddr() netip.AddrPort {
-	return c.f.outside.(*udp.TesterConn).Addr
+	return c.f.outside.(*udp.TesterConn).GetAddr()
+}
+
+// SetUDPAddr moves this node to a new underlay address, standing in for a laptop waking up on a different
+// network. Register the new address with the router as well or nothing will route back.
+func (c *Control) SetUDPAddr(addr netip.AddrPort) {
+	c.f.outside.(*udp.TesterConn).SetAddr(addr)
+}
+
+// SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of
+// whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker.
+func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {
+	c.f.lightHouse.localAddrsFn = fn
 }
 
 func (c *Control) KillPendingTunnel(vpnIp netip.Addr) bool {

dns_server.go

@@ -97,8 +97,7 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
 	newAddr := getDnsServerAddr(c)
 
 	d.serverMu.Lock()
-	running := d.server
+	running := d.server != nil
-	runningStarted := d.started
 	sameAddr := d.addr == newAddr
 	d.addr = newAddr
 	d.enabled.Store(enabled)
@@ -112,7 +111,7 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
 	}
 
 	if !enabled {
-		if running != nil {
+		if running {
 			d.Stop()
 		}
 		// Drop any records that accumulated while enabled; a later re-enable
@@ -121,12 +120,12 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
 		return nil
 	}
 
-	if running == nil {
+	if !running {
 		// Was disabled (or never started); bring it up now.
 		go d.Start()
 	} else if !sameAddr {
-		d.shutdownServer(running, runningStarted, "reload")
+		// Stop clears the slot before shutting down, otherwise the Start below can find the dying server and refuse
-		// Old Start goroutine has now exited; bring up a fresh listener on the new address.
+		d.Stop()
 		go d.Start()
 	}
 
@@ -162,7 +161,9 @@ func (d *dnsServer) Start() {
 
 	started := make(chan struct{})
 	d.serverMu.Lock()
-	if d.ctx.Err() != nil {
+	// Re-check enabled under the lock, a disable that raced our check above snapshots the slot under it too.
+	// Two reloads in quick succession can both spawn a Start, the loser would orphan the live listener past Stop
+	if d.ctx.Err() != nil || d.server != nil || !d.enabled.Load() {
 		d.serverMu.Unlock()
 		return
 	}
@@ -200,6 +201,14 @@ func (d *dnsServer) Start() {
 		close(started)
 	}
 
+	// Release our slot, unless a reload already replaced us, so a dead listener can't block a future Start
+	d.serverMu.Lock()
+	if d.server == server {
+		d.server = nil
+		d.started = nil
+	}
+	d.serverMu.Unlock()
+
 	if err != nil {
 		d.l.Warn("Failed to run the DNS responder", "error", err)
 	}

dns_server_test.go

@@ -194,14 +194,51 @@ func TestDnsServer_reload_initial_serveDnsWithoutLighthouse(t *testing.T) {
 }
 
 func TestDnsServer_reload_sameAddr_noOp(t *testing.T) {
+	port := freeUDPPort(t)
 	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", "0", true, true)
+	setDnsConfig(c, "127.0.0.1", port, true, true)
-
 	require.NoError(t, ds.reload(c, true))
-	// No server running yet, no addr change. Reload should not spawn anything.
+
+	go ds.Start()
+	waitForBind(t, ds)
+
+	ds.serverMu.Lock()
+	before := ds.server
+	ds.serverMu.Unlock()
+	require.NotNil(t, before)
+
+	// Same address, so the running listener must be left alone rather than rebuilt under live queries
 	require.NoError(t, ds.reload(c, false))
 	assert.True(t, ds.enabled.Load())
-	assert.Nil(t, ds.server)
+
+	ds.serverMu.Lock()
+	after := ds.server
+	ds.serverMu.Unlock()
+	assert.Same(t, before, after, "a same-address reload must not restart the listener")
+
+	ds.Stop()
+}
+
+// The branch the old sameAddr test was accidentally hitting: enabled with nothing running means reload starts it.
+func TestDnsServer_reload_whenNotRunning_starts(t *testing.T) {
+	port := freeUDPPort(t)
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", port, true, true)
+
+	// initial only records config, it never starts anything
+	require.NoError(t, ds.reload(c, true))
+	ds.serverMu.Lock()
+	assert.Nil(t, ds.server, "the initial reload must not start a listener")
+	ds.serverMu.Unlock()
+
+	require.NoError(t, ds.reload(c, false))
+	waitForBind(t, ds)
+
+	ds.serverMu.Lock()
+	assert.NotNil(t, ds.server, "a reload with nothing running should bring DNS up")
+	ds.serverMu.Unlock()
+
+	ds.Stop()
 }
 
 func TestDnsServer_StartStop_lifecycle(t *testing.T) {
@@ -427,3 +464,168 @@ func waitFor(t *testing.T, cond func() bool) {
 	}
 	t.Fatal("timed out waiting for condition")
 }
+
+// Two reloads in quick succession, or a HUP before Control.Start, can race two Starts at the same listener.
+func TestDnsServer_Start_isIdempotent(t *testing.T) {
+	port := freeUDPPort(t)
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", port, true, true)
+	require.NoError(t, ds.reload(c, true))
+
+	go ds.Start()
+	waitForBind(t, ds)
+
+	ds.serverMu.Lock()
+	first := ds.server
+	ds.serverMu.Unlock()
+	require.NotNil(t, first)
+
+	// If the second Start replaces the tracked server, Stop kills the wrong one and the port leaks
+	done := make(chan struct{})
+	go func() {
+		ds.Start()
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(time.Second * 5):
+		t.Fatal("second Start never returned")
+	}
+
+	ds.serverMu.Lock()
+	second := ds.server
+	ds.serverMu.Unlock()
+	assert.Same(t, first, second, "a second Start must not replace the running server")
+
+	// The real proof, after Stop the port must actually be free
+	ds.Stop()
+	waitFor(t, func() bool {
+		pc, err := net.ListenPacket("udp", "127.0.0.1:"+port)
+		if err != nil {
+			return false
+		}
+		_ = pc.Close()
+		return true
+	})
+}
+
+// An address change must actually end up listening on the new port. Start's guard refuses when a server is already
+// installed, so reload has to clear the slot before shutting the old one down.
+func TestDnsServer_reload_addrChange_restarts(t *testing.T) {
+	first := freeUDPPort(t)
+	second := freeUDPPort(t)
+
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", first, true, true)
+	require.NoError(t, ds.reload(c, true))
+
+	go ds.Start()
+	waitForBind(t, ds)
+
+	// Cycle a few times, the failure this guards against depends on which goroutine wins serverMu
+	for i := range 8 {
+		want := second
+		if i%2 == 1 {
+			want = first
+		}
+		setDnsConfig(c, "127.0.0.1", want, true, true)
+		require.NoError(t, ds.reload(c, false))
+		waitForBind(t, ds)
+
+		ds.serverMu.Lock()
+		srv := ds.server
+		ds.serverMu.Unlock()
+		require.NotNil(t, srv, "reload left DNS down instead of restarting it")
+		require.Equal(t, "127.0.0.1:"+want, srv.Addr, "reload should be serving the new address")
+	}
+
+	// Land back on second so the port assertions below are meaningful
+	setDnsConfig(c, "127.0.0.1", second, true, true)
+	require.NoError(t, ds.reload(c, false))
+	waitForBind(t, ds)
+
+	// The old port must be released and the new one actually held
+	waitFor(t, func() bool {
+		pc, err := net.ListenPacket("udp", "127.0.0.1:"+first)
+		if err != nil {
+			return false
+		}
+		_ = pc.Close()
+		return true
+	})
+	_, err := net.ListenPacket("udp", "127.0.0.1:"+second)
+	require.Error(t, err, "the new address should be bound by the DNS responder")
+
+	ds.Stop()
+}
+
+// A listener that dies on its own must release the slot, or a later same-addr reload sees it as running and no-ops.
+func TestDnsServer_Start_bindFailure_releasesSlot(t *testing.T) {
+	port := freeUDPPort(t)
+	blocker, err := net.ListenPacket("udp", "127.0.0.1:"+port)
+	require.NoError(t, err)
+
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", port, true, true)
+	require.NoError(t, ds.reload(c, true))
+
+	ds.Start() // returns once the bind fails
+
+	ds.serverMu.Lock()
+	assert.Nil(t, ds.server, "a listener that failed to bind must not stay parked in the slot")
+	ds.serverMu.Unlock()
+
+	// With the slot released, a reload can retry once the port frees up
+	require.NoError(t, blocker.Close())
+	require.NoError(t, ds.reload(c, false))
+	waitForBind(t, ds)
+
+	ds.serverMu.Lock()
+	assert.NotNil(t, ds.server, "a same-addr reload should retry after a failed bind")
+	ds.serverMu.Unlock()
+
+	ds.Stop()
+}
+
+// A disable that lands while Start is between its unlocked check and the guard must not leave a listener behind.
+func TestDnsServer_Start_refusesWhenDisabledUnderLock(t *testing.T) {
+	port := freeUDPPort(t)
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", port, true, true)
+	require.NoError(t, ds.reload(c, true))
+	require.True(t, ds.enabled.Load())
+
+	// Holding serverMu parks Start on the lock, the only way to land the disable in that window on purpose
+	ds.serverMu.Lock()
+
+	done := make(chan struct{})
+	go func() {
+		ds.Start()
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		ds.serverMu.Unlock()
+		t.Fatal("Start returned early, the test never exercised the window")
+	case <-time.After(time.Millisecond * 100):
+	}
+
+	// The disable reload's critical section. It sees nothing running, so it never calls Stop.
+	ds.enabled.Store(false)
+	ds.serverMu.Unlock()
+
+	select {
+	case <-done:
+	case <-time.After(time.Second * 5):
+		t.Fatal("Start never returned")
+	}
+
+	ds.serverMu.Lock()
+	assert.Nil(t, ds.server, "Start must not install a listener a disable already cancelled")
+	ds.serverMu.Unlock()
+
+	pc, err := net.ListenPacket("udp", "127.0.0.1:"+port)
+	require.NoError(t, err, "an orphaned listener is still holding the port")
+	_ = pc.Close()
+}

e2e/handshakes_test.go

@@ -725,6 +725,70 @@ func TestReestablishRelays(t *testing.T) {
 
 }
 
+func TestRelayHandshakeOverDisestablishedEntry(t *testing.T) {
+	t.Parallel()
+	// If them tears down the tunnel while me keeps Established relay state, me's next
+	// handshake flows through the relay with no fresh CreateRelayRequest and lands on
+	// them's Disestablished terminal relay entry. them must re-establish that entry, or
+	// its first transmit deletes its only relay and the tunnel is born transmit-dead:
+	// them can receive but every send is silently dropped.
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
+	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay  ", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them   ", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
+
+	// Teach my how to get to the relay and that their can be reached via the relay
+	myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
+	myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
+	relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+
+	// Build a router so we don't have to reason who gets which packet
+	r := router.NewR(t, myControl, relayControl, theirControl)
+	defer r.RenderFlow()
+
+	// Start the servers
+	myControl.Start()
+	relayControl.Start()
+	theirControl.Start()
+
+	t.Log("Trigger a handshake from me to them via the relay")
+	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+
+	p := r.RouteForAllUntilTxTun(theirControl)
+	assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
+	oldIdx := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false).LocalIndex
+
+	t.Log("Close the tunnel on them only, marking their relay entry Disestablished")
+	theirControl.CloseTunnel(myVpnIpNet[0].Addr(), true)
+
+	t.Log("Re-handshake from me, riding the still-Established relay state")
+	myControl.ReHandshake(theirVpnIpNet[0].Addr())
+	for {
+		h := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
+		if h != nil && h.LocalIndex != oldIdx && h.RemoteIndex != 0 {
+			break
+		}
+		r.RouteForAllExitFunc(func(*udp.Packet, *nebula.Control) router.ExitType {
+			return router.RouteAndExit
+		})
+	}
+
+	hAtThem := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
+	require.NotNil(t, hAtThem, "them should have completed the relayed handshake")
+	require.Equal(t, []netip.Addr{relayVpnIpNet[0].Addr()}, hAtThem.CurrentRelaysToMe, "them should know a relay for the new tunnel")
+
+	t.Log("Send from them to me; their only relay entry must survive the transmit")
+	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them")))
+	require.Never(t, func() bool {
+		h := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
+		return h == nil || len(h.CurrentRelaysToMe) == 0
+	}, time.Second, 10*time.Millisecond, "them deleted its only relay entry; the tunnel is permanently transmit-dead")
+
+	p = r.RouteForAllUntilTxTun(myControl)
+	assertUdpPacket(t, []byte("Hi from them"), p, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), 80, 80)
+	r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
+}
+
 func TestStage1RaceRelays(t *testing.T) {
 	t.Parallel()
 	//NOTE: this is a race between me and relay resulting in a full tunnel from me to them via relay

e2e/rebind_test.go

@@ -0,0 +1,225 @@
+//go:build e2e_testing
+// +build e2e_testing
+
+package e2e
+
+import (
+	"net/netip"
+	"testing"
+	"time"
+
+	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/cert_test"
+	"github.com/slackhq/nebula/e2e/router"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/udp"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// reportedAddrs is what the lighthouse would hand a peer asking where vpnAddr is.
+func reportedAddrs(t *testing.T, lh *nebula.Control, vpnAddr netip.Addr) []netip.AddrPort {
+	t.Helper()
+	cm := lh.QueryLighthouse(vpnAddr)
+	if cm == nil {
+		return nil
+	}
+	var out []netip.AddrPort
+	for _, c := range *cm {
+		out = append(out, c.Reported...)
+		out = append(out, c.Learned...)
+	}
+	return out
+}
+
+// waitForLighthouseMsg routes until a lighthouse message lands on lh, or gives up. Reports whether one arrived.
+func waitForLighthouseMsg(t *testing.T, r *router.R, lh *nebula.Control, wait time.Duration) bool {
+	t.Helper()
+	h := &header.H{}
+	return r.RouteForAllExitFuncOrTimeout(wait, func(p *udp.Packet, c *nebula.Control) router.ExitType {
+		if c != lh {
+			return router.KeepRouting
+		}
+		// Punches are a single byte and never parse, they are just not what we are after
+		if err := h.Parse(p.Data); err != nil {
+			return router.KeepRouting
+		}
+		if h.Type == header.LightHouse {
+			return router.RouteAndExit
+		}
+		return router.KeepRouting
+	})
+}
+
+// A laptop that changes networks has to tell the lighthouse promptly, otherwise the lighthouse keeps handing peers
+// the old address and their punches land nowhere. On a long lighthouse interval the only thing that closes that
+// window is the rebind, which on darwin the network change monitor drives. The e2e build compiles the monitor out,
+// so we call RebindUDPServer directly, which is the same thing the monitor does.
+func TestRebindSendsLighthouseUpdate(t *testing.T) {
+	t.Parallel()
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
+	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
+		"lighthouse": m{"am_lighthouse": true},
+	})
+
+	// 600s interval, so nothing scheduled can send an update during this test. A rebind is the only thing that can.
+	myControl, _, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{
+		"lighthouse": m{
+			"hosts":    []any{lhVpnIpNet[0].Addr().String()},
+			"interval": 600,
+		},
+		"static_host_map": m{
+			lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
+		},
+	})
+
+	r := router.NewR(t, lhControl, myControl)
+	defer r.RenderFlow()
+
+	lhControl.Start()
+	myControl.Start()
+
+	// Let the startup registration finish, then clear everything it left behind
+	require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an initial registration")
+	r.RouteFor(time.Millisecond * 400)
+
+	// Nothing should be talking to the lighthouse on its own now
+	require.False(t, waitForLighthouseMsg(t, r, lhControl, time.Millisecond*200),
+		"nothing should reach the lighthouse before the rebind")
+
+	myControl.RebindUDPServer()
+
+	assert.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5),
+		"a rebind should push an update to the lighthouse rather than waiting out the interval")
+
+	lhControl.Stop()
+	myControl.Stop()
+}
+
+// The other half of a rebind: every live tunnel requeries the lighthouse on its next send. That query is what makes
+// the lighthouse tell the peer to punch toward our new address, which is the part that actually revives a tunnel
+// whose remote NAT state died while we were on a different network.
+func TestRebindRequeriesPeersOnNextSend(t *testing.T) {
+	t.Parallel()
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
+	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
+		"lighthouse": m{"am_lighthouse": true},
+	})
+
+	lhCfg := m{
+		"lighthouse": m{
+			"hosts":    []any{lhVpnIpNet[0].Addr().String()},
+			"interval": 600,
+			// Without this the peers advertise this machine's real addresses and then try to punch at them,
+			// which the router has no route for.
+			"local_allow_list": m{
+				"10.0.0.0/24": true,
+				"::/0":        false,
+			},
+		},
+		"static_host_map": m{
+			lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
+		},
+	}
+
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", lhCfg)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.3/24", lhCfg)
+
+	r := router.NewR(t, lhControl, myControl, theirControl)
+	defer r.RenderFlow()
+
+	lhControl.Start()
+	myControl.Start()
+	theirControl.Start()
+	r.RouteFor(time.Millisecond * 500)
+
+	// Point the peers at each other directly, this test is about the rebind and not about lighthouse discovery
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("initial")))
+	r.RouteFor(time.Second)
+	require.NotNil(t, myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false), "expected a tunnel to them")
+	r.RouteFor(time.Millisecond * 300)
+
+	// Assert on what the peer sees rather than on lighthouse traffic. A query for them makes the lighthouse send
+	// them a punch notification, which is the whole point. Our own update to the lighthouse sends them nothing,
+	// so this cannot be satisfied by the update the rebind itself pushes.
+	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("quiet")))
+	require.False(t, waitForLighthouseMsg(t, r, theirControl, time.Millisecond*300),
+		"an ordinary send should not requery the lighthouse")
+
+	myControl.RebindUDPServer()
+	r.RouteFor(time.Millisecond * 300) // let the update the rebind itself sends pass by
+
+	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("after rebind")))
+	assert.True(t, waitForLighthouseMsg(t, r, theirControl, time.Second*5),
+		"the first send after a rebind should requery the lighthouse, which then tells the peer to punch at us")
+
+	lhControl.Stop()
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+// The scenario this whole thing exists for: a laptop sleeps at the office and wakes up at home on a new address.
+// Until it tells the lighthouse, the lighthouse keeps handing peers the office address, so their punches land
+// nowhere and the tunnel stays dead. On a long interval the rebind is the only thing that closes that window.
+func TestRebindAdvertisesNewAddressAfterMove(t *testing.T) {
+	t.Parallel()
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
+	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
+		"lighthouse": m{"am_lighthouse": true},
+	})
+
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{
+		"lighthouse": m{
+			"hosts":    []any{lhVpnIpNet[0].Addr().String()},
+			"interval": 600,
+		},
+		"static_host_map": m{
+			lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
+		},
+	})
+
+	// Advertise wherever we currently are rather than this machine's real NICs, read fresh each time so a move
+	// is picked up.
+	myControl.SetLocalAddrsFn(func(*nebula.LocalAllowList) []netip.Addr {
+		return []netip.Addr{myControl.GetUDPAddr().Addr()}
+	})
+
+	r := router.NewR(t, lhControl, myControl)
+	defer r.RenderFlow()
+
+	lhControl.Start()
+	myControl.Start()
+
+	require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an initial registration")
+	r.RouteFor(time.Millisecond * 400)
+
+	require.Contains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), myUdpAddr,
+		"the lighthouse should know the address we started on")
+
+	// Wake up somewhere else
+	newAddr := netip.MustParseAddrPort("10.0.0.99:4242")
+	myControl.SetUDPAddr(newAddr)
+	r.AddRoute(newAddr.Addr(), newAddr.Port(), myControl)
+
+	// Nothing has told the lighthouse, and with interval 600 nothing scheduled will
+	r.RouteFor(time.Millisecond * 400)
+	require.NotContains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), newAddr,
+		"the lighthouse should still be handing out the old address before the rebind")
+
+	myControl.RebindUDPServer()
+	require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an update after the rebind")
+	r.RouteFor(time.Millisecond * 400)
+
+	assert.Contains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), newAddr,
+		"after the rebind the lighthouse should hand peers our new address")
+
+	lhControl.Stop()
+	myControl.Stop()
+}

e2e/recovery_timing_test.go

@@ -0,0 +1,136 @@
+//go:build e2e_testing
+// +build e2e_testing
+
+package e2e
+
+import (
+	"testing"
+	"time"
+
+	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/cert_test"
+	"github.com/slackhq/nebula/e2e/router"
+	"github.com/slackhq/nebula/udp"
+)
+
+// TestRecoveryTiming measures how long a tunnel takes to come back after the peer stops accepting our traffic,
+// which is what a laptop waking on a new network looks like from the peer's side: its NAT has no state for where
+// we are now, so everything we send disappears.
+//
+// It is a measurement, not a pass/fail assertion. Recovery is timed to the moment the peer punches back at us,
+// since that is when its NAT opens and the tunnel is usable again.
+//
+//	go test -tags e2e_testing -v -run TestRecoveryTiming ./e2e/
+func TestRecoveryTiming(t *testing.T) {
+	for _, tc := range []struct {
+		name   string
+		rebind bool
+	}{
+		{"no trigger", false},
+		{"rebind counter", true},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			d, lost := measureRecovery(t, tc.rebind)
+			t.Logf("RESULT %-16s recovered in %-9v (%d packets lost)", tc.name, d.Round(time.Millisecond), lost)
+		})
+	}
+}
+
+// measureRecovery returns how long until the peer punched back, and how many of our packets died meanwhile. When
+// rebind is true we call RebindUDPServer once the tunnel goes dark, which is what the darwin network change
+// monitor does and what iOS has always done. When false, nothing tells nebula anything is wrong.
+func measureRecovery(t *testing.T, rebind bool) (time.Duration, int) {
+	t.Helper()
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
+	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
+		"lighthouse": m{"am_lighthouse": true},
+	})
+
+	peerCfg := m{
+		"lighthouse": m{
+			"hosts":    []any{lhVpnIpNet[0].Addr().String()},
+			"interval": 600,
+			"local_allow_list": m{
+				"10.0.0.0/24": true,
+				"::/0":        false,
+			},
+		},
+		"static_host_map": m{
+			lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
+		},
+	}
+
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", peerCfg)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.3/24", peerCfg)
+
+	r := router.NewR(t, lhControl, myControl, theirControl)
+	defer r.RenderFlow()
+	defer func() {
+		lhControl.Stop()
+		myControl.Stop()
+		theirControl.Stop()
+	}()
+
+	lhControl.Start()
+	myControl.Start()
+	theirControl.Start()
+	r.RouteFor(time.Millisecond * 500)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("establish")))
+	r.RouteFor(time.Second)
+	if myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false) == nil {
+		t.Fatal("failed to establish the tunnel we are measuring")
+	}
+	r.RouteFor(time.Millisecond * 500)
+
+	// From here the peer's NAT has no state for us, everything we send it disappears
+	start := time.Now()
+	blackholed := 0
+	var recovered time.Duration
+
+	if rebind {
+		myControl.RebindUDPServer()
+	}
+
+	// Keep the tun busy the way someone retrying a stalled connection would
+	stop := make(chan struct{})
+	defer close(stop)
+	go func() {
+		tick := time.NewTicker(time.Millisecond * 200)
+		defer tick.Stop()
+		for {
+			select {
+			case <-stop:
+				return
+			case <-tick.C:
+				myControl.InjectTunPacket(BuildTunUDPPacket(
+					theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("retry")))
+			}
+		}
+	}()
+
+	r.RouteForAllExitFuncOrTimeout(time.Second*30, func(p *udp.Packet, c *nebula.Control) router.ExitType {
+		if c == theirControl && p.From == myControl.GetUDPAddr() {
+			blackholed++
+			return router.Drop
+		}
+
+		// The peer reaching us directly is the moment its NAT opened, whether that is a punch or a handshake
+		if c == myControl && p.From == theirUdpAddr {
+			recovered = time.Since(start)
+			return router.RouteAndExit
+		}
+
+		return router.KeepRouting
+	})
+
+	if recovered == 0 {
+		t.Fatalf("no recovery within 30s (%d packets blackholed)", blackholed)
+	}
+	return recovered, blackholed
+}

e2e/router/router.go

@@ -114,6 +114,28 @@ type packet struct {
 	packet *udp.Packet
 	tun    bool // a packet pulled off a tun device
 	rx     bool // the packet was received by a udp device
+
+	// h is the nebula header, parsed once when the packet is recorded. parseErr says why there isn't one, which
+	// the flow log reports rather than hiding. Punchy sends a single byte, so an unparseable packet is normal.
+	h        header.H
+	parseErr error
+}
+
+// fromAddr and toAddr are the addresses this packet actually travelled between. Reading them off the control
+// instead would misreport the whole history once a test moves a node. Tun packets are synthesized without
+// addresses, so they fall back to the control.
+func (p *packet) fromAddr() netip.AddrPort {
+	if p.tun || !p.packet.From.IsValid() {
+		return p.from.GetUDPAddr()
+	}
+	return p.packet.From
+}
+
+func (p *packet) toAddr() netip.AddrPort {
+	if p.tun || !p.packet.To.IsValid() {
+		return p.to.GetUDPAddr()
+	}
+	return p.packet.To
 }
 
 func (p *packet) WasReceived() {
@@ -131,6 +153,9 @@ const (
 	ExitNow ExitType = 1
 	// RouteAndExit routes this packet and exits immediately afterwards
 	RouteAndExit ExitType = 2
+	// Drop discards this packet without delivering it and keeps routing. Use it to simulate a blackhole, such as
+	// a restrictive NAT refusing traffic from an address it has not seen.
+	Drop ExitType = 3
 )
 
 type ExitFunc func(packet *udp.Packet, receiver *nebula.Control) ExitType
@@ -141,7 +166,9 @@ type ExitFunc func(packet *udp.Packet, receiver *nebula.Control) ExitType
 func NewR(t testing.TB, controls ...*nebula.Control) *R {
 	ctx, cancel := context.WithCancel(context.Background())
 
-	if err := os.MkdirAll("mermaid", 0755); err != nil {
+	// t.Name() contains a slash for subtests, so the flow log can land in a nested directory
+	fn := filepath.Join("mermaid", fmt.Sprintf("%s.md", t.Name()))
+	if err := os.MkdirAll(filepath.Dir(fn), 0755); err != nil {
 		panic(err)
 	}
 
@@ -152,7 +179,7 @@ func NewR(t testing.TB, controls ...*nebula.Control) *R {
 		outNat:       make(map[outNatKey]netip.AddrPort),
 		flow:         []flowEntry{},
 		ignoreFlows:  []ignoreFlow{},
-		fn:           filepath.Join("mermaid", fmt.Sprintf("%s.md", t.Name())),
+		fn:           fn,
 		t:            t,
 		cancelRender: cancel,
 	}
@@ -249,7 +276,7 @@ func (r *R) renderFlow() {
 			continue
 		}
 
-		addr := e.packet.from.GetUDPAddr()
+		addr := e.packet.fromAddr()
 		if _, ok := participants[addr]; ok {
 			continue
 		}
@@ -268,7 +295,6 @@ func (r *R) renderFlow() {
 	}
 
 	// Print packets
-	h := &header.H{}
 	for _, e := range r.flow {
 		if e.packet == nil {
 			//fmt.Fprintf(f, "    note over %s: %s\n", strings.Join(participantsVals, ", "), e.note)
@@ -280,21 +306,22 @@ func (r *R) renderFlow() {
 			fmt.Fprintln(f, r.formatUdpPacket(p))
 
 		} else {
-			if err := h.Parse(p.packet.Data); err != nil {
-				panic(err)
-			}
-
 			line := "--x"
 			if p.rx {
 				line = "->>"
 			}
 
-			fmt.Fprintf(f,
+			detail := fmt.Sprintf("%s(%s), index %v, counter: %v",
-				"    %s%s%s: %s(%s), index %v, counter: %v\n",
+				p.h.TypeName(), p.h.SubTypeName(), p.h.RemoteIndex, p.h.MessageCounter)
-				normalizeName(p.from.GetUDPAddr().String()),
+			if p.parseErr != nil {
+				detail = fmt.Sprintf("unparsed, %v (%d bytes)", p.parseErr, len(p.packet.Data))
+			}
+
+			fmt.Fprintf(f, "    %s%s%s: %s\n",
+				normalizeName(p.fromAddr().String()),
 				line,
-				normalizeName(p.to.GetUDPAddr().String()),
+				normalizeName(p.toAddr().String()),
-				h.TypeName(), h.SubTypeName(), h.RemoteIndex, h.MessageCounter,
+				detail,
 			)
 		}
 	}
@@ -408,21 +435,24 @@ func (r *R) unlockedInjectFlow(from, to *nebula.Control, p *udp.Packet, tun bool
 
 	r.renderHostmaps(fmt.Sprintf("Packet %v", len(r.flow)))
 
-	if len(r.ignoreFlows) > 0 {
 	var h header.H
-		err := h.Parse(p.Data)
+	var parseErr error
-		if err != nil {
+	if !tun {
-			panic(err)
+		parseErr = h.Parse(p.Data)
 	}
 
+	// Decide before copying, the copy comes from a freelist and an ignored packet would never be released
 	for _, i := range r.ignoreFlows {
-			if !tun {
+		if tun {
-				if i.messageType == h.Type && i.subType == h.Subtype {
+			if i.tun.HasValue && i.tun.IsTrue {
 				return nil
 			}
-			} else if i.tun.HasValue && i.tun.IsTrue {
+			continue
-				return nil
 		}
+
+		// A packet we could not parse has no type to match against, so no rule can ignore it
+		if parseErr == nil && i.messageType == h.Type && i.subType == h.Subtype {
+			return nil
 		}
 	}
 
@@ -431,6 +461,8 @@ func (r *R) unlockedInjectFlow(from, to *nebula.Control, p *udp.Packet, tun bool
 		to:       to,
 		packet:   p.Copy(),
 		tun:      tun,
+		h:        h,
+		parseErr: parseErr,
 	}
 
 	r.flow = append(r.flow, flowEntry{packet: fp})
@@ -660,6 +692,10 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
 			p.Release()
 			return
 
+		case Drop:
+			// Record it so the flow log shows the attempt, but never hand it to the receiver
+			r.unlockedInjectFlow(sender, receiver, p, false)
+
 		case KeepRouting:
 			fp := r.unlockedInjectFlow(sender, receiver, p, false)
 			receiver.InjectUDPPacket(p)
@@ -690,6 +726,85 @@ func (r *R) RouteUntilAfterMsgType(sender *nebula.Control, msgType header.Messag
 	})
 }
 
+// RouteFor routes everything that shows up for the given duration and then returns. Use it to let a test settle
+// deterministically rather than sleeping and hoping: a single FlushAll races a completing handshake, which queues
+// more packets right behind it.
+func (r *R) RouteFor(d time.Duration) {
+	r.RouteForAllExitFuncOrTimeout(d, func(*udp.Packet, *nebula.Control) ExitType {
+		return KeepRouting
+	})
+}
+
+// RouteForAllExitFuncOrTimeout is RouteForAllExitFunc with a deadline, reporting whether whatDo asked to exit
+// before time ran out. The unbounded version blocks forever on a quiet network, so this is what a test needs to
+// assert that something does NOT happen, or to route for a fixed settling period.
+func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc) bool {
+	sc := make([]reflect.SelectCase, 0, len(r.controls)+1)
+	cm := make([]*nebula.Control, 0, len(r.controls))
+
+	for _, c := range r.controls {
+		sc = append(sc, reflect.SelectCase{
+			Dir:  reflect.SelectRecv,
+			Chan: reflect.ValueOf(c.GetUDPTxChan()),
+			Send: reflect.Value{},
+		})
+		cm = append(cm, c)
+	}
+
+	timer := time.NewTimer(timeout)
+	defer timer.Stop()
+	sc = append(sc, reflect.SelectCase{
+		Dir:  reflect.SelectRecv,
+		Chan: reflect.ValueOf(timer.C),
+		Send: reflect.Value{},
+	})
+
+	for {
+		x, rx, _ := reflect.Select(sc)
+		if x == len(cm) {
+			return false
+		}
+
+		r.Lock()
+		p := rx.Interface().(*udp.Packet)
+		receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
+		if receiver == nil {
+			r.Unlock()
+			panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String())
+		}
+
+		e := whatDo(p, receiver)
+		switch e {
+		case ExitNow:
+			r.Unlock()
+			p.Release()
+			return true
+
+		case RouteAndExit:
+			fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
+			receiver.InjectUDPPacket(p)
+			fp.WasReceived()
+			r.Unlock()
+			p.Release()
+			return true
+
+		case Drop:
+			// Record it so the flow log shows the attempt, but never hand it to the receiver
+			r.unlockedInjectFlow(cm[x], receiver, p, false)
+
+		case KeepRouting:
+			fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
+			receiver.InjectUDPPacket(p)
+			fp.WasReceived()
+
+		default:
+			panic(fmt.Sprintf("Unknown exitFunc return: %v", e))
+		}
+		r.Unlock()
+		p.Release()
+	}
+}
+
 func (r *R) RouteForAllUntilAfterMsgTypeTo(receiver *nebula.Control, msgType header.MessageType, subType header.MessageSubType) {
 	h := &header.H{}
 	r.RouteForAllExitFunc(func(p *udp.Packet, r *nebula.Control) ExitType {
@@ -782,6 +897,10 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
 			p.Release()
 			return
 
+		case Drop:
+			// Record it so the flow log shows the attempt, but never hand it to the receiver
+			r.unlockedInjectFlow(cm[x], receiver, p, false)
+
 		case KeepRouting:
 			fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
 			receiver.InjectUDPPacket(p)

examples/config.yml

@@ -146,6 +146,14 @@ listen:
   # Default true; set to false to leave WDF in charge of inbound decisions on the listener port. Not reloadable.
   #windows_bypass_wdf: true
 
+  # On macOS only
+  # macOS scopes the udp socket to the interface it was created on, so moving between networks (wifi to wired,
+  # office to home) leaves Nebula sending out an interface that no longer has a route. When true, Nebula watches
+  # the routing socket and rebinds the listener once the change settles.
+  # iOS does not use this, the host app drives the same rebind itself.
+  # Default true. Not reloadable.
+  #rebind_on_network_change: true
+
   # By default, Nebula replies to packets it has no tunnel for with a "recv_error" packet. This packet helps speed up reconnection
   # in the case that Nebula on either side did not shut down cleanly. This response can be abused as a way to discover if Nebula is running
   # on a host though. This option lets you configure if you want to send "recv_error" packets always, never, or only to private network remotes.
@@ -254,20 +262,6 @@ tun:
   # Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic
   mtu: 1300
 
-  # Linux only. pin_threads pins each tun reader/encrypt OS thread to a single CPU. This keeps every goroutine's
-  # sends flowing through one XPS-selected NIC TX ring, so packets within a flow stay ordered on the wire
-  # instead of being sprayed across multiple TX rings and reordered. Not reloadable.
-  #pin_threads: true
-
-  # Linux only. cpu_affinity overrides which CPUs the tun reader threads pin to: a list of CPU IDs, one per routine
-  # (see the top-level `routines` setting). Lists shorter than `routines` are modulo-cycled across the queues; extra
-  # entries are ignored. IDs must be within the process's allowed CPU set, so this respects taskset / cgroup cpusets;
-  # a non-integer or not-allowed entry disables the override and falls back to spreading queues across the allowed
-  # CPUs. Only meaningful while pin_threads is true. Not reloadable.
-  #cpu_affinity:
-  #  - 2
-  #  - 4
-
   # Route based MTU overrides, you have known vpn ip paths that can support larger MTUs you can increase/decrease them here
   routes:
     #- mtu: 8800

examples/service_scripts/nebula.service

@@ -8,6 +8,15 @@ Before=sshd.service
 Type=notify
 NotifyAccess=main
 SyslogIdentifier=nebula
+
+# Uncomment to run as an unprivileged user with only CAP_NET_ADMIN. Requires a
+# nebula user that owns the config directory. Add CAP_NET_BIND_SERVICE to both
+# lines if any listener (lighthouse DNS, listen.port, stats, sshd) binds <1024.
+#User=nebula
+#Group=nebula
+#CapabilityBoundingSet=CAP_NET_ADMIN
+#AmbientCapabilities=CAP_NET_ADMIN
+
 ExecReload=/bin/kill -HUP $MAINPID
 ExecStart=/usr/local/bin/nebula -config /etc/nebula/config.yml
 Restart=always

go.mod

@@ -1,6 +1,6 @@
 module github.com/slackhq/nebula
 
-go 1.25.0
+go 1.26.0
 
 require (
 	dario.cat/mergo v1.0.2
@@ -22,14 +22,15 @@ require (
 	github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
 	github.com/stretchr/testify v1.11.1
 	github.com/vishvananda/netlink v1.3.1
+	github.com/wlynxg/anet v0.0.5
 	go.uber.org/goleak v1.3.0
 	go.yaml.in/yaml/v3 v3.0.4
-	golang.org/x/crypto v0.53.0
+	golang.org/x/crypto v0.54.0
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
-	golang.org/x/net v0.56.0
+	golang.org/x/net v0.57.0
-	golang.org/x/sync v0.21.0
+	golang.org/x/sync v0.22.0
-	golang.org/x/sys v0.46.0
+	golang.org/x/sys v0.47.0
-	golang.org/x/term v0.44.0
+	golang.org/x/term v0.45.0
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
 	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
 	golang.zx2c4.com/wireguard/windows v1.0.1

go.sum

@@ -149,6 +149,8 @@ github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW
 github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
 github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
 github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
+github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU=
+github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
@@ -162,8 +164,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
+golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
-golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
+golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
@@ -182,8 +184,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
-golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -191,8 +193,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
-golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -208,11 +210,11 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
-golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
-golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

handshake_manager.go

@@ -295,7 +295,13 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 		hm.messageMetrics.Tx(header.Handshake, hh.machine.Subtype(), 1)
 		err := hm.outside.WriteTo(stage0, addr)
 		if err != nil {
-			hostinfo.logger(hm.l).Error("Failed to send handshake message",
+			// These repeat every attempt, so match the success log below and only shout when the remotes changed
+			level := slog.LevelDebug
+			if remotesHaveChanged {
+				level = slog.LevelError
+			}
+
+			hostinfo.logger(hm.l).Log(context.Background(), level, "Failed to send handshake message",
 				"udpAddr", addr,
 				"initiatorIndex", hostinfo.localIndexId,
 				"handshake", hsFields,
@@ -529,8 +535,10 @@ func (hm *HandshakeManager) DeleteHostInfo(hostinfo *HostInfo) {
 
 func (hm *HandshakeManager) unlockedDeleteHostInfo(hostinfo *HostInfo) {
 	for _, addr := range hostinfo.vpnAddrs {
+		if cur, ok := hm.vpnIps[addr]; ok && cur.hostinfo == hostinfo {
 			delete(hm.vpnIps, addr)
 		}
+	}
 
 	if len(hm.vpnIps) == 0 {
 		hm.vpnIps = map[netip.Addr]*HandshakeHostInfo{}
@@ -1077,7 +1085,7 @@ func (hm *HandshakeManager) sendHandshakeResponse(via ViaSender, msg []byte, hos
 		hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
 		// We received a valid handshake on this relay, so make sure the relay
 		// state reflects that, in case it had been marked Disestablished.
-		via.relayHI.relayState.UpdateRelayForByIdxState(via.remoteIdx, Established)
+		via.relayHI.relayState.UpdateRelayForByIdxState(via.relay.LocalIndex, Established)
 		f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false)
 		f.l.Info("Handshake message sent", append(logFields, "relay", via.relayHI.vpnAddrs[0])...)
 	}

hostmap.go

@@ -287,7 +287,6 @@ type HostInfo struct {
 type ViaSender struct {
 	UdpAddr   netip.AddrPort
 	relayHI   *HostInfo // relayHI is the host info object of the relay
-	remoteIdx uint32    // remoteIdx is the index included in the header of the received packet
 	relay     *Relay    // relay contains the rest of the relay information, including the PeerIP of the host trying to communicate with us.
 	IsRelayed bool      // IsRelayed is true if the packet was sent through a relay
 }
@@ -869,10 +868,28 @@ func (i *HostInfo) logger(l *slog.Logger) *slog.Logger {
 
 // Utility functions
 
-func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
+func localAddrs(l *slog.Logger, allowList *LocalAllowList) ([]netip.Addr, error) {
+	return collectLocalAddrs(l, allowList, localInterfaces, localInterfaceAddrs)
+}
+
+// collectLocalAddrs takes its enumerators as arguments so tests can drive the filtering and the
+// failure branches without depending on the addresses of whatever host they run on. It reports
+// failures to the caller rather than logging them, because it runs on every lighthouse update and
+// only the caller can tell a new failure from a repeat of the same one.
+func collectLocalAddrs(
+	l *slog.Logger,
+	allowList *LocalAllowList,
+	interfaces func() ([]net.Interface, error),
+	interfaceAddrs func(*net.Interface) ([]net.Addr, error),
+) ([]netip.Addr, error) {
 	//FIXME: This function is pretty garbage
 	var finalAddrs []netip.Addr
-	ifaces, _ := net.Interfaces()
+	var errs []error
+	ifaces, err := interfaces()
+	if err != nil {
+		return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err)
+	}
+
 	for _, i := range ifaces {
 		allow := allowList.AllowName(i.Name)
 		if l.Enabled(context.Background(), logging.LevelTrace) {
@@ -885,7 +902,12 @@ func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
 		if !allow {
 			continue
 		}
-		addrs, _ := i.Addrs()
+		addrs, err := interfaceAddrs(&i)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("failed to get addresses for %s: %w", i.Name, err))
+			continue
+		}
+
 		for _, rawAddr := range addrs {
 			var addr netip.Addr
 			switch v := rawAddr.(type) {
@@ -920,5 +942,5 @@ func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
 			}
 		}
 	}
-	return finalAddrs
+	return finalAddrs, errors.Join(errs...)
 }

hostmap_test.go

@@ -1,6 +1,8 @@
 package nebula
 
 import (
+	"errors"
+	"net"
 	"net/netip"
 	"slices"
 	"testing"
@@ -401,3 +403,83 @@ func TestHostMap_RelayState(t *testing.T) {
 	assert.Equal(t, []netip.Addr{}, h1.relayState.relays)
 
 }
+
+func TestCollectLocalAddrs(t *testing.T) {
+	ifaces := []net.Interface{
+		{Index: 1, Name: "lo"},
+		{Index: 2, Name: "eth0"},
+		{Index: 3, Name: "docker0"},
+	}
+	addrs := map[string][]net.Addr{
+		"lo": {
+			&net.IPNet{IP: net.ParseIP("127.0.0.1"), Mask: net.CIDRMask(8, 32)},
+			&net.IPNet{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)},
+		},
+		"eth0": {
+			&net.IPNet{IP: net.ParseIP("10.0.0.5"), Mask: net.CIDRMask(24, 32)},
+			&net.IPNet{IP: net.ParseIP("fe80::1"), Mask: net.CIDRMask(64, 128)},
+			&net.IPAddr{IP: net.ParseIP("fd00::5")},
+		},
+		"docker0": {
+			&net.IPNet{IP: net.ParseIP("172.17.0.1"), Mask: net.CIDRMask(16, 32)},
+		},
+	}
+
+	enumerate := func() ([]net.Interface, error) { return ifaces, nil }
+	addrsFor := func(i *net.Interface) ([]net.Addr, error) { return addrs[i.Name], nil }
+
+	// Loopback and link local are dropped, everything else on every interface is kept.
+	out, err := collectLocalAddrs(test.NewLogger(), nil, enumerate, addrsFor)
+	require.NoError(t, err)
+	assert.Equal(t, []netip.Addr{
+		netip.MustParseAddr("10.0.0.5"),
+		netip.MustParseAddr("fd00::5"),
+		netip.MustParseAddr("172.17.0.1"),
+	}, out)
+
+	// An interface the allow list rejects by name is never asked for its addresses.
+	c := config.NewC(test.NewLogger())
+	c.Settings["allowlist"] = map[string]any{
+		"interfaces": map[string]any{`docker.*`: false},
+	}
+	al, err := NewLocalAllowListFromConfig(c, "allowlist")
+	require.NoError(t, err)
+
+	asked := make(map[string]struct{})
+	countingAddrsFor := func(i *net.Interface) ([]net.Addr, error) {
+		asked[i.Name] = struct{}{}
+		return addrs[i.Name], nil
+	}
+	out, err = collectLocalAddrs(test.NewLogger(), al, enumerate, countingAddrsFor)
+	require.NoError(t, err)
+	assert.Equal(t, []netip.Addr{
+		netip.MustParseAddr("10.0.0.5"),
+		netip.MustParseAddr("fd00::5"),
+	}, out)
+	assert.NotContains(t, asked, "docker0")
+
+	// A failure to enumerate interfaces at all is reported rather than silently advertising nothing.
+	out, err = collectLocalAddrs(
+		test.NewLogger(),
+		nil,
+		func() ([]net.Interface, error) { return nil, errors.New("netlinkrib: permission denied") },
+		addrsFor,
+	)
+	assert.Nil(t, out)
+	require.EqualError(t, err, "failed to enumerate local interfaces: netlinkrib: permission denied")
+
+	// One interface failing is reported and skipped, the rest are still collected.
+	out, err = collectLocalAddrs(
+		test.NewLogger(),
+		nil,
+		enumerate,
+		func(i *net.Interface) ([]net.Addr, error) {
+			if i.Name == "eth0" {
+				return nil, errors.New("nope")
+			}
+			return addrs[i.Name], nil
+		},
+	)
+	assert.Equal(t, []netip.Addr{netip.MustParseAddr("172.17.0.1")}, out)
+	require.EqualError(t, err, "failed to get addresses for eth0: nope")
+}

inside.go

@@ -37,7 +37,7 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 		// routes packets from the Nebula addr to the Nebula addr through the Nebula
 		// TUN device.
 		if immediatelyForwardToSelf {
-			_, err := f.queues[q].Write(packet)
+			_, err := f.readers[q].Write(packet)
 			if err != nil {
 				f.l.Error("Failed to forward to tun", "error", err)
 			}
@@ -96,7 +96,7 @@ func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
 		return
 	}
 
-	_, err := f.queues[q].Write(out)
+	_, err := f.readers[q].Write(out)
 	if err != nil {
 		f.l.Error("Failed to write to tun", "error", err)
 	}
@@ -408,7 +408,7 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 		if err != nil {
 			hostinfo.logger(f.l).Error("Failed to write outgoing packet",
 				"error", err,
-				"udpAddr", remote,
+				"udpAddr", hr,
 			)
 		}
 	} else {

interface.go

@@ -4,9 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"log/slog"
 	"net/netip"
-	"runtime"
 	"slices"
 	"sync"
 	"sync/atomic"
@@ -20,9 +20,7 @@ import (
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/overlay"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/udp"
-	"github.com/slackhq/nebula/util"
 )
 
 const mtu = 9001
@@ -51,18 +49,6 @@ type InterfaceConfig struct {
 	reQueryWait     time.Duration
 
 	ConntrackCacheTimeout time.Duration
-
-	// CpuAffinity, when non-empty, names the CPUs each TUN reader goroutine
-	// should pin to. Queue i pins to CpuAffinity[i % len(CpuAffinity)] —
-	// shorter lists than `routines` cycle. Empty list keeps the default
-	// pin-to-(i % NumCPU) behavior. Only consulted when PinThreads is true.
-	CpuAffinity []int
-	// PinThreads controls whether each TUN reader OS thread is pinned to a
-	// single CPU (via tun.pin_threads, default true). Pinning keeps each
-	// goroutine's UDP sends on one XPS-selected NIC TX ring so per-flow
-	// packets stay ordered on the wire.
-	PinThreads bool
-
 	l                     *slog.Logger
 }
 
@@ -87,15 +73,6 @@ type Interface struct {
 	routines              int
 	disconnectInvalid     atomic.Bool
 	closed                atomic.Bool
-	// cpuAffinity, when non-empty, names the CPUs each TUN reader goroutine
-	// should pin to. Queue i pins to cpuAffinity[i % len(cpuAffinity)].
-	// Empty falls back to the default pin-to-(allowed CPU) behavior.
-	// Only consulted when pinThreads is true.
-	cpuAffinity []int
-	// pinThreads controls whether listenIn pins each TUN reader OS thread to
-	// a CPU at all (tun.pin_threads, default true). When false, threads are
-	// left free to migrate as on stock nebula.
-	pinThreads   bool
 	relayManager          *relayManager
 
 	tryPromoteEvery atomic.Uint32
@@ -113,7 +90,7 @@ type Interface struct {
 
 	ctx     context.Context
 	writers []udp.Conn
-	queues  []tio.Queue
+	readers []io.ReadWriteCloser
 	wg      sync.WaitGroup
 
 	// fatalErr holds the first unexpected reader error that caused shutdown.
@@ -212,6 +189,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 		routines:              c.routines,
 		version:               c.version,
 		writers:               make([]udp.Conn, c.routines),
+		readers:               make([]io.ReadWriteCloser, c.routines),
 		myVpnNetworks:         cs.myVpnNetworks,
 		myVpnNetworksTable:    cs.myVpnNetworksTable,
 		myVpnAddrs:            cs.myVpnAddrs,
@@ -220,8 +198,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 		relayManager:          c.relayManager,
 		connectionManager:     c.connectionManager,
 		conntrackCacheTimeout: c.ConntrackCacheTimeout,
-		cpuAffinity:           c.CpuAffinity,
-		pinThreads:            c.PinThreads,
 
 		metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)),
 		messageMetrics:   c.MessageMetrics,
@@ -264,26 +240,26 @@ func (f *Interface) activate() error {
 		"boringcrypto", boringEnabled(),
 	)
 
-	if f.routines > 1 && !f.outside.SupportsMultipleReaders() {
+	if f.routines > 1 {
+		if !f.inside.SupportsMultiqueue() || !f.outside.SupportsMultipleReaders() {
 			f.routines = 1
-		f.l.Warn("multiple udp readers are not supported on this platform, falling back to a single routine")
+			f.l.Warn("routines is not supported on this platform, falling back to a single routine")
+		}
 	}
 
-	// Prepare the tun queues. A device that can't open that many hands back
+	metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
-	// fewer (a single queue on platforms without multiqueue support) and we
+
-	// size the reader routines to what we actually got.
+	// Prepare n tun queues
-	queues, err := f.inside.Queues(f.routines)
+	var reader io.ReadWriteCloser = f.inside
+	for i := 0; i < f.routines; i++ {
+		if i > 0 {
+			reader, err = f.inside.NewMultiQueueReader()
 			if err != nil {
 				return err
 			}
-	if len(queues) < f.routines {
-		f.l.Warn("tun multiqueue is not supported on this platform, falling back to fewer routines",
-			"requested", f.routines, "opened", len(queues))
-		f.routines = len(queues)
 		}
-	f.queues = queues
+		f.readers[i] = reader
-
+	}
-	metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
 
 	// On error the caller owns the cleanup, Control.Start cancels the service context
 	// before releasing our resources so a waiter never observes a live context
@@ -305,7 +281,7 @@ func (f *Interface) run() {
 	// Launch n queues to read packets from tun dev
 	for i := 0; i < f.routines; i++ {
 		f.wg.Go(func() {
-			f.listenIn(f.queues[i], i)
+			f.listenIn(f.readers[i], i)
 		})
 	}
 
@@ -360,29 +336,8 @@ func (f *Interface) listenOut(i int) {
 	f.l.Debug("underlay reader is done", "reader", i)
 }
 
-func (f *Interface) listenIn(queue tio.Queue, i int) {
+func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
-	// Pinning this thread (and goroutine) to a single CPU keeps every UDP send from this goroutine going through
+	packet := make([]byte, mtu)
-	// the same TX ring on the nic (XPS selects the ring by CPU), so the wire sees per-flow order. Skip entirely
-	// when tun.pin_threads is false.
-	if f.pinThreads {
-		var cpu int
-		if n := len(f.cpuAffinity); n > 0 {
-			// Explicit tun.cpu_affinity list wins; parseCpuAffinity already
-			// validated the entries against the allowed CPU set.
-			cpu = f.cpuAffinity[i%n]
-		} else if allowed, err := util.AllowedCPUs(); err == nil && len(allowed) > 0 {
-			// Default: spread queues across the CPUs we're actually allowed to
-			// run on. Under a cpuset/taskset mask these aren't 0..NumCPU-1, so
-			// i % NumCPU would pick unrunnable IDs and every pin would fail.
-			cpu = allowed[i%len(allowed)]
-		} else {
-			cpu = i % runtime.NumCPU()
-		}
-		if err := util.PinThreadToCPU(cpu); err != nil {
-			f.l.Warn("failed to pin tun reader to CPU", "queue", i, "cpu", cpu, "err", err)
-		}
-	}
-
 	out := make([]byte, mtu)
 	fwPacket := &firewall.Packet{}
 	nb := make([]byte, 12, 12)
@@ -390,7 +345,7 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
 	conntrackCache := firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout)
 
 	for {
-		pkts, err := queue.Read()
+		n, err := reader.Read(packet)
 		if err != nil {
 			// Same shutdown noise handling as listenOut
 			if !f.closed.Load() && f.ctx.Err() == nil {
@@ -400,11 +355,7 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
 			break
 		}
 
-		for _, pkt := range pkts {
+		f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get())
-			// borrowed: pkt.Bytes is owned by the queue and only valid until
-			// the next Read; consumeInsidePacket reads it synchronously.
-			f.consumeInsidePacket(pkt.Bytes, fwPacket, nb, out, i, conntrackCache.Get())
-		}
 	}
 
 	f.l.Debug("overlay reader is done", "reader", i)

lighthouse.go

@@ -36,6 +36,14 @@ type LightHouse struct {
 	myVpnNetworksTable *bart.Lite
 	punchy             *Punchy
 
+	// localAddrsFn enumerates the underlay addresses we advertise. It is a field so tests can supply simulated
+	// addresses rather than whatever this machine's NICs happen to be. Set it before Start.
+	localAddrsFn func(*LocalAllowList) []netip.Addr
+
+	// lastLocalAddrsErr is the previous localAddrsFn failure. Enumeration runs on every update, so an
+	// unchanged failure is demoted to Debug rather than warning every lighthouse.interval forever.
+	lastLocalAddrsErr atomic.Pointer[string]
+
 	// Local cache of answers from light houses
 	// map of vpn addr to answers
 	addrMap map[netip.Addr]*RemoteList
@@ -107,6 +115,12 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
 		queryChan:          make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)),
 		l:                  l,
 	}
+	h.localAddrsFn = func(al *LocalAllowList) []netip.Addr {
+		addrs, err := localAddrs(h.l, al)
+		h.logLocalAddrsErr(err)
+		return addrs
+	}
+
 	lighthouses := make([]netip.Addr, 0)
 	h.lighthouses.Store(&lighthouses)
 	staticList := make(map[netip.Addr]struct{})
@@ -905,6 +919,26 @@ func (lh *LightHouse) TriggerUpdate() {
 	}
 }
 
+// logLocalAddrsErr reports a localAddrs failure at Warn the first time it is seen and at Debug while
+// it persists unchanged, so a permanent failure does not warn on every update forever.
+func (lh *LightHouse) logLocalAddrsErr(err error) {
+	if err == nil {
+		lh.lastLocalAddrsErr.Store(nil)
+		return
+	}
+
+	msg := err.Error()
+	prev := lh.lastLocalAddrsErr.Swap(&msg)
+	if prev != nil && *prev == msg {
+		if lh.l.Enabled(context.Background(), slog.LevelDebug) {
+			lh.l.Debug("Failed to collect local addresses to advertise", "error", err)
+		}
+		return
+	}
+
+	lh.l.Warn("Failed to collect local addresses to advertise", "error", err)
+}
+
 func (lh *LightHouse) SendUpdate() {
 	var v4 []*V4AddrPort
 	var v6 []*V6AddrPort
@@ -918,7 +952,7 @@ func (lh *LightHouse) SendUpdate() {
 	}
 
 	lal := lh.GetLocalAllowList()
-	for _, e := range localAddrs(lh.l, lal) {
+	for _, e := range lh.localAddrsFn(lal) {
 		if lh.myVpnNetworksTable.Contains(e) {
 			continue
 		}

lighthouse_test.go

@@ -1,7 +1,9 @@
 package nebula
 
 import (
+	"bytes"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"net/netip"
 	"testing"
@@ -738,3 +740,32 @@ func TestLighthouse_DeletesWork(t *testing.T) {
 	out = lh.Query(testHost)
 	assert.Nil(t, out)
 }
+
+func TestLightHouse_logLocalAddrsErr(t *testing.T) {
+	out := &bytes.Buffer{}
+	lh := &LightHouse{l: test.NewLoggerWithOutput(out)}
+
+	// The first sighting of a failure warns.
+	lh.logLocalAddrsErr(errors.New("permission denied"))
+	assert.Contains(t, out.String(), "level=WARN")
+	assert.Contains(t, out.String(), "permission denied")
+
+	// Repeating unchanged does not warn again, which is what keeps a permanent failure from warning
+	// on every lighthouse.interval for the life of the process.
+	out.Reset()
+	lh.logLocalAddrsErr(errors.New("permission denied"))
+	assert.NotContains(t, out.String(), "level=WARN")
+
+	// A different failure is a new event and warns.
+	out.Reset()
+	lh.logLocalAddrsErr(errors.New("something else"))
+	assert.Contains(t, out.String(), "level=WARN")
+	assert.Contains(t, out.String(), "something else")
+
+	// Recovering resets, so the same failure returning later warns again.
+	out.Reset()
+	lh.logLocalAddrsErr(nil)
+	assert.Empty(t, out.String())
+	lh.logLocalAddrsErr(errors.New("something else"))
+	assert.Contains(t, out.String(), "level=WARN")
+}

localaddrs.go

@@ -0,0 +1,13 @@
+//go:build !android
+
+package nebula
+
+import "net"
+
+func localInterfaces() ([]net.Interface, error) {
+	return net.Interfaces()
+}
+
+func localInterfaceAddrs(i *net.Interface) ([]net.Addr, error) {
+	return i.Addrs()
+}

localaddrs_android.go

@@ -0,0 +1,32 @@
+//go:build android
+
+package nebula
+
+import (
+	"net"
+
+	"github.com/wlynxg/anet"
+)
+
+// anet relies on //go:linkname and so needs -ldflags=-checklinkname=0 on Go 1.23+. Nebula ships no
+// Android binaries of its own, so that burden falls on consumers linking Android artifacts.
+
+func init() {
+	// anet only takes its bind-free path when it believes it is on API 30+, and detecting the running
+	// device's level requires cgo. Pin it so a CGO_ENABLED=0 build cannot quietly fall back to the
+	// denied path. The bind-free path is correct on older releases too, just unnecessary there.
+	anet.SetAndroidVersion(11)
+}
+
+// The app sandbox denies bind() on netlink_route_socket, so the stdlib's RTM_GETLINK enumeration
+// fails with EACCES and we advertise no underlay addresses at all. anet reads RTM_GETADDR from an
+// unbound socket instead, so this must not be collapsed back into net.Interfaces.
+func localInterfaces() ([]net.Interface, error) {
+	return anet.Interfaces()
+}
+
+// net.Interface.Addrs goes back through the denied netlink path, so addresses have to come from anet
+// as well. anet cannot report HardwareAddr, which localAddrs does not read.
+func localInterfaceAddrs(i *net.Interface) ([]net.Addr, error) {
+	return anet.InterfaceAddrsByInterface(i)
+}

main.go

@@ -7,7 +7,6 @@ import (
 	"net"
 	"net/netip"
 	"runtime/debug"
-	"slices"
 	"strings"
 	"time"
 
@@ -232,8 +231,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		relayManager:          NewRelayManager(ctx, l, hostMap, c),
 		punchy:                punchy,
 		ConntrackCacheTimeout: conntrackCacheTimeout,
-		CpuAffinity:           parseCpuAffinity(c, l, routines),
-		PinThreads:            c.GetBool("tun.pin_threads", true),
 		l:                     l,
 	}
 
@@ -271,6 +268,8 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 
 	attachCommands(l, c, ssh, ifce)
 
+	networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c)
+
 	return &Control{
 		state:                  StateReady,
 		f:                      ifce,
@@ -281,74 +280,11 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		statsStart:             stats.Start,
 		dnsStart:               ds.Start,
 		lighthouseStart:        lightHouse.StartUpdateWorker,
+		networkChangeStart:     networkChanges.Start,
 		connectionManagerStart: connManager.Start,
 	}, nil
 }
 
-// parseCpuAffinity reads `tun.cpu_affinity` from the config — a list of
-// integer CPU IDs, one per TUN reader goroutine. Empty / unset returns nil
-// (listenIn falls back to spreading queues across the allowed CPU set).
-// Length mismatch with `routines` is a warning, not an error: shorter lists
-// are modulo-cycled across queues, longer lists' tail is ignored. Invalid
-// entries (non-integer, or a CPU ID we're not allowed to run on) are also a
-// warning and disable the override entirely so we don't silently pin to the
-// wrong CPU. Entries are validated against the process's current affinity
-// mask (util.AllowedCPUs) rather than 0..NumCPU-1: under a cgroup cpuset or
-// taskset the runnable IDs are frequently not that contiguous range, and
-// pinning to an unrunnable ID always fails. If the allowed set can't be
-// determined we fall back to a plain non-negative check.
-func parseCpuAffinity(c *config.C, l *slog.Logger, routines int) []int {
-	raw := c.Get("tun.cpu_affinity")
-	if raw == nil {
-		return nil
-	}
-	rv, ok := raw.([]any)
-	if !ok {
-		l.Warn("tun.cpu_affinity must be a list of integers; ignoring", "value", raw)
-		return nil
-	}
-	// allowed is the set of CPU IDs we're actually permitted to run on. A nil
-	// slice (unsupported platform or lookup error) means "can't tell", so we
-	// only apply the weaker non-negative check in that case.
-	allowed, err := util.AllowedCPUs()
-	if err != nil {
-		l.Warn("could not determine allowed CPUs; validating tun.cpu_affinity against non-negative only", "error", err)
-		allowed = nil
-	}
-	cpus := make([]int, 0, len(rv))
-	for i, e := range rv {
-		var cpu int
-		switch v := e.(type) {
-		case int:
-			cpu = v
-		case int64:
-			cpu = int(v)
-		case float64:
-			cpu = int(v)
-		default:
-			l.Warn("tun.cpu_affinity entry not an integer; ignoring affinity",
-				"index", i, "value", e)
-			return nil
-		}
-		if cpu < 0 {
-			l.Warn("tun.cpu_affinity entry out of range; ignoring affinity",
-				"index", i, "cpu", cpu)
-			return nil
-		}
-		if len(allowed) > 0 && !slices.Contains(allowed, cpu) {
-			l.Warn("tun.cpu_affinity entry not in allowed CPU set; ignoring affinity",
-				"index", i, "cpu", cpu, "allowed", allowed)
-			return nil
-		}
-		cpus = append(cpus, cpu)
-	}
-	if len(cpus) != routines {
-		l.Warn("tun.cpu_affinity length doesn't match routines; queues will modulo-cycle through the list",
-			"affinity_len", len(cpus), "routines", routines)
-	}
-	return cpus
-}
-
 func moduleVersion() string {
 	info, ok := debug.ReadBuildInfo()
 	if !ok {

main_test.go

@@ -1,51 +0,0 @@
-package nebula
-
-import (
-	"testing"
-
-	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/test"
-	"github.com/slackhq/nebula/util"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestParseCpuAffinity(t *testing.T) {
-	l := test.NewLogger()
-
-	// newConfig returns a config.C with tun.cpu_affinity set to v. A nil v
-	// leaves the key unset.
-	newConfig := func(v any) *config.C {
-		c := config.NewC(l)
-		if v != nil {
-			c.Settings["tun"] = map[string]any{"cpu_affinity": v}
-		}
-		return c
-	}
-
-	// unset -> nil (listenIn falls back to spreading across the allowed set)
-	assert.Nil(t, parseCpuAffinity(newConfig(nil), l, 1))
-
-	// Pick a CPU we're actually allowed to run on so a valid list survives
-	// validation regardless of the host's affinity mask.
-	allowed, _ := util.AllowedCPUs()
-	validCPU := 0
-	if len(allowed) > 0 {
-		validCPU = allowed[0]
-	}
-
-	// valid list -> parsed through unchanged
-	assert.Equal(t, []int{validCPU, validCPU}, parseCpuAffinity(newConfig([]any{validCPU, validCPU}), l, 2))
-
-	// a negative entry is out of range on every platform -> disables the override
-	assert.Nil(t, parseCpuAffinity(newConfig([]any{validCPU, -1}), l, 2))
-
-	// a non-integer entry -> disables the override
-	assert.Nil(t, parseCpuAffinity(newConfig([]any{validCPU, "not-a-cpu"}), l, 2))
-
-	// a CPU id outside the allowed set -> disables the override. Only assertable
-	// where we can enumerate the allowed set (e.g. linux); 1<<20 is far beyond
-	// any representable CPU id so it can never be in the mask.
-	if len(allowed) > 0 {
-		assert.Nil(t, parseCpuAffinity(newConfig([]any{1 << 20}), l, 1))
-	}
-}

outside.go

@@ -102,27 +102,31 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 		return
 	}
 
+	if len(packet) < header.Len+hostinfo.ConnectionState.dKey.Overhead() {
+		f.messageMetrics.RxInvalid(1)
+		if f.l.Enabled(context.Background(), slog.LevelDebug) {
+			f.l.Debug("packet too small", "from", via, "length", len(packet))
+		}
+		return
+	}
+
 	// All remaining packets are encrypted
-	ci := hostinfo.ConnectionState
-	if !ci.window.Check(f.l, h.MessageCounter) {
-		return
-	}
-
-	// Relay packets are special
 	if isMessageRelay {
+		// Relay packets are special, this branch should always early-return
+		if err = hostinfo.ConnectionState.VerifyRelay(f.l, h.MessageCounter, packet, nb); err != nil {
+			if f.l.Enabled(context.Background(), slog.LevelDebug) {
+				hostinfo.logger(f.l).Debug("Failed to verify relay packet", "error", err, "from", via, "header", h)
+			}
+			return
+		}
 		f.handleOutsideRelayPacket(hostinfo, via, out, packet, h, fwPacket, lhf, nb, q, localCache)
-
 		return
 	}
 
-	out, err = f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
+	out, err = hostinfo.ConnectionState.Decrypt(f.l, h.MessageCounter, out, packet, nb)
 	if err != nil {
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			hostinfo.logger(f.l).Debug("Failed to decrypt packet",
+			hostinfo.logger(f.l).Debug("Failed to decrypt packet", "error", err, "from", via, "header", h)
-				"error", err,
-				"from", via,
-				"header", h,
-			)
 		}
 		return
 	}
@@ -151,7 +155,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 			// No-op, useful for the Roaming and connectionManager side-effects above
 		case header.TestRequest:
 			//recycle the input packet ciphertext as our output buffer
-			f.send(header.Test, header.TestReply, ci, hostinfo, out, nb, packet)
+			f.send(header.Test, header.TestReply, hostinfo.ConnectionState, hostinfo, out, nb, packet)
 		default:
 			hostinfo.logger(f.l).Error("IsValidSubType was true, but unexpected test subtype seen", "from", via, "header", h)
 			return
@@ -170,27 +174,8 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 }
 
 func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
-	// The entire body is sent as AD, not encrypted.
+	// Successfully validated the thing. Get rid of the Relay header and the AEAD tag
-	// The packet consists of a 16-byte parsed Nebula header, Associated Data-protected payload, and a trailing 16-byte AEAD signature value.
+	signedPayload := packet[header.Len : len(packet)-hostinfo.ConnectionState.dKey.Overhead()]
-	// The packet is guaranteed to be at least 16 bytes at this point, b/c it got past the h.Parse() call above. If it's
-	// otherwise malformed (meaning, there is no trailing 16 byte AEAD value), then this will result in at worst a 0-length slice
-	// which will gracefully fail in the DecryptDanger call.
-	signedPayload := packet[:len(packet)-hostinfo.ConnectionState.dKey.Overhead()]
-	signatureValue := packet[len(packet)-hostinfo.ConnectionState.dKey.Overhead():]
-	var err error
-	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, signedPayload, signatureValue, h.MessageCounter, nb)
-	if err != nil {
-		return
-	}
-	// Advance the replay window now that the frame is authenticated
-	if !hostinfo.ConnectionState.window.Update(f.l, h.MessageCounter) {
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			hostinfo.logger(f.l).Debug("dropping out of window relay packet", "header", h)
-		}
-		return
-	}
-	// Successfully validated the thing. Get rid of the Relay header.
-	signedPayload = signedPayload[header.Len:]
 	// Pull the Roaming parts up here, and return in all call paths.
 	f.handleHostRoaming(hostinfo, via)
 	// Track usage of both the HostInfo and the Relay for the received & authenticated packet
@@ -214,7 +199,6 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
 		via = ViaSender{
 			UdpAddr:   via.UdpAddr,
 			relayHI:   hostinfo,
-			remoteIdx: relay.RemoteIndex,
 			relay:     relay,
 			IsRelayed: true,
 		}
@@ -235,9 +219,10 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
 		if targetRelay.State == Established {
 			switch targetRelay.Type {
 			case ForwardingType:
-				// Forward this packet through the relay tunnel
+				// Forward this packet through the relay tunnel, rebuilding it in place.
-				// Find the target HostInfo
+				// Encode overwrites the old outer header, and the new AEAD tag lands where the old one was
-				f.SendVia(targetHI, targetRelay, signedPayload, nb, out, false)
+				fwdBuf := packet[:0:len(packet)] // Cap to len(packet) to protect memory from a larger parent buffer
+				f.SendVia(targetHI, targetRelay, signedPayload, nb, fwdBuf, true)
 			case TerminalType:
 				hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
 				return
@@ -504,20 +489,6 @@ func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
 	return nil
 }
 
-func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, out []byte, packet []byte, h *header.H, nb []byte) ([]byte, error) {
-	var err error
-	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], mc, nb)
-	if err != nil {
-		return nil, err
-	}
-
-	if !hostinfo.ConnectionState.window.Update(f.l, mc) {
-		return nil, ErrOutOfWindow
-	}
-
-	return out, nil
-}
-
 func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache) {
 	err := newPacket(out, true, fwPacket)
 	if err != nil {
@@ -542,7 +513,7 @@ func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, out []byte, p
 		return
 	}
 
-	_, err = f.queues[q].Write(out)
+	_, err = f.readers[q].Write(out)
 	if err != nil {
 		f.l.Error("Failed to write to tun", "error", err)
 	}

overlay/device.go

@@ -4,25 +4,15 @@ import (
 	"io"
 	"net/netip"
 
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 )
 
-// defaultBatchBufSize is the per-Queue scratch size for Read. 65535 covers
-// any single IP packet.
-const defaultBatchBufSize = 65535
-
 type Device interface {
-	io.Closer
+	io.ReadWriteCloser
 	Activate() error
 	Networks() []netip.Prefix
 	Name() string
 	RoutesFor(netip.Addr) routing.Gateways
-	// Queues returns the device's packet queues, opening additional ones as
+	SupportsMultiqueue() bool
-	// needed until there are n. Platforms without multiqueue support return
+	NewMultiQueueReader() (io.ReadWriteCloser, error)
-	// their single queue regardless of n, so callers must size reader loops
-	// to len(result), not n; implementations never return more than n. An
-	// error means a queue that should have opened could not; the caller owns
-	// cleanup via Close. Called once, during interface activation.
-	Queues(n int) ([]tio.Queue, error)
 }

overlay/overlaytest/noop.go

@@ -3,9 +3,10 @@
 package overlaytest
 
 import (
+	"errors"
+	"io"
 	"net/netip"
 
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 )
 
@@ -30,16 +31,20 @@ func (NoopTun) Name() string {
 	return "noop"
 }
 
-func (NoopTun) Read() ([]tio.Packet, error) {
+func (NoopTun) Read([]byte) (int, error) {
-	return nil, nil
+	return 0, nil
 }
 
 func (NoopTun) Write([]byte) (int, error) {
 	return 0, nil
 }
 
-func (NoopTun) Queues(int) ([]tio.Queue, error) {
+func (NoopTun) SupportsMultiqueue() bool {
-	return []tio.Queue{NoopTun{}}, nil
+	return false
+}
+
+func (NoopTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, errors.New("unsupported")
 }
 
 func (NoopTun) Close() error {

overlay/tio/blockon_linux.go

@@ -1,45 +0,0 @@
-//go:build linux && !android
-// +build linux,!android
-
-package tio
-
-import (
-	"os"
-
-	"golang.org/x/sys/unix"
-)
-
-// blockOn parks the calling goroutine until fd is ready (events is POLLIN for
-// reads, POLLOUT for writes) or shutdownFd signals teardown. It builds the
-// pollfd array on the stack every call, so concurrent callers on the same
-// Queue never share Revents storage.
-//
-// Returns os.ErrClosed when shutdown was signaled (POLLIN on shutdownFd)
-// or either fd reported a problem condition (POLLHUP|POLLNVAL|POLLERR).
-func blockOn(fd, shutdownFd int32, events int16) error {
-	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
-	pfds := [2]unix.PollFd{
-		{Fd: fd, Events: events},
-		{Fd: shutdownFd, Events: unix.POLLIN},
-	}
-	var err error
-	for {
-		_, err = unix.Poll(pfds[:], -1)
-		if err != unix.EINTR {
-			break
-		}
-	}
-	tunEvents := pfds[0].Revents
-	shutdownEvents := pfds[1].Revents
-	// Check err before trusting the potentially bogus bits we just got.
-	if err != nil {
-		return err
-	}
-	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
-		return os.ErrClosed
-	}
-	if tunEvents&problemFlags != 0 {
-		return os.ErrClosed
-	}
-	return nil
-}

overlay/tio/queueset_poll_linux.go

@@ -1,90 +0,0 @@
-//go:build linux && !android
-// +build linux,!android
-
-package tio
-
-import (
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"sync/atomic"
-
-	"golang.org/x/sys/unix"
-)
-
-type pollQueueSet struct {
-	pq []*Poll
-	// pqi is exactly the same as pq, but stored as the interface type
-	pqi        []Queue
-	shutdownFd int
-	closed     atomic.Bool
-}
-
-func NewPollQueueSet() (QueueSet, error) {
-	shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create eventfd: %w", err)
-	}
-
-	out := &pollQueueSet{
-		pq:         []*Poll{},
-		pqi:        []Queue{},
-		shutdownFd: shutdownFd,
-	}
-
-	return out, nil
-}
-
-func (c *pollQueueSet) Queues() []Queue {
-	return c.pqi
-}
-
-func (c *pollQueueSet) Add(fd int) error {
-	x, err := newPoll(fd, c.shutdownFd)
-	if err != nil {
-		return err
-	}
-	c.pq = append(c.pq, x)
-	c.pqi = append(c.pqi, x)
-
-	return nil
-}
-
-func (c *pollQueueSet) wakeForShutdown() error {
-	var buf [8]byte
-	binary.NativeEndian.PutUint64(buf[:], 1)
-	_, err := unix.Write(int(c.shutdownFd), buf[:])
-	return err
-}
-
-func (c *pollQueueSet) Close() error {
-	if c.closed.Swap(true) {
-		return nil
-	}
-
-	errs := []error{}
-
-	// Wake any reader blocked in poll so it observes POLLIN on the shutdown
-	// eventfd and returns os.ErrClosed.
-	if err := c.wakeForShutdown(); err != nil {
-		errs = append(errs, err)
-	}
-
-	// Close the per-queue tun fds; this also unblocks any in-flight reads.
-	// The per-queue Close deliberately leaves shutdownFd alone - it belongs
-	// to this container.
-	for _, x := range c.pq {
-		if err := x.Close(); err != nil {
-			errs = append(errs, err)
-		}
-	}
-
-	// Close the shutdown eventfd last: every reader's pollfd set references
-	// it, so it must outlive the wake + per-queue teardown above.
-	if err := unix.Close(c.shutdownFd); err != nil {
-		errs = append(errs, err)
-	}
-	c.shutdownFd = -1
-
-	return errors.Join(errs...)
-}

overlay/tio/single.go

@@ -1,50 +0,0 @@
-package tio
-
-import "io"
-
-// singleQueue adapts a legacy one-datagram-per-Read source into a Queue.
-// Read fills a private scratch buffer and returns exactly one Packet whose
-// Bytes borrow from that buffer, valid only until the next Read, per the
-// Queue contract. Single-reader like every Queue; Write is exactly as safe
-// for concurrent use as the underlying source's Write.
-type singleQueue struct {
-	rw     io.ReadWriter
-	closer io.Closer // nil: Close is a no-op (the source is shared and owned elsewhere)
-	buf    []byte
-	ret    [1]Packet
-}
-
-// NewSingleQueue wraps a one-datagram-per-Read ReadWriteCloser (a legacy tun
-// device) into a Queue. bufSize is the per-queue read scratch size and must
-// be at least the largest datagram the source can return. Close closes rwc.
-func NewSingleQueue(rwc io.ReadWriteCloser, bufSize int) Queue {
-	return &singleQueue{rw: rwc, closer: rwc, buf: make([]byte, bufSize)}
-}
-
-// NewSingleQueueNoClose is NewSingleQueue for a source owned by someone else,
-// e.g. several queues sharing one device. Close on the returned Queue is a
-// no-op so one queue can't tear the shared source out from under its
-// siblings; the owner remains responsible for closing the source itself.
-func NewSingleQueueNoClose(rw io.ReadWriter, bufSize int) Queue {
-	return &singleQueue{rw: rw, buf: make([]byte, bufSize)}
-}
-
-func (q *singleQueue) Read() ([]Packet, error) {
-	n, err := q.rw.Read(q.buf)
-	if err != nil {
-		return nil, err
-	}
-	q.ret[0] = Packet{Bytes: q.buf[:n]}
-	return q.ret[:], nil
-}
-
-func (q *singleQueue) Write(p []byte) (int, error) {
-	return q.rw.Write(p)
-}
-
-func (q *singleQueue) Close() error {
-	if q.closer == nil {
-		return nil
-	}
-	return q.closer.Close()
-}

overlay/tio/tio.go

@@ -1,52 +0,0 @@
-package tio
-
-import (
-	"io"
-)
-
-// QueueSet holds one or many Queue objects and helps close them in an orderly way.
-type QueueSet interface {
-	io.Closer
-	Queues() []Queue
-
-	// Add takes a tun fd, adds it to the set, and prepares it for use as a Queue.
-	Add(fd int) error
-}
-
-// Queue is a readable/writable packet queue. Concurrency contract: a single
-// read goroutine drives Read; plain Write is safe for concurrent callers.
-type Queue interface {
-	io.Closer
-
-	// Read returns one or more packets. The returned Packet.Bytes slices
-	// are borrowed from the Queue's internal buffer and are only valid
-	// until the next Read or Close on this Queue - callers must encrypt
-	// or copy each slice before the next call. Single-reader only: not
-	// safe for concurrent Reads (it reuses per-queue rx scratch each call).
-	Read() ([]Packet, error)
-
-	// Write emits a single packet on the plaintext (outside→inside)
-	// delivery path. Safe for concurrent use.
-	Write(p []byte) (int, error)
-}
-
-// Packet is the unit Queue.Read returns. Bytes points into the queue's
-// internal buffer and is only valid until the next Read or Close on the
-// queue that produced it.
-type Packet struct {
-	Bytes []byte
-}
-
-// Clone returns a Packet whose Bytes is a freshly allocated copy of p.Bytes,
-// safe to retain past the next Read or Close on the originating Queue.
-// Use this only when a caller genuinely needs to outlive the borrowed-slice
-// contract — the hot path reads should continue to consume the borrow
-// synchronously to avoid the allocation.
-func (p Packet) Clone() Packet {
-	if p.Bytes == nil {
-		return p
-	}
-	cp := make([]byte, len(p.Bytes))
-	copy(cp, p.Bytes)
-	return Packet{Bytes: cp}
-}

overlay/tio/tio_poll_linux.go

@@ -1,116 +0,0 @@
-//go:build linux && !android
-// +build linux,!android
-
-package tio
-
-import (
-	"fmt"
-	"os"
-	"sync/atomic"
-
-	"golang.org/x/sys/unix"
-)
-
-// Maximum size we accept for a single read from a TUN. 65535 covers any
-// single IP packet.
-const tunReadBufSize = 65535
-
-type Poll struct {
-	fd         int
-	shutdownFd int
-	closed     atomic.Bool
-
-	readBuf  []byte
-	batchRet [1]Packet
-}
-
-// newPoll wraps an existing tun fd. On failure it does NOT close fd: the
-// caller owns fd and is the sole closer (see pollQueueSet.Add callers in
-// overlay/tun_linux.go, which unix.Close on Add error). This keeps closes
-// at exactly one on every path.
-func newPoll(fd int, shutdownFd int) (*Poll, error) {
-	if err := unix.SetNonblock(fd, true); err != nil {
-		return nil, fmt.Errorf("failed to set Poll device as nonblocking: %w", err)
-	}
-
-	out := &Poll{
-		fd:         fd,
-		shutdownFd: shutdownFd,
-		readBuf:    make([]byte, tunReadBufSize),
-	}
-	return out, nil
-}
-
-// blockOnRead waits until the Poll fd is readable or shutdown has been signaled.
-// Returns os.ErrClosed if Close was called.
-func (t *Poll) blockOnRead() error {
-	return blockOn(int32(t.fd), int32(t.shutdownFd), unix.POLLIN)
-}
-
-func (t *Poll) blockOnWrite() error {
-	return blockOn(int32(t.fd), int32(t.shutdownFd), unix.POLLOUT)
-}
-
-func (t *Poll) Read() ([]Packet, error) {
-	n, err := t.readOne(t.readBuf)
-	if err != nil {
-		return nil, err
-	}
-	t.batchRet[0] = Packet{Bytes: t.readBuf[:n]}
-	return t.batchRet[:], nil
-}
-
-func (t *Poll) readOne(to []byte) (int, error) {
-	for {
-		n, errno := unix.Read(t.fd, to)
-		if errno == nil {
-			return n, nil
-		}
-		switch errno {
-		case unix.EAGAIN:
-			if err := t.blockOnRead(); err != nil {
-				return 0, err
-			}
-		case unix.EINTR:
-			// retry
-		case unix.EBADF:
-			return 0, os.ErrClosed
-		default:
-			return 0, errno
-		}
-	}
-}
-
-// Write is safe for concurrent use
-func (t *Poll) Write(from []byte) (int, error) {
-	for {
-		n, errno := unix.Write(t.fd, from)
-		if errno == nil {
-			return n, nil
-		}
-		switch errno {
-		case unix.EAGAIN:
-			if err := t.blockOnWrite(); err != nil {
-				return 0, err
-			}
-		case unix.EINTR:
-			// retry
-		case unix.EBADF:
-			return 0, os.ErrClosed
-		default:
-			return 0, errno
-		}
-	}
-}
-
-func (t *Poll) Close() error {
-	if t.closed.Swap(true) {
-		return nil
-	}
-	//shutdownFd is owned by the container, so we should not close it
-	// Close the underlying fd but do NOT null t.fd: a reader may still be
-	// loading it in readOne, and mutating the field would race that load.
-	// It gets EBADF -> os.ErrClosed (or wakes via the shutdown eventfd's
-	// ppoll first). closed.Swap already guarantees we only close once.
-	return unix.Close(t.fd)
-}

overlay/tio/tun_file_linux_test.go

@@ -1,208 +0,0 @@
-//go:build linux && !android && !e2e_testing
-// +build linux,!android,!e2e_testing
-
-package tio
-
-import (
-	"errors"
-	"os"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-	"golang.org/x/sys/unix"
-)
-
-// newReadPipe returns a read fd. The matching write fd is registered for cleanup.
-// The caller takes ownership of the read fd (pass it into a QueueSet).
-func newReadPipe(t *testing.T) int {
-	t.Helper()
-	var fds [2]int
-	if err := unix.Pipe2(fds[:], unix.O_CLOEXEC); err != nil {
-		t.Fatalf("pipe2: %v", err)
-	}
-	t.Cleanup(func() { _ = unix.Close(fds[1]) })
-	return fds[0]
-}
-
-func TestPoll_WakeForShutdown_WakesFriends(t *testing.T) {
-	pipe1 := newReadPipe(t)
-	pipe2 := newReadPipe(t)
-	parent, err := NewPollQueueSet()
-	require.NoError(t, err)
-	require.NoError(t, parent.Add(pipe1))
-	require.NoError(t, parent.Add(pipe2))
-	t.Cleanup(func() {
-		_ = unix.Close(pipe1)
-		_ = unix.Close(pipe2)
-	})
-
-	readers := parent.Queues()
-	errs := make([]error, len(readers))
-	var wg sync.WaitGroup
-	for i, r := range readers {
-		wg.Add(1)
-		go func(i int, r Queue) {
-			defer wg.Done()
-			_, errs[i] = r.Read()
-		}(i, r)
-	}
-
-	time.Sleep(50 * time.Millisecond)
-
-	if err := parent.Close(); err != nil {
-		t.Fatalf("Close: %v", err)
-	}
-
-	done := make(chan struct{})
-	go func() { wg.Wait(); close(done) }()
-	select {
-	case <-done:
-	case <-time.After(2 * time.Second):
-		t.Fatal("readers did not wake")
-	}
-
-	for i, err := range errs {
-		if !errors.Is(err, os.ErrClosed) {
-			t.Errorf("reader %d: expected os.ErrClosed, got %v", i, err)
-		}
-	}
-}
-
-// TestPoll_ConcurrentWrite_NoRace hammers a single Poll queue from two writer
-// goroutines while a reader drains the other end of the pipe. The writers
-// overflow the pipe buffer, so both repeatedly park in blockOnWrite at the same
-// time — the exact scenario that raced on the old shared writePoll member
-// array. Run under -race; a shared-array regression trips the detector here.
-func TestPoll_ConcurrentWrite_NoRace(t *testing.T) {
-	var fds [2]int
-	require.NoError(t, unix.Pipe2(fds[:], unix.O_CLOEXEC))
-	readFd, writeFd := fds[0], fds[1]
-
-	shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = unix.Close(shutdownFd) })
-
-	p, err := newPoll(writeFd, shutdownFd)
-	require.NoError(t, err)
-
-	const writers = 2
-	const perWriter = 4000
-	payload := make([]byte, 100)
-	total := writers * perWriter * len(payload)
-
-	// Reader: drain the read end (blocking) until every writer's bytes are
-	// consumed, so the writers keep making progress rather than wedging on a
-	// permanently full pipe.
-	readDone := make(chan struct{})
-	go func() {
-		defer close(readDone)
-		buf := make([]byte, 4096)
-		got := 0
-		for got < total {
-			n, rerr := unix.Read(readFd, buf)
-			got += n
-			if rerr != nil {
-				if rerr == unix.EINTR {
-					continue
-				}
-				return
-			}
-			if n == 0 { // EOF
-				return
-			}
-		}
-	}()
-
-	var wg sync.WaitGroup
-	for w := 0; w < writers; w++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			for i := 0; i < perWriter; i++ {
-				if _, werr := p.Write(payload); werr != nil {
-					t.Errorf("write: %v", werr)
-					return
-				}
-			}
-		}()
-	}
-	wg.Wait()
-
-	select {
-	case <-readDone:
-	case <-time.After(10 * time.Second):
-		t.Fatal("reader did not drain")
-	}
-
-	require.NoError(t, p.Close())
-	_ = unix.Close(readFd)
-}
-
-// TestPoll_NewPoll_DoesNotCloseFdOnFailure pins the ownership rule: when
-// newPoll fails, it must leave fd open so the caller (pollQueueSet.Add's
-// callers in tun_linux.go) is the sole closer. If newPoll also closed fd,
-// the poll path would double-close on Add error. We force the failure with
-// an O_PATH descriptor: fcntl(F_SETFL) — which SetNonblock performs — is not
-// permitted on O_PATH fds and fails with EBADF, while the fd itself stays
-// open so we can observe that newPoll left it alone.
-func TestPoll_NewPoll_DoesNotCloseFdOnFailure(t *testing.T) {
-	fd, err := unix.Open("/", unix.O_PATH|unix.O_CLOEXEC, 0)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = unix.Close(fd) })
-
-	p, err := newPoll(fd, 1)
-	require.Error(t, err, "SetNonblock on an O_PATH fd should fail")
-	require.Nil(t, p)
-
-	// If newPoll had closed fd, F_GETFD would report it closed. It staying
-	// open proves newPoll left the fd for the caller to close exactly once.
-	require.True(t, fdOpen(t, fd), "newPoll must not close fd on failure; caller is the sole closer")
-}
-
-func TestPoll_Close_Idempotent(t *testing.T) {
-	tf, err := newPoll(newReadPipe(t), 1)
-	require.NoError(t, err)
-	if err := tf.Close(); err != nil {
-		t.Fatalf("first Close: %v", err)
-	}
-	if err := tf.Close(); err != nil {
-		t.Fatalf("second Close should be a no-op, got %v", err)
-	}
-}
-
-// fdOpen reports whether fd currently refers to an open file description.
-// A closed (or never-allocated) fd makes F_GETFD fail with EBADF.
-func fdOpen(t *testing.T, fd int) bool {
-	t.Helper()
-	_, err := unix.FcntlInt(uintptr(fd), unix.F_GETFD, 0)
-	if err == nil {
-		return true
-	}
-	if errors.Is(err, unix.EBADF) {
-		return false
-	}
-	t.Fatalf("unexpected fcntl(F_GETFD) error on fd %d: %v", fd, err)
-	return false
-}
-
-// TestPollQueueSet_Close_ClosesShutdownFd is the regression test for the
-// leaked shutdown eventfd: the container that owns shutdownFd must close it in
-// Close, and a second Close must be a safe no-op.
-func TestPollQueueSet_Close_ClosesShutdownFd(t *testing.T) {
-	qs, err := NewPollQueueSet()
-	require.NoError(t, err)
-	c, ok := qs.(*pollQueueSet)
-	require.True(t, ok)
-	require.NoError(t, qs.Add(newReadPipe(t)))
-
-	shutdownFd := c.shutdownFd
-	require.True(t, fdOpen(t, shutdownFd), "shutdown eventfd should be open before Close")
-
-	require.NoError(t, qs.Close())
-	require.False(t, fdOpen(t, shutdownFd), "shutdown eventfd should be closed after Close")
-
-	// Second Close must not touch fds (shutdownFd is now -1) and must return nil.
-	require.NoError(t, qs.Close())
-}

overlay/tun_android.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 )
@@ -64,7 +63,7 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
 	return r
 }
 
-func (t *tun) Activate() error {
+func (t tun) Activate() error {
 	return nil
 }
 
@@ -97,6 +96,10 @@ func (t *tun) Name() string {
 	return "android"
 }
 
-func (t *tun) Queues(int) ([]tio.Queue, error) {
+func (t *tun) SupportsMultiqueue() bool {
-	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
+	return false
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for android")
 }

overlay/tun_darwin.go

@@ -6,6 +6,7 @@ package overlay
 import (
 	"errors"
 	"fmt"
+	"io"
 	"log/slog"
 	"net/netip"
 	"os"
@@ -15,7 +16,6 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	netroute "golang.org/x/net/route"
@@ -606,6 +606,10 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) Queues(int) ([]tio.Queue, error) {
+func (t *tun) SupportsMultiqueue() bool {
-	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
+	return false
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
 }

overlay/tun_disabled.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/rcrowley/go-metrics"
 	"github.com/slackhq/nebula/iputil"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 )
 
@@ -24,23 +23,6 @@ type disabledTun struct {
 	l  *slog.Logger
 }
 
-// Read hands the next queued packet to a reader, copying it into b. Reads
-// from concurrent queues are safe: the channel receive serializes them and
-// each queue copies into its own private scratch buffer.
-func (t *disabledTun) Read(b []byte) (int, error) {
-	r, ok := <-t.read
-	if !ok {
-		return 0, io.EOF
-	}
-
-	t.tx.Inc(1)
-	if t.l.Enabled(context.Background(), slog.LevelDebug) {
-		t.l.Debug("Write payload", "raw", prettyPacket(r))
-	}
-
-	return copy(b, r), nil
-}
-
 func newDisabledTun(vpnNetworks []netip.Prefix, queueLen int, metricsEnabled bool, l *slog.Logger) *disabledTun {
 	tun := &disabledTun{
 		vpnNetworks: vpnNetworks,
@@ -75,6 +57,24 @@ func (*disabledTun) Name() string {
 	return "disabled"
 }
 
+func (t *disabledTun) Read(b []byte) (int, error) {
+	r, ok := <-t.read
+	if !ok {
+		return 0, io.EOF
+	}
+
+	if len(r) > len(b) {
+		return 0, fmt.Errorf("packet larger than mtu: %d > %d bytes", len(r), len(b))
+	}
+
+	t.tx.Inc(1)
+	if t.l.Enabled(context.Background(), slog.LevelDebug) {
+		t.l.Debug("Write payload", "raw", prettyPacket(r))
+	}
+
+	return copy(b, r), nil
+}
+
 func (t *disabledTun) handleICMPEchoRequest(b []byte) bool {
 	out := make([]byte, len(b))
 	out = iputil.CreateICMPEchoResponse(b, out)
@@ -106,14 +106,12 @@ func (t *disabledTun) Write(b []byte) (int, error) {
 	return len(b), nil
 }
 
-func (t *disabledTun) Queues(n int) ([]tio.Queue, error) {
+func (t *disabledTun) SupportsMultiqueue() bool {
-	out := make([]tio.Queue, n)
+	return true
-	for i := range out {
-		// NoClose: the shared channel and metrics are owned by the
-		// disabledTun; Close on the device tears them down once for everybody.
-		out[i] = tio.NewSingleQueueNoClose(t, defaultBatchBufSize)
 }
-	return out, nil
+
+func (t *disabledTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return t, nil
 }
 
 func (t *disabledTun) Close() error {

overlay/tun_file_linux_test.go

@@ -0,0 +1,120 @@
+//go:build linux && !android && !e2e_testing
+// +build linux,!android,!e2e_testing
+
+package overlay
+
+import (
+	"errors"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// newReadPipe returns a read fd. The matching write fd is registered for cleanup.
+// The caller takes ownership of the read fd (pass it to newTunFd / newFriend).
+func newReadPipe(t *testing.T) int {
+	t.Helper()
+	var fds [2]int
+	if err := unix.Pipe2(fds[:], unix.O_CLOEXEC); err != nil {
+		t.Fatalf("pipe2: %v", err)
+	}
+	t.Cleanup(func() { _ = unix.Close(fds[1]) })
+	return fds[0]
+}
+
+func TestTunFile_WakeForShutdown_UnblocksRead(t *testing.T) {
+	tf, err := newTunFd(newReadPipe(t))
+	if err != nil {
+		t.Fatalf("newTunFd: %v", err)
+	}
+	t.Cleanup(func() { _ = tf.Close() })
+
+	done := make(chan error, 1)
+	go func() {
+		_, err := tf.Read(make([]byte, 64))
+		done <- err
+	}()
+
+	// Verify Read is actually blocked in poll.
+	select {
+	case err := <-done:
+		t.Fatalf("Read returned before shutdown signal: %v", err)
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	if err := tf.wakeForShutdown(); err != nil {
+		t.Fatalf("wakeForShutdown: %v", err)
+	}
+
+	select {
+	case err := <-done:
+		if !errors.Is(err, os.ErrClosed) {
+			t.Fatalf("expected os.ErrClosed, got %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("Read did not wake on shutdown")
+	}
+}
+
+func TestTunFile_WakeForShutdown_WakesFriends(t *testing.T) {
+	parent, err := newTunFd(newReadPipe(t))
+	if err != nil {
+		t.Fatalf("newTunFd: %v", err)
+	}
+	friend, err := parent.newFriend(newReadPipe(t))
+	if err != nil {
+		_ = parent.Close()
+		t.Fatalf("newFriend: %v", err)
+	}
+	t.Cleanup(func() {
+		_ = friend.Close()
+		_ = parent.Close()
+	})
+
+	readers := []*tunFile{parent, friend}
+	errs := make([]error, len(readers))
+	var wg sync.WaitGroup
+	for i, r := range readers {
+		wg.Add(1)
+		go func(i int, r *tunFile) {
+			defer wg.Done()
+			_, errs[i] = r.Read(make([]byte, 64))
+		}(i, r)
+	}
+
+	time.Sleep(50 * time.Millisecond)
+
+	if err := parent.wakeForShutdown(); err != nil {
+		t.Fatalf("wakeForShutdown: %v", err)
+	}
+
+	done := make(chan struct{})
+	go func() { wg.Wait(); close(done) }()
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("readers did not wake")
+	}
+
+	for i, err := range errs {
+		if !errors.Is(err, os.ErrClosed) {
+			t.Errorf("reader %d: expected os.ErrClosed, got %v", i, err)
+		}
+	}
+}
+
+func TestTunFile_Close_Idempotent(t *testing.T) {
+	tf, err := newTunFd(newReadPipe(t))
+	if err != nil {
+		t.Fatalf("newTunFd: %v", err)
+	}
+	if err := tf.Close(); err != nil {
+		t.Fatalf("first Close: %v", err)
+	}
+	if err := tf.Close(); err != nil {
+		t.Fatalf("second Close should be a no-op, got %v", err)
+	}
+}

overlay/tun_freebsd.go

@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"io"
 	"io/fs"
 	"log/slog"
 	"net/netip"
@@ -19,7 +20,7 @@ import (
 	"github.com/gaissmai/bart"
 
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
+
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	netroute "golang.org/x/net/route"
@@ -560,8 +561,12 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) Queues(int) ([]tio.Queue, error) {
+func (t *tun) SupportsMultiqueue() bool {
-	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
+	return false
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
 }
 
 func (t *tun) addRoutes(logErrors bool) error {

overlay/tun_ios.go

@@ -16,7 +16,6 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	"golang.org/x/sys/unix"
@@ -160,6 +159,10 @@ func (t *tun) Name() string {
 	return "iOS"
 }
 
-func (t *tun) Queues(int) ([]tio.Queue, error) {
+func (t *tun) SupportsMultiqueue() bool {
-	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
+	return false
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for ios")
 }

overlay/tun_linux.go

@@ -4,7 +4,9 @@
 package overlay
 
 import (
+	"encoding/binary"
 	"fmt"
+	"io"
 	"log/slog"
 	"net"
 	"net/netip"
@@ -17,15 +19,180 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
 )
 
+// tunFile wraps a TUN file descriptor with poll-based reads. The FD provided will be changed to non-blocking.
+// A shared eventfd allows Close to wake all readers blocked in poll.
+type tunFile struct {
+	fd         int
+	shutdownFd int
+	lastOne    bool
+	readPoll   [2]unix.PollFd
+	writePoll  [2]unix.PollFd
+	closed     bool
+}
+
+// newFriend makes a tunFile for a MultiQueueReader that copies the shutdown eventfd from the parent tun
+func (r *tunFile) newFriend(fd int) (*tunFile, error) {
+	if err := unix.SetNonblock(fd, true); err != nil {
+		return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
+	}
+	return &tunFile{
+		fd:         fd,
+		shutdownFd: r.shutdownFd,
+		readPoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLIN},
+			{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
+		},
+		writePoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLOUT},
+			{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
+		},
+	}, nil
+}
+
+func newTunFd(fd int) (*tunFile, error) {
+	if err := unix.SetNonblock(fd, true); err != nil {
+		return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
+	}
+
+	shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create eventfd: %w", err)
+	}
+
+	out := &tunFile{
+		fd:         fd,
+		shutdownFd: shutdownFd,
+		lastOne:    true,
+		readPoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLIN},
+			{Fd: int32(shutdownFd), Events: unix.POLLIN},
+		},
+		writePoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLOUT},
+			{Fd: int32(shutdownFd), Events: unix.POLLIN},
+		},
+	}
+
+	return out, nil
+}
+
+func (r *tunFile) blockOnRead() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(r.readPoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	//always reset these!
+	tunEvents := r.readPoll[0].Revents
+	shutdownEvents := r.readPoll[1].Revents
+	r.readPoll[0].Revents = 0
+	r.readPoll[1].Revents = 0
+	//do the err check before trusting the potentially bogus bits we just got
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	} else if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
+}
+
+func (r *tunFile) blockOnWrite() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(r.writePoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	//always reset these!
+	tunEvents := r.writePoll[0].Revents
+	shutdownEvents := r.writePoll[1].Revents
+	r.writePoll[0].Revents = 0
+	r.writePoll[1].Revents = 0
+	//do the err check before trusting the potentially bogus bits we just got
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	} else if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
+}
+
+func (r *tunFile) Read(buf []byte) (int, error) {
+	for {
+		if n, err := unix.Read(r.fd, buf); err == nil {
+			return n, nil
+		} else if err == unix.EAGAIN {
+			if err = r.blockOnRead(); err != nil {
+				return 0, err
+			}
+			continue
+		} else if err == unix.EINTR {
+			continue
+		} else if err == unix.EBADF {
+			return 0, os.ErrClosed
+		} else {
+			return 0, err
+		}
+	}
+}
+
+func (r *tunFile) Write(buf []byte) (int, error) {
+	for {
+		if n, err := unix.Write(r.fd, buf); err == nil {
+			return n, nil
+		} else if err == unix.EAGAIN {
+			if err = r.blockOnWrite(); err != nil {
+				return 0, err
+			}
+			continue
+		} else if err == unix.EINTR {
+			continue
+		} else if err == unix.EBADF {
+			return 0, os.ErrClosed
+		} else {
+			return 0, err
+		}
+	}
+}
+
+func (r *tunFile) wakeForShutdown() error {
+	var buf [8]byte
+	binary.NativeEndian.PutUint64(buf[:], 1)
+	_, err := unix.Write(int(r.readPoll[1].Fd), buf[:])
+	return err
+}
+
+func (r *tunFile) Close() error {
+	if r.closed { // avoid closing more than once. Technically a fd could get re-used, which would be a problem
+		return nil
+	}
+	r.closed = true
+	if r.lastOne {
+		_ = unix.Close(r.shutdownFd)
+	}
+	return unix.Close(r.fd)
+}
+
 type tun struct {
-	readers     tio.QueueSet
+	*tunFile
+	readers     []*tunFile
 	closeLock   sync.Mutex
 	Device      string
 	vpnNetworks []netip.Prefix
@@ -82,58 +249,45 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
 	return t, nil
 }
 
-// openTunDev opens /dev/net/tun, creating the device node first if it's
+func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) {
-// missing (docker containers occasionally omit it).
-func openTunDev() (int, error) {
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
-	if err == nil {
+	if err != nil {
-		return fd, nil
+		// If /dev/net/tun doesn't exist, try to create it (will happen in docker)
+		if os.IsNotExist(err) {
+			err = os.MkdirAll("/dev/net", 0755)
+			if err != nil {
+				return nil, fmt.Errorf("/dev/net/tun doesn't exist, failed to mkdir -p /dev/net: %w", err)
 			}
-	if !os.IsNotExist(err) {
+			err = unix.Mknod("/dev/net/tun", unix.S_IFCHR|0600, int(unix.Mkdev(10, 200)))
-		return -1, err
+			if err != nil {
-	}
+				return nil, fmt.Errorf("failed to create /dev/net/tun: %w", err)
-	if err = os.MkdirAll("/dev/net", 0755); err != nil {
-		return -1, fmt.Errorf("/dev/net/tun doesn't exist, failed to mkdir -p /dev/net: %w", err)
-	}
-	if err = unix.Mknod("/dev/net/tun", unix.S_IFCHR|0600, int(unix.Mkdev(10, 200))); err != nil {
-		return -1, fmt.Errorf("failed to create /dev/net/tun: %w", err)
 			}
+
 			fd, err = unix.Open("/dev/net/tun", os.O_RDWR, 0)
 			if err != nil {
-		return -1, fmt.Errorf("created /dev/net/tun, but still failed: %w", err)
+				return nil, fmt.Errorf("created /dev/net/tun, but still failed: %w", err)
 			}
-	return fd, nil
+		} else {
-}
-
-// tunSetIff runs TUNSETIFF with the given flags and returns the kernel-chosen
-// device name on success.
-func tunSetIff(fd int, name string, flags uint16) (string, error) {
-	var req ifReq
-	req.Flags = flags
-	copy(req.Name[:], name)
-	if err := ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
-		return "", err
-	}
-	return strings.Trim(string(req.Name[:]), "\x00"), nil
-}
-
-func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) {
-	baseFlags := uint16(unix.IFF_TUN | unix.IFF_NO_PI)
-	if multiqueue {
-		baseFlags |= unix.IFF_MULTI_QUEUE
-	}
-	nameStr := c.GetString("tun.dev", "")
-
-	fd, err := openTunDev()
-	if err != nil {
 			return nil, err
 		}
-	name, err := tunSetIff(fd, nameStr, baseFlags)
-	if err != nil {
-		_ = unix.Close(fd)
-		return nil, &NameError{Name: nameStr, Underlying: err}
 	}
 
+	var req ifReq
+	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI)
+	if multiqueue {
+		req.Flags |= unix.IFF_MULTI_QUEUE
+	}
+	nameStr := c.GetString("tun.dev", "")
+	copy(req.Name[:], nameStr)
+	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
+		_ = unix.Close(fd)
+		return nil, &NameError{
+			Name:       nameStr,
+			Underlying: err,
+		}
+	}
+	name := strings.Trim(string(req.Name[:]), "\x00")
+
 	t, err := newTunGeneric(c, l, fd, vpnNetworks)
 	if err != nil {
 		return nil, err
@@ -144,22 +298,16 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue
 	return t, nil
 }
 
-// newTunGeneric does all the stuff common to different tun initialization
+// newTunGeneric does all the stuff common to different tun initialization paths. It will close your files on error.
-// paths. It will close your files on error.
 func newTunGeneric(c *config.C, l *slog.Logger, fd int, vpnNetworks []netip.Prefix) (*tun, error) {
-	qs, err := tio.NewPollQueueSet()
+	tfd, err := newTunFd(fd)
 	if err != nil {
 		_ = unix.Close(fd)
 		return nil, err
 	}
-	err = qs.Add(fd)
-	if err != nil {
-		_ = unix.Close(fd)
-		return nil, err
-	}
-
 	t := &tun{
-		readers:                   qs,
+		tunFile:                   tfd,
+		readers:                   []*tunFile{tfd},
 		closeLock:                 sync.Mutex{},
 		vpnNetworks:               vpnNetworks,
 		TXQueueLen:                c.GetInt("tun.tx_queue", 500),
@@ -258,41 +406,36 @@ func (t *tun) reload(c *config.C, initial bool) error {
 	return nil
 }
 
-// Queues opens additional kernel multiqueue fds until the device has n
+func (t *tun) SupportsMultiqueue() bool {
-// queues, then returns them all. The first queue was opened by newTun.
+	return true
-func (t *tun) Queues(n int) ([]tio.Queue, error) {
-	for len(t.readers.Queues()) < n {
-		if err := t.addQueue(); err != nil {
-			return nil, err
-		}
-	}
-	return t.readers.Queues(), nil
 }
 
-// addQueue opens one more IFF_MULTI_QUEUE fd on the device and adds it to
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-// the queue set.
-func (t *tun) addQueue() error {
 	t.closeLock.Lock()
 	defer t.closeLock.Unlock()
 
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	flags := uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
+	var req ifReq
-	if _, err = tunSetIff(fd, t.Device, flags); err != nil {
+	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
+	copy(req.Name[:], t.Device)
+	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
 		_ = unix.Close(fd)
-		return err
+		return nil, err
 	}
 
-	err = t.readers.Add(fd)
+	out, err := t.tunFile.newFriend(fd)
 	if err != nil {
 		_ = unix.Close(fd)
-		return err
+		return nil, err
 	}
 
-	return nil
+	t.readers = append(t.readers, out)
+
+	return out, nil
 }
 
 func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
@@ -735,10 +878,32 @@ func (t *tun) Close() error {
 		t.routeChan = nil
 	}
 
+	// Signal all readers blocked in poll to wake up and exit
+	_ = t.tunFile.wakeForShutdown()
+
 	if t.ioctlFd > 0 {
 		_ = unix.Close(int(t.ioctlFd))
 		t.ioctlFd = 0
 	}
 
-	return t.readers.Close()
+	for i := range t.readers {
+		if i == 0 {
+			continue //we want to close the zeroth reader last
+		}
+		err := t.readers[i].Close()
+		if err != nil {
+			t.l.Error("error closing tun reader", "reader", i, "error", err)
+		} else {
+			t.l.Info("closed tun reader", "reader", i)
+		}
+	}
+
+	//this is t.readers[0] too
+	err := t.tunFile.Close()
+	if err != nil {
+		t.l.Error("error closing tun reader", "reader", 0, "error", err)
+	} else {
+		t.l.Info("closed tun reader", "reader", 0)
+	}
+	return err
 }

overlay/tun_netbsd.go

@@ -6,6 +6,7 @@ package overlay
 import (
 	"errors"
 	"fmt"
+	"io"
 	"log/slog"
 	"net/netip"
 	"os"
@@ -16,7 +17,6 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	netroute "golang.org/x/net/route"
@@ -390,8 +390,12 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) Queues(int) ([]tio.Queue, error) {
+func (t *tun) SupportsMultiqueue() bool {
-	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
+	return false
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for netbsd")
 }
 
 func (t *tun) addRoutes(logErrors bool) error {

overlay/tun_openbsd.go

@@ -6,6 +6,7 @@ package overlay
 import (
 	"errors"
 	"fmt"
+	"io"
 	"log/slog"
 	"net/netip"
 	"os"
@@ -16,7 +17,6 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	netroute "golang.org/x/net/route"
@@ -138,8 +138,8 @@ func tunWritev(fd int, iovecs []unix.Iovec) (n int, err error)
 //go:noescape
 func tunReadv(fd int, iovecs []unix.Iovec) (n int, err error)
 
-// Read pulls one IP packet off the tun device, scattering the 4 byte protocol header away from
+// Read pulls one IP packet off the tun device, scattering the 4 byte protocol header away from the
-// the packet so the payload lands directly in to.
+// packet so the payload lands directly in to.
 func (t *tun) Read(to []byte) (int, error) {
 	var head [4]byte
 
@@ -369,8 +369,12 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) Queues(int) ([]tio.Queue, error) {
+func (t *tun) SupportsMultiqueue() bool {
-	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
+	return false
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for openbsd")
 }
 
 func (t *tun) addRoutes(logErrors bool) error {

overlay/tun_tester.go

@@ -14,7 +14,6 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/udp"
 )
@@ -178,6 +177,10 @@ func (t *TestTun) Read(b []byte) (int, error) {
 	return n, nil
 }
 
-func (t *TestTun) Queues(int) ([]tio.Queue, error) {
+func (t *TestTun) SupportsMultiqueue() bool {
-	return []tio.Queue{tio.NewSingleQueue(t, udp.MTU)}, nil
+	return false
+}
+
+func (t *TestTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented")
 }

overlay/tun_windows.go

@@ -6,6 +6,7 @@ package overlay
 import (
 	"crypto"
 	"fmt"
+	"io"
 	"log/slog"
 	"net/netip"
 	"os"
@@ -17,7 +18,6 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	"github.com/slackhq/nebula/wintun"
@@ -47,10 +47,6 @@ type winTun struct {
 	tun *wintun.NativeTun
 }
 
-func (t *winTun) Read(b []byte) (int, error) {
-	return t.tun.Read(b, 0)
-}
-
 func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (Device, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in Windows")
 }
@@ -259,12 +255,20 @@ func (t *winTun) Name() string {
 	return t.Device
 }
 
+func (t *winTun) Read(b []byte) (int, error) {
+	return t.tun.Read(b, 0)
+}
+
 func (t *winTun) Write(b []byte) (int, error) {
 	return t.tun.Write(b, 0)
 }
 
-func (t *winTun) Queues(int) ([]tio.Queue, error) {
+func (t *winTun) SupportsMultiqueue() bool {
-	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
+	return false
+}
+
+func (t *winTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for windows")
 }
 
 func (t *winTun) Close() error {

overlay/user.go

@@ -6,7 +6,6 @@ import (
 	"net/netip"
 
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 )
 
@@ -47,16 +46,12 @@ func (d *UserDevice) RoutesFor(ip netip.Addr) routing.Gateways {
 	return routing.Gateways{routing.NewGateway(ip, 1)}
 }
 
-func (d *UserDevice) Queues(n int) ([]tio.Queue, error) {
+func (d *UserDevice) SupportsMultiqueue() bool {
-	out := make([]tio.Queue, n)
+	return true
-	for i := range out {
-		// All queues share the underlying pipes (the io.Pipe serializes
-		// concurrent callers) but each owns a private scratch buffer so
-		// concurrent Reads across queues never alias. NoClose: the pipes are
-		// owned by the UserDevice and torn down once by UserDevice.Close.
-		out[i] = tio.NewSingleQueueNoClose(d, defaultBatchBufSize)
 }
-	return out, nil
+
+func (d *UserDevice) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return d, nil
 }
 
 func (d *UserDevice) Pipe() (*io.PipeReader, *io.PipeWriter) {
@@ -66,11 +61,9 @@ func (d *UserDevice) Pipe() (*io.PipeReader, *io.PipeWriter) {
 func (d *UserDevice) Read(p []byte) (n int, err error) {
 	return d.outboundReader.Read(p)
 }
-
 func (d *UserDevice) Write(p []byte) (n int, err error) {
 	return d.inboundWriter.Write(p)
 }
-
 func (d *UserDevice) Close() error {
 	d.inboundWriter.Close()
 	d.outboundWriter.Close()

overlay/user_test.go

@@ -1,163 +0,0 @@
-package overlay
-
-import (
-	"fmt"
-	"net/netip"
-	"sync"
-	"testing"
-
-	"github.com/slackhq/nebula/overlay/tio"
-)
-
-// newTestUserDevice returns the concrete *UserDevice so tests can reach Pipe()
-// and the internal queue plumbing.
-func newTestUserDevice(t *testing.T) *UserDevice {
-	t.Helper()
-	dev, err := NewUserDevice([]netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")})
-	if err != nil {
-		t.Fatalf("NewUserDevice: %v", err)
-	}
-	ud, ok := dev.(*UserDevice)
-	if !ok {
-		t.Fatalf("NewUserDevice returned %T, want *UserDevice", dev)
-	}
-	return ud
-}
-
-// TestUserDeviceReadersDistinctBuffers ensures each Queue is actually different
-func TestUserDeviceReadersDistinctBuffers(t *testing.T) {
-	d := newTestUserDevice(t)
-
-	readers, err := d.Queues(2)
-	if err != nil {
-		t.Fatalf("Queues: %v", err)
-	}
-	if len(readers) != 2 {
-		t.Fatalf("Queues(2) returned %d queues, want 2", len(readers))
-	}
-
-	// Distinct queue objects.
-	if readers[0] == readers[1] {
-		t.Fatal("Queues(2) returned the same queue object twice")
-	}
-
-	// Drive one packet through each queue and confirm the borrowed bytes from
-	// the first read are NOT clobbered by the second read. With a shared
-	// buffer, reading pkt1 into q1 would corrupt q0's still-borrowed slice.
-	_, ow := d.Pipe()
-
-	pkt0 := []byte("packet-zero-aaaaaaaa")
-	pkt1 := []byte("packet-one-bbbbbbbbb")
-
-	// The pipe is unbuffered, so writes block until a reader consumes them.
-	// Serialize: write pkt0 (read on q0), then write pkt1 (read on q1).
-	go func() {
-		if _, err := ow.Write(pkt0); err != nil {
-			t.Errorf("write pkt0: %v", err)
-		}
-		if _, err := ow.Write(pkt1); err != nil {
-			t.Errorf("write pkt1: %v", err)
-		}
-	}()
-
-	got0, err := readers[0].Read()
-	if err != nil {
-		t.Fatalf("q0.Read: %v", err)
-	}
-	if len(got0) != 1 || string(got0[0].Bytes) != string(pkt0) {
-		t.Fatalf("q0 first read = %q, want %q", firstBytes(got0), pkt0)
-	}
-	// Hold onto q0's borrowed slice across q1's read.
-	borrowed := got0[0].Bytes
-
-	got1, err := readers[1].Read()
-	if err != nil {
-		t.Fatalf("q1.Read: %v", err)
-	}
-	if len(got1) != 1 || string(got1[0].Bytes) != string(pkt1) {
-		t.Fatalf("q1 read = %q, want %q", firstBytes(got1), pkt1)
-	}
-
-	// q0's borrowed bytes must still hold pkt0 - a shared buffer would now
-	// show pkt1's contents.
-	if string(borrowed) != string(pkt0) {
-		t.Fatalf("q0 borrowed bytes were clobbered by q1's read: got %q, want %q", borrowed, pkt0)
-	}
-}
-
-// TestUserDeviceReadersConcurrentRace exercises two queues reading distinct
-// packets concurrently. Run it under `go test -race`: with the old
-// shared-buffer implementation the concurrent Reads raced on readBuf/batchRet
-// and corrupted each other's returned slices.
-func TestUserDeviceReadersConcurrentRace(t *testing.T) {
-	d := newTestUserDevice(t)
-	readers, err := d.Queues(2)
-	if err != nil {
-		t.Fatalf("Queues: %v", err)
-	}
-	_, ow := d.Pipe()
-
-	const iterations = 200
-
-	errs := make(chan error, 3)
-
-	// Each reader parks in Read on the shared outboundReader; io.Pipe hands
-	// each write to whichever reader is currently waiting. We only care that
-	// concurrent Reads into distinct buffers are race-free, so any parked
-	// reader may serve any write.
-	var wg sync.WaitGroup
-	run := func(idx int) {
-		defer wg.Done()
-		for i := 0; i < iterations; i++ {
-			pkts, err := readers[idx].Read()
-			if err != nil {
-				errs <- err
-				return
-			}
-			if len(pkts) != 1 {
-				errs <- fmt.Errorf("reader %d: got %d packets, want 1", idx, len(pkts))
-				return
-			}
-			// Touch every byte of the borrowed slice while the other reader
-			// may be mid-Read; a shared buffer would race here.
-			total := 0
-			for _, c := range pkts[0].Bytes {
-				total += int(c)
-			}
-			_ = total
-		}
-	}
-
-	wg.Add(2)
-	go run(0)
-	go run(1)
-
-	// Feed 2*iterations packets. io.Pipe copies each write straight into the
-	// waiting reader's private buffer, so reusing buf between writes is safe.
-	go func() {
-		buf := make([]byte, 32)
-		for i := 0; i < 2*iterations; i++ {
-			for j := range buf {
-				buf[j] = byte(i + j)
-			}
-			if _, err := ow.Write(buf); err != nil {
-				errs <- err
-				return
-			}
-		}
-	}()
-
-	wg.Wait()
-	select {
-	case err := <-errs:
-		t.Fatalf("concurrent reader failed: %v", err)
-	default:
-	}
-}
-
-func firstBytes(p []tio.Packet) []byte {
-	if len(p) == 0 {
-		return nil
-	}
-	return p[0].Bytes
-}

udp/netchange.go

@@ -0,0 +1,61 @@
+package udp
+
+import (
+	"context"
+	"log/slog"
+
+	"github.com/slackhq/nebula/config"
+)
+
+// NetworkChangeMonitor rebinds the udp listener when the local network moves out from under it.
+//
+// Detection lives here in the udp package, next to the socket it concerns and the platform matrix that already knows
+// which sockets go stale. What to do about a change — updating the lighthouse, requerying tunnels — is not the udp
+// package's business, so Start takes the reaction as a plain function. Passing it at Start rather than holding it
+// keeps this package from referencing whatever owns the rebind.
+//
+// On platforms whose sockets do not go stale, watchNetworkChanges hands back a nil channel and Start returns.
+type NetworkChangeMonitor struct {
+	l       *slog.Logger
+	ctx     context.Context
+	enabled bool
+}
+
+// NewNetworkChangeMonitor builds a monitor for local network changes. The returned monitor is always usable: Start
+// is safe to call unconditionally, it no-ops when disabled or on a platform that does not need it.
+func NewNetworkChangeMonitor(ctx context.Context, l *slog.Logger, c *config.C) *NetworkChangeMonitor {
+	return &NetworkChangeMonitor{
+		l:       l,
+		ctx:     ctx,
+		enabled: c.GetBool("listen.rebind_on_network_change", true),
+	}
+}
+
+// Start watches for network changes until the context is cancelled, calling rebind once per settled change. It
+// blocks, so callers run it in a goroutine, and it no-ops when disabled, unsupported, or with nothing to rebind.
+func (m *NetworkChangeMonitor) Start(rebind func()) {
+	if !m.enabled || rebind == nil || m.ctx.Err() != nil {
+		return
+	}
+
+	changes, err := watchNetworkChanges(m.ctx, m.l)
+	if err != nil {
+		// Not fatal. Everything else still works, we just won't notice a network change on our own.
+		m.l.Error("Failed to watch for network changes, will not rebind the udp listener when the network moves",
+			"error", err,
+		)
+		return
+	}
+
+	if changes == nil {
+		// This platform's sockets don't go stale, so there is nothing to watch for.
+		return
+	}
+
+	m.l.Info("Watching for network changes to rebind the udp listener")
+
+	for range changes {
+		m.l.Info("Local network changed, rebinding the udp listener")
+		rebind()
+	}
+}

udp/netchange_darwin.go

@@ -0,0 +1,164 @@
+//go:build darwin && !ios && !e2e_testing
+// +build darwin,!ios,!e2e_testing
+
+package udp
+
+import (
+	"context"
+	"encoding/binary"
+	"errors"
+	"log/slog"
+	"os"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+const (
+	// netChangeSettleWindow is how long we keep swallowing routing messages after the first interesting one. A
+	// single network change is never a single message, it is a burst: the link drops, addresses go away, new ones
+	// arrive, routes get rewritten. Reporting part way through that just means reporting again.
+	netChangeSettleWindow = time.Second
+
+	// netChangeReadBuffer is sized well past any rt_msghdr plus its addresses. A short read would be discarded by
+	// the kernel, so being generous here is how we avoid missing a message.
+	netChangeReadBuffer = 4096
+)
+
+// watchNetworkChanges reports when the local network moves out from under us, so the listener can be rebound.
+//
+// Darwin scopes a udp socket to whatever interface it came up on. Move between networks and we keep sending out an
+// interface that no longer has a route, which surfaces as an instant "no route to host" with no packet ever leaving
+// the box. Rebind clears that, but only if something notices the change and calls it. iOS has always been told by
+// the host app off NWPathMonitor. This is the equivalent for everything else that runs on darwin.
+//
+// The returned channel is buffered and coalescing: a send is dropped if one is already pending, since both mean the
+// same thing to a reader. It is closed when ctx is cancelled or the routing socket fails, so a caller can simply
+// range over it. Platforms whose sockets do not need rebinding return a nil channel and no error.
+func watchNetworkChanges(ctx context.Context, l *slog.Logger) (<-chan struct{}, error) {
+	sock, err := openRouteSocket()
+	if err != nil {
+		return nil, err
+	}
+
+	changes := make(chan struct{}, 1)
+
+	go func() {
+		defer close(changes)
+		defer func() { _ = sock.Close() }()
+
+		// Closing the socket is what unblocks the read in watchRouteSocket, so this turns cancellation into a
+		// close. It is scoped to this call so it cannot outlive the watch it belongs to.
+		done := make(chan struct{})
+		defer close(done)
+		go func() {
+			select {
+			case <-ctx.Done():
+				_ = sock.Close()
+			case <-done:
+			}
+		}()
+
+		watchRouteSocket(l, sock, changes)
+	}()
+
+	return changes, nil
+}
+
+// watchRouteSocket blocks reading the routing socket, reporting once per settled burst of changes. It returns when
+// the socket is closed, which is how cancellation gets us out of here.
+func watchRouteSocket(l *slog.Logger, sock *os.File, changes chan<- struct{}) {
+	buf := make([]byte, netChangeReadBuffer)
+
+	for {
+		n, err := sock.Read(buf)
+		if err != nil {
+			logRouteSocketError(l, err)
+			return
+		}
+
+		if !isNetworkChange(buf[:n]) {
+			continue
+		}
+
+		// Swallow the rest of the burst. The deadline is absolute and not extended by what arrives, so this always
+		// ends after the settle window no matter how chatty the socket is. Changes that land after the window
+		// simply produce another report, which is the correct outcome anyway.
+		deadline := time.Now().Add(netChangeSettleWindow)
+		for {
+			if err = sock.SetReadDeadline(deadline); err != nil {
+				logRouteSocketError(l, err)
+				return
+			}
+
+			if _, err = sock.Read(buf); err != nil {
+				if os.IsTimeout(err) {
+					break
+				}
+				logRouteSocketError(l, err)
+				return
+			}
+		}
+
+		if err = sock.SetReadDeadline(time.Time{}); err != nil {
+			logRouteSocketError(l, err)
+			return
+		}
+
+		select {
+		case changes <- struct{}{}:
+		default:
+			// One already pending, and a second "the network moved" tells the reader nothing new.
+		}
+	}
+}
+
+// logRouteSocketError reports a routing socket failure unless it is just us shutting the socket down.
+func logRouteSocketError(l *slog.Logger, err error) {
+	if errors.Is(err, os.ErrClosed) {
+		return
+	}
+
+	l.Error("Error reading the routing socket, will no longer notice local network changes", "error", err)
+}
+
+// openRouteSocket returns the routing socket as a non blocking os.File. Going through os.File puts reads on the go
+// poller, which buys us both a working read deadline and a Close that unblocks a read in progress.
+func openRouteSocket() (*os.File, error) {
+	fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = unix.SetNonblock(fd, true); err != nil {
+		_ = unix.Close(fd)
+		return nil, err
+	}
+
+	return os.NewFile(uintptr(fd), "route"), nil
+}
+
+// isNetworkChange reports whether a routing message means our local addressing may have moved out from under us.
+//
+// We read the header instead of parsing the message because the type is the only part we need, and a full parse can
+// fail on shapes we don't care about, which would turn "a message I can't parse" into "a change I missed".
+// rt_msghdr, if_msghdr and ifa_msghdr all begin with the same three fields, so this is the same for every type.
+func isNetworkChange(msg []byte) bool {
+	if len(msg) < 4 {
+		return false
+	}
+
+	// u_short msglen, u_char version, u_char type
+	if int(binary.NativeEndian.Uint16(msg[0:2])) > len(msg) || msg[2] != unix.RTM_VERSION {
+		return false
+	}
+
+	switch msg[3] {
+	case unix.RTM_NEWADDR, unix.RTM_DELADDR, unix.RTM_IFINFO:
+		// An address arrived or left, or a link changed state. Anything else on this socket is either a route
+		// churning underneath us, which a rebind doesn't help with, or unrelated traffic.
+		return true
+	default:
+		return false
+	}
+}

udp/netchange_darwin_test.go

@@ -0,0 +1,244 @@
+//go:build darwin && !ios && !e2e_testing
+// +build darwin,!ios,!e2e_testing
+
+package udp
+
+import (
+	"context"
+	"encoding/binary"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
+	"golang.org/x/sys/unix"
+)
+
+// routeMsg builds the first four bytes of a routing message, which is all isNetworkChange reads.
+func routeMsg(msgType uint8, extra int) []byte {
+	msg := make([]byte, 4+extra)
+	binary.NativeEndian.PutUint16(msg[0:2], uint16(len(msg)))
+	msg[2] = unix.RTM_VERSION
+	msg[3] = msgType
+	return msg
+}
+
+func TestIsNetworkChange(t *testing.T) {
+	// The three that mean our addressing may have moved
+	assert.True(t, isNetworkChange(routeMsg(unix.RTM_NEWADDR, 0)))
+	assert.True(t, isNetworkChange(routeMsg(unix.RTM_DELADDR, 0)))
+	assert.True(t, isNetworkChange(routeMsg(unix.RTM_IFINFO, 0)))
+
+	// Route churn is not something a rebind helps with
+	assert.False(t, isNetworkChange(routeMsg(unix.RTM_ADD, 0)))
+	assert.False(t, isNetworkChange(routeMsg(unix.RTM_DELETE, 0)))
+	assert.False(t, isNetworkChange(routeMsg(unix.RTM_GET, 0)))
+
+	// Garbage must not be mistaken for a change
+	assert.False(t, isNetworkChange(nil), "empty")
+	assert.False(t, isNetworkChange([]byte{0, 0, 0}), "short header")
+
+	wrongVersion := routeMsg(unix.RTM_NEWADDR, 0)
+	wrongVersion[2] = unix.RTM_VERSION + 1
+	assert.False(t, isNetworkChange(wrongVersion), "wrong rtm_version")
+
+	lying := routeMsg(unix.RTM_NEWADDR, 0)
+	binary.NativeEndian.PutUint16(lying[0:2], 512)
+	assert.False(t, isNetworkChange(lying), "msglen longer than what we read")
+}
+
+// socketPair returns a connected pair of datagram sockets, the first wrapped the same way the routing socket is. It
+// stands in for the kernel so the watch loop can be driven with synthetic messages.
+func socketPair(t *testing.T) (*os.File, int) {
+	t.Helper()
+
+	fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_DGRAM, 0)
+	require.NoError(t, err)
+	require.NoError(t, unix.SetNonblock(fds[0], true))
+
+	f := os.NewFile(uintptr(fds[0]), "route")
+	t.Cleanup(func() {
+		_ = f.Close()
+		_ = unix.Close(fds[1])
+	})
+
+	return f, fds[1]
+}
+
+func TestWatchRouteSocketCoalescesABurst(t *testing.T) {
+	sock, kernel := socketPair(t)
+	changes := make(chan struct{}, 1)
+
+	done := make(chan struct{})
+	go func() {
+		watchRouteSocket(test.NewLogger(), sock, changes)
+		close(done)
+	}()
+
+	// One network change is a burst of messages. All of these land inside the settle window, so they must produce
+	// exactly one report rather than one apiece.
+	for range 5 {
+		_, err := unix.Write(kernel, routeMsg(unix.RTM_NEWADDR, 8))
+		require.NoError(t, err)
+	}
+	// Uninteresting messages in the middle of a burst must not add a report of their own either.
+	_, err := unix.Write(kernel, routeMsg(unix.RTM_ADD, 8))
+	require.NoError(t, err)
+
+	select {
+	case <-changes:
+	case <-time.After(netChangeSettleWindow * 4):
+		t.Fatal("a burst should have reported a change")
+	}
+
+	// Nothing more from that burst
+	select {
+	case <-changes:
+		t.Fatal("a burst should report exactly once")
+	case <-time.After(netChangeSettleWindow):
+	}
+
+	// A change after the window has closed is a separate event and gets its own report.
+	_, err = unix.Write(kernel, routeMsg(unix.RTM_IFINFO, 8))
+	require.NoError(t, err)
+	select {
+	case <-changes:
+	case <-time.After(netChangeSettleWindow * 4):
+		t.Fatal("a later change should report again")
+	}
+
+	// Closing the socket is how the real thing shuts down
+	require.NoError(t, sock.Close())
+	select {
+	case <-done:
+	case <-time.After(time.Second * 5):
+		t.Fatal("watchRouteSocket did not return after the socket was closed")
+	}
+}
+
+func TestWatchRouteSocketIgnoresUninterestingMessages(t *testing.T) {
+	sock, kernel := socketPair(t)
+	changes := make(chan struct{}, 1)
+
+	done := make(chan struct{})
+	go func() {
+		watchRouteSocket(test.NewLogger(), sock, changes)
+		close(done)
+	}()
+
+	for _, msgType := range []uint8{unix.RTM_ADD, unix.RTM_DELETE, unix.RTM_GET, unix.RTM_MISS} {
+		_, err := unix.Write(kernel, routeMsg(msgType, 8))
+		require.NoError(t, err)
+	}
+
+	select {
+	case <-changes:
+		t.Fatal("route churn alone must not report a change")
+	case <-time.After(netChangeSettleWindow * 2):
+	}
+
+	require.NoError(t, sock.Close())
+	select {
+	case <-done:
+	case <-time.After(time.Second * 5):
+		t.Fatal("watchRouteSocket did not return after the socket was closed")
+	}
+}
+
+// TestWatchRouteSocketDropsRatherThanBlocks covers the coalescing send. A reader that is busy rebinding must not
+// wedge the watcher, and a second pending "the network moved" tells it nothing new anyway.
+func TestWatchRouteSocketDropsRatherThanBlocks(t *testing.T) {
+	sock, kernel := socketPair(t)
+	changes := make(chan struct{}, 1)
+
+	done := make(chan struct{})
+	go func() {
+		watchRouteSocket(test.NewLogger(), sock, changes)
+		close(done)
+	}()
+
+	// Nobody is reading changes, so after the first report the buffer is full for the rest of this test
+	for range 3 {
+		_, err := unix.Write(kernel, routeMsg(unix.RTM_NEWADDR, 8))
+		require.NoError(t, err)
+		time.Sleep(netChangeSettleWindow + time.Millisecond*250)
+	}
+
+	// The watcher must still be alive and responsive to a close
+	require.NoError(t, sock.Close())
+	select {
+	case <-done:
+	case <-time.After(time.Second * 5):
+		t.Fatal("watchRouteSocket wedged on a full channel")
+	}
+
+	assert.Len(t, changes, 1, "the pending report should have coalesced, not queued")
+}
+
+// TestWatchNetworkChangesStopsWithContext covers the detection path against a real routing socket, including that
+// cancelling the context closes the channel so a ranging caller falls out of its loop.
+func TestWatchNetworkChangesStopsWithContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	changes, err := watchNetworkChanges(ctx, test.NewLogger())
+	require.NoError(t, err)
+	require.NotNil(t, changes, "darwin should support watching")
+
+	drained := make(chan struct{})
+	go func() {
+		for range changes {
+		}
+		close(drained)
+	}()
+
+	cancel()
+	select {
+	case <-drained:
+	case <-time.After(time.Second * 5):
+		t.Fatal("cancelling the context should close the changes channel")
+	}
+}
+
+// TestNetworkChangeMonitorStopsWithContext drives the whole monitor against a real routing socket: Start must block
+// watching, and cancelling the context (which is all Control does on shutdown, it never stops the monitor directly)
+// must return it and clean up the watch goroutines.
+func TestNetworkChangeMonitorStopsWithContext(t *testing.T) {
+	// IgnoreCurrent because other tests in this package leave readers running; we only care about what this test
+	// leaks itself.
+	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	l := test.NewLogger()
+	c := config.NewC(l)
+	require.NoError(t, c.LoadString("listen:\n  rebind_on_network_change: true\n"))
+	m := NewNetworkChangeMonitor(ctx, l, c)
+
+	done := make(chan struct{})
+	go func() {
+		m.Start(func() {})
+		close(done)
+	}()
+
+	// Start should be sitting on the routing socket, not have fallen out. If it returned early it either failed to
+	// watch or no-op'd, both of which we want to catch.
+	select {
+	case <-done:
+		t.Fatal("Start returned instead of watching")
+	case <-time.After(time.Millisecond * 250):
+	}
+
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(time.Second * 5):
+		t.Fatal("Start did not return after the context was cancelled")
+	}
+
+	// Starting again after the context is dead must not open anything.
+	m.Start(func() {})
+}

udp/netchange_generic.go

@@ -0,0 +1,22 @@
+//go:build !darwin || ios || e2e_testing
+// +build !darwin ios e2e_testing
+
+package udp
+
+import (
+	"context"
+	"log/slog"
+)
+
+// watchNetworkChanges is a no-op outside of darwin.
+//
+// Darwin is the platform that scopes a udp socket to the interface it came up on, so it is the platform whose socket
+// goes stale when the local network changes. Everywhere else Rebind has nothing to do, so there is nothing to watch
+// for. iOS is excluded on purpose even though it is darwin: the host app already drives the rebind off NWPathMonitor,
+// and two things racing to rebind the same socket is worse than one.
+//
+// A nil channel means "not supported here", which callers must treat as "do not start a watcher" rather than
+// selecting on it, since a receive from a nil channel blocks forever.
+func watchNetworkChanges(_ context.Context, _ *slog.Logger) (<-chan struct{}, error) {
+	return nil, nil
+}

udp/netchange_test.go

@@ -0,0 +1,39 @@
+package udp
+
+import (
+	"context"
+	"testing"
+
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func newMonitor(t *testing.T, ctx context.Context, cfg string) *NetworkChangeMonitor {
+	t.Helper()
+	l := test.NewLogger()
+	c := config.NewC(l)
+	require.NoError(t, c.LoadString(cfg))
+	return NewNetworkChangeMonitor(ctx, l, c)
+}
+
+func TestNetworkChangeMonitorDefaultsOn(t *testing.T) {
+	// Says nothing about rebinding, so this covers the default.
+	m := newMonitor(t, context.Background(), "listen:\n  host: 0.0.0.0\n")
+	assert.True(t, m.enabled, "should default to on")
+}
+
+func TestNetworkChangeMonitorDisabledIsANoOp(t *testing.T) {
+	m := newMonitor(t, context.Background(), "listen:\n  rebind_on_network_change: false\n")
+	require.False(t, m.enabled)
+
+	// Must return without opening a socket. If it watched anything this would block.
+	m.Start(func() {})
+}
+
+func TestNetworkChangeMonitorNilRebindIsANoOp(t *testing.T) {
+	// Nothing to rebind, so there is no point watching, on any platform.
+	m := newMonitor(t, context.Background(), "listen:\n  rebind_on_network_change: true\n")
+	m.Start(nil)
+}

udp/udp_darwin.go

@@ -187,6 +187,9 @@ func (u *StdConn) SupportsMultipleReaders() bool {
 	return false
 }
 
+// Rebind clears the interface the kernel scoped this socket to, so that sends are routed against the current
+// routing table instead of the interface we happened to be on when the socket was created. Darwin pins sockets
+// this way on its own, which is what strands us after the underlying network changes.
 func (u *StdConn) Rebind() error {
 	var err error
 	if u.isV4 {
@@ -195,9 +198,5 @@ func (u *StdConn) Rebind() error {
 		err = syscall.SetsockoptInt(int(u.sysFd), syscall.IPPROTO_IPV6, syscall.IPV6_BOUND_IF, 0)
 	}
 
-	if err != nil {
+	return err
-		u.l.Error("Failed to rebind udp socket", "error", err)
-	}
-
-	return nil
 }

udp/udp_linux.go

@@ -4,12 +4,13 @@
 package udp
 
 import (
-	"context"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"log/slog"
 	"net"
 	"net/netip"
+	"sync/atomic"
 	"syscall"
 	"unsafe"
 
@@ -19,58 +20,51 @@ import (
 )
 
 type StdConn struct {
-	udpConn *net.UDPConn
+	sysFd  int
-	rawConn syscall.RawConn
+	closed atomic.Bool
 	isV4   bool
 	l      *slog.Logger
 	batch  int
 }
 
-func setReusePort(network, address string, c syscall.RawConn) error {
-	var opErr error
-	err := c.Control(func(fd uintptr) {
-		opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
-		//CloseOnExec already set by the runtime
-	})
-	if err != nil {
-		return err
-	}
-	return opErr
-}
-
 func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
-	listen := netip.AddrPortFrom(ip, uint16(port))
+	af := unix.AF_INET6
-	lc := net.ListenConfig{}
+	if ip.Is4() {
+		af = unix.AF_INET
+	}
+	syscall.ForkLock.RLock()
+	fd, err := unix.Socket(af, unix.SOCK_DGRAM, unix.IPPROTO_UDP)
+	if err == nil {
+		unix.CloseOnExec(fd)
+	}
+	syscall.ForkLock.RUnlock()
+	if err != nil {
+		return nil, fmt.Errorf("unable to open socket: %w", err)
+	}
+
 	if multi {
-		lc.Control = setReusePort
+		if err = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
+			_ = unix.Close(fd)
+			return nil, fmt.Errorf("unable to set SO_REUSEPORT: %w", err)
 		}
-	//this context is only used during the bind operation, you can't cancel it to kill the socket
-	pc, err := lc.ListenPacket(context.Background(), "udp", listen.String())
-	if err != nil {
-		return nil, fmt.Errorf("unable to open socket: %s", err)
-	}
-	udpConn := pc.(*net.UDPConn)
-	rawConn, err := udpConn.SyscallConn()
-	if err != nil {
-		_ = udpConn.Close()
-		return nil, err
-	}
-	//gotta find out if we got an AF_INET6 socket or not:
-	out := &StdConn{
-		udpConn: udpConn,
-		rawConn: rawConn,
-		l:       l,
-		batch:   batch,
 	}
 
-	af, err := out.getSockOptInt(unix.SO_DOMAIN)
+	var sa unix.Sockaddr
-	if err != nil {
+	if ip.Is4() {
-		_ = out.Close()
+		sa4 := &unix.SockaddrInet4{Port: port}
-		return nil, err
+		sa4.Addr = ip.As4()
+		sa = sa4
+	} else {
+		sa6 := &unix.SockaddrInet6{Port: port}
+		sa6.Addr = ip.As16()
+		sa = sa6
+	}
+	if err = unix.Bind(fd, sa); err != nil {
+		_ = unix.Close(fd)
+		return nil, fmt.Errorf("unable to bind to socket: %w", err)
 	}
-	out.isV4 = af == unix.AF_INET
 
-	return out, nil
+	return &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch}, nil
 }
 
 func (u *StdConn) SupportsMultipleReaders() bool {
@@ -81,133 +75,110 @@ func (u *StdConn) Rebind() error {
 	return nil
 }
 
-func (u *StdConn) getSockOptInt(opt int) (int, error) {
-	if u.rawConn == nil {
-		return 0, fmt.Errorf("no UDP connection")
-	}
-	var out int
-	var opErr error
-	err := u.rawConn.Control(func(fd uintptr) {
-		out, opErr = unix.GetsockoptInt(int(fd), unix.SOL_SOCKET, opt)
-	})
-	if err != nil {
-		return 0, err
-	}
-	return out, opErr
-}
-
-func (u *StdConn) setSockOptInt(opt int, n int) error {
-	if u.rawConn == nil {
-		return fmt.Errorf("no UDP connection")
-	}
-	var opErr error
-	err := u.rawConn.Control(func(fd uintptr) {
-		opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, opt, n)
-	})
-	if err != nil {
-		return err
-	}
-	return opErr
-}
-
 func (u *StdConn) SetRecvBuffer(n int) error {
-	return u.setSockOptInt(unix.SO_RCVBUFFORCE, n)
+	return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUFFORCE, n)
 }
 
 func (u *StdConn) SetSendBuffer(n int) error {
-	return u.setSockOptInt(unix.SO_SNDBUFFORCE, n)
+	return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUFFORCE, n)
 }
 
 func (u *StdConn) SetSoMark(mark int) error {
-	return u.setSockOptInt(unix.SO_MARK, mark)
+	return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK, mark)
 }
 
 func (u *StdConn) GetRecvBuffer() (int, error) {
-	return u.getSockOptInt(unix.SO_RCVBUF)
+	return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUF)
 }
 
 func (u *StdConn) GetSendBuffer() (int, error) {
-	return u.getSockOptInt(unix.SO_SNDBUF)
+	return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUF)
 }
 
 func (u *StdConn) GetSoMark() (int, error) {
-	return u.getSockOptInt(unix.SO_MARK)
+	return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK)
 }
 
 func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
-	a := u.udpConn.LocalAddr()
+	sa, err := unix.Getsockname(u.sysFd)
-
+	if err != nil {
-	switch v := a.(type) {
+		return netip.AddrPort{}, err
-	case *net.UDPAddr:
-		addr, ok := netip.AddrFromSlice(v.IP)
-		if !ok {
-			return netip.AddrPort{}, fmt.Errorf("LocalAddr returned invalid IP address: %s", v.IP)
 	}
-		return netip.AddrPortFrom(addr, uint16(v.Port)), nil
+	switch sa := sa.(type) {
-
+	case *unix.SockaddrInet4:
+		return netip.AddrPortFrom(netip.AddrFrom4(sa.Addr), uint16(sa.Port)), nil
+	case *unix.SockaddrInet6:
+		return netip.AddrPortFrom(netip.AddrFrom16(sa.Addr), uint16(sa.Port)), nil
 	default:
-		return netip.AddrPort{}, fmt.Errorf("LocalAddr returned: %#v", a)
+		return netip.AddrPort{}, fmt.Errorf("unsupported sock type: %T", sa)
 	}
 }
 
-func recvmmsg(fd uintptr, msgs []rawMessage) (int, bool, error) {
+// recvmmsg does one blocking recvmmsg (MSG_WAITFORONE), reading up to len(msgs) datagrams
-	var errno syscall.Errno
+func (u *StdConn) recvmmsg(msgs []rawMessage) (int, error) {
-	n, _, errno := unix.Syscall6(
+	r, _, errno := unix.Syscall6(
 		unix.SYS_RECVMMSG,
-		fd,
+		uintptr(u.sysFd),
 		uintptr(unsafe.Pointer(&msgs[0])),
 		uintptr(len(msgs)),
 		unix.MSG_WAITFORONE,
 		0,
 		0,
 	)
-	if errno == syscall.EAGAIN || errno == syscall.EWOULDBLOCK {
-		// No data available, block for I/O and try again.
-		return int(n), false, nil
-	}
 	if errno != 0 {
-		return int(n), true, &net.OpError{Op: "recvmmsg", Err: errno}
+		if u.closed.Load() {
+			return 0, net.ErrClosed
 		}
-	return int(n), true, nil
+		return 0, &net.OpError{Op: "recvmmsg", Err: errno}
+	}
+	n := int(r)
+	if (n == 0 || msgs[0].Len == 0) && u.closed.Load() {
+		return 0, net.ErrClosed
+	}
+	return n, nil
 }
 
-func (u *StdConn) listenOutSingle(r EncReader) error {
+// recvmsg does one blocking recvmsg into msgs[0]
-	var err error
+func (u *StdConn) recvmsg(msgs []rawMessage) (int, error) {
-	var n int
+	r, _, errno := unix.Syscall6(
-	var from netip.AddrPort
+		unix.SYS_RECVMSG,
-	buffer := make([]byte, MTU)
+		uintptr(u.sysFd),
-
+		uintptr(unsafe.Pointer(&msgs[0].Hdr)),
-	for {
+		0,
-		n, from, err = u.udpConn.ReadFromUDPAddrPort(buffer)
+		0,
-		if err != nil {
+		0,
-			return err
+		0,
+	)
+	if errno != 0 {
+		if u.closed.Load() {
+			return 0, net.ErrClosed
 		}
-		from = netip.AddrPortFrom(from.Addr().Unmap(), from.Port())
+		return 0, &net.OpError{Op: "recvmsg", Err: errno}
-		r(from, buffer[:n])
 	}
+	if r == 0 && u.closed.Load() {
+		return 0, net.ErrClosed
+	}
+	msgs[0].Len = uint32(r)
+	return 1, nil
 }
 
-func (u *StdConn) listenOutBatch(r EncReader) error {
+func (u *StdConn) ListenOut(r EncReader) error {
 	var ip netip.Addr
-	var n int
-	var operr error
-
 	msgs, buffers, names := u.PrepareRawMessages(u.batch)
-
+	read := u.recvmmsg
-	//reader needs to capture variables from this function, since it's used as a lambda with rawConn.Read
+	if u.batch == 1 {
-	//defining it outside the loop so it gets re-used
+		read = u.recvmsg
-	reader := func(fd uintptr) (done bool) {
-		n, done, operr = recvmmsg(fd, msgs)
-		return done
 	}
 
 	for {
-		err := u.rawConn.Read(reader)
+		n, err := read(msgs)
 		if err != nil {
-			return err
+			if errors.Is(err, unix.EINTR) {
+				continue // interrupted by a signal, retry the read
 			}
-		if operr != nil {
+			// net.ErrClosed after Close() is teardown, absorbed by the caller's
-			return operr
+			// closed flag like the other platforms; anything else is a real error.
+			return err
 		}
 
 		for i := 0; i < n; i++ {
@@ -222,26 +193,68 @@ func (u *StdConn) listenOutBatch(r EncReader) error {
 	}
 }
 
-func (u *StdConn) ListenOut(r EncReader) error {
+func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
-	if u.batch == 1 {
+	if u.isV4 {
-		return u.listenOutSingle(r)
+		return u.writeTo4(b, ip)
-	} else {
+	}
-		return u.listenOutBatch(r)
+	return u.writeTo6(b, ip)
+}
+
+func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
+	var rsa unix.RawSockaddrInet6
+	rsa.Family = unix.AF_INET6
+	rsa.Addr = ip.Addr().As16()
+	binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
+
+	for {
+		_, _, err := unix.Syscall6(
+			unix.SYS_SENDTO,
+			uintptr(u.sysFd),
+			uintptr(unsafe.Pointer(&b[0])),
+			uintptr(len(b)),
+			uintptr(0),
+			uintptr(unsafe.Pointer(&rsa)),
+			uintptr(unix.SizeofSockaddrInet6),
+		)
+		if err != 0 {
+			return &net.OpError{Op: "sendto", Err: err}
+		}
+		return nil
 	}
 }
 
-func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
+func (u *StdConn) writeTo4(b []byte, ip netip.AddrPort) error {
-	_, err := u.udpConn.WriteToUDPAddrPort(b, ip)
+	if !ip.Addr().Is4() {
-	return err
+		return ErrInvalidIPv6RemoteForSocket
+	}
+
+	var rsa unix.RawSockaddrInet4
+	rsa.Family = unix.AF_INET
+	rsa.Addr = ip.Addr().As4()
+	binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
+
+	for {
+		_, _, err := unix.Syscall6(
+			unix.SYS_SENDTO,
+			uintptr(u.sysFd),
+			uintptr(unsafe.Pointer(&b[0])),
+			uintptr(len(b)),
+			uintptr(0),
+			uintptr(unsafe.Pointer(&rsa)),
+			uintptr(unix.SizeofSockaddrInet4),
+		)
+		if err != 0 {
+			return &net.OpError{Op: "sendto", Err: err}
+		}
+		return nil
+	}
 }
 
 func (u *StdConn) ReloadConfig(c *config.C) {
 	b := c.GetInt("listen.read_buffer", 0)
 	if b > 0 {
-		err := u.SetRecvBuffer(b)
+		if err := u.SetRecvBuffer(b); err == nil {
-		if err == nil {
+			if s, err := u.GetRecvBuffer(); err == nil {
-			s, err := u.GetRecvBuffer()
-			if err == nil {
 				u.l.Info("listen.read_buffer was set", "size", s)
 			} else {
 				u.l.Warn("Failed to get listen.read_buffer", "error", err)
@@ -253,10 +266,8 @@ func (u *StdConn) ReloadConfig(c *config.C) {
 
 	b = c.GetInt("listen.write_buffer", 0)
 	if b > 0 {
-		err := u.SetSendBuffer(b)
+		if err := u.SetSendBuffer(b); err == nil {
-		if err == nil {
+			if s, err := u.GetSendBuffer(); err == nil {
-			s, err := u.GetSendBuffer()
-			if err == nil {
 				u.l.Info("listen.write_buffer was set", "size", s)
 			} else {
 				u.l.Warn("Failed to get listen.write_buffer", "error", err)
@@ -269,10 +280,8 @@ func (u *StdConn) ReloadConfig(c *config.C) {
 	b = c.GetInt("listen.so_mark", 0)
 	s, err := u.GetSoMark()
 	if b > 0 || (err == nil && s != 0) {
-		err := u.SetSoMark(b)
+		if err := u.SetSoMark(b); err == nil {
-		if err == nil {
+			if s, err := u.GetSoMark(); err == nil {
-			s, err := u.GetSoMark()
-			if err == nil {
 				u.l.Info("listen.so_mark was set", "mark", s)
 			} else {
 				u.l.Warn("Failed to get listen.so_mark", "error", err)
@@ -285,28 +294,20 @@ func (u *StdConn) ReloadConfig(c *config.C) {
 
 func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
 	var vallen uint32 = 4 * unix.SK_MEMINFO_VARS
-
+	_, _, err := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(u.sysFd), uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
-	if u.rawConn == nil {
+	if err != 0 {
-		return fmt.Errorf("no UDP connection")
-	}
-	var opErr error
-	err := u.rawConn.Control(func(fd uintptr) {
-		_, _, syserr := unix.Syscall6(unix.SYS_GETSOCKOPT, fd, uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
-		if syserr != 0 {
-			opErr = syserr
-		}
-	})
-	if err != nil {
 		return err
 	}
-	return opErr
+	return nil
 }
 
 func (u *StdConn) Close() error {
-	if u.udpConn != nil {
+	u.closed.Store(true)
-		return u.udpConn.Close()
+	// Wake the reader parked in recvmmsg/recvmsg. shutdown(2) on an unconnected socket
-	}
+	// returns ENOTCONN but still wakes it, so ignore the error.
-	return nil
+	// The reader then sees closed and stops touching the fd, making the Close below safe.
+	_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
+	return unix.Close(u.sysFd)
 }
 
 func NewUDPStatsEmitter(udpConns []Conn) func() {

udp/udp_linux_test.go

@@ -0,0 +1,179 @@
+//go:build linux && !android && !e2e_testing
+
+package udp
+
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+	"net"
+	"net/netip"
+	"os"
+	"runtime"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+func testLogger() *slog.Logger {
+	return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))
+}
+
+// TestShutdownWakesAfterRx_Mechanism exercises the kernel quirk our teardown
+// relies on: once a socket has received a packet, shutdown(2) wakes a blocked
+// recvmmsg with n>=1/Len==0 (not n==0). recvmmsg must turn that into net.ErrClosed
+// once Close set closed, so a parked reader exits instead of spinning.
+func TestShutdownWakesAfterRx_Mechanism(t *testing.T) {
+	c, err := NewListener(testLogger(), netip.MustParseAddr("127.0.0.1"), 0, true, 64)
+	if err != nil {
+		t.Fatalf("NewListener: %v", err)
+	}
+	sc := c.(*StdConn)
+	addr, err := sc.LocalAddr()
+	if err != nil {
+		t.Fatalf("LocalAddr: %v", err)
+	}
+	msgs, _, _ := sc.PrepareRawMessages(sc.batch)
+
+	// Receive a real packet so the socket has carried data.
+	send, err := net.Dial("udp", addr.String())
+	if err != nil {
+		t.Fatalf("dial: %v", err)
+	}
+	if _, err := send.Write([]byte("hello")); err != nil {
+		t.Fatalf("write: %v", err)
+	}
+	time.Sleep(50 * time.Millisecond)
+	n, err := sc.recvmmsg(msgs)
+	t.Logf("drain of real packet: n=%d err=%v msgs[0].Len=%d", n, err, msgs[0].Len)
+	_ = send.Close()
+
+	// Block a reader on the now-empty queue, then tear down as Close() does.
+	// recvmmsg must return net.ErrClosed (not hang, not spin) even post-rx.
+	done := make(chan error, 1)
+	go func() {
+		_, err := sc.recvmmsg(msgs)
+		done <- err
+	}()
+	time.Sleep(150 * time.Millisecond) // let it park in recvmmsg
+
+	sc.closed.Store(true)
+	if serr := unix.Shutdown(sc.sysFd, unix.SHUT_RDWR); serr != nil {
+		t.Logf("shutdown returned %v (expected ENOTCONN on unconnected UDP)", serr)
+	}
+
+	select {
+	case err := <-done:
+		if !errors.Is(err, net.ErrClosed) {
+			t.Errorf("recvmmsg after post-rx shutdown returned %v, want net.ErrClosed", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatalf("HANG: recvmmsg did not return after shutdown following a received packet")
+	}
+	_ = unix.Close(sc.sysFd)
+}
+
+// TestListenOutTeardown_TrafficPatterns reproduces the field report: a blocking
+// reader must tear down cleanly on Close() regardless of what the socket has
+// carried. The three cases the report called out:
+//
+//	no traffic ever      -> works (shutdown wakes recvmmsg with n==0)
+//	ping once, then idle -> historically HUNG: once the socket has received a
+//	                        packet, shutdown(2) wakes recvmmsg with n>=1/Len==0,
+//	                        which an n==0-only teardown check misses
+//	continuous traffic   -> works (a real packet is always arriving)
+//
+// All three must return within the deadline; a hang dumps goroutines so the
+// stuck reader is visible.
+func TestListenOutTeardown_TrafficPatterns(t *testing.T) {
+	cases := []struct {
+		name    string
+		traffic func(send net.Conn, stop <-chan struct{})
+	}{
+		{"no_traffic_ever", func(net.Conn, <-chan struct{}) {}},
+		{"ping_once_then_idle", func(send net.Conn, _ <-chan struct{}) {
+			_, _ = send.Write([]byte("hello"))
+		}},
+		{"continuous", func(send net.Conn, stop <-chan struct{}) {
+			for {
+				select {
+				case <-stop:
+					return
+				default:
+					_, _ = send.Write([]byte("hello"))
+					time.Sleep(2 * time.Millisecond)
+				}
+			}
+		}},
+	}
+
+	// batch 1 exercises the recvmsg path, batch 64 the recvmmsg path; both must
+	// tear down cleanly.
+	for _, batch := range []int{1, 64} {
+		for _, tc := range cases {
+			t.Run(fmt.Sprintf("batch%d/%s", batch, tc.name), func(t *testing.T) {
+				runTeardownCase(t, batch, tc.name, tc.traffic)
+			})
+		}
+	}
+}
+
+func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net.Conn, stop <-chan struct{})) {
+	c, err := NewListener(testLogger(), netip.MustParseAddr("127.0.0.1"), 0, true, batch)
+	if err != nil {
+		t.Fatalf("NewListener: %v", err)
+	}
+	sc := c.(*StdConn)
+	addr, err := sc.LocalAddr()
+	if err != nil {
+		t.Fatalf("LocalAddr: %v", err)
+	}
+
+	var received atomic.Int64
+	loopDone := make(chan error, 1)
+	go func() {
+		loopDone <- sc.ListenOut(func(netip.AddrPort, []byte) {
+			received.Add(1)
+		})
+	}()
+
+	send, err := net.Dial("udp", addr.String())
+	if err != nil {
+		t.Fatalf("dial: %v", err)
+	}
+	defer send.Close()
+
+	stop := make(chan struct{})
+	trafficDone := make(chan struct{})
+	go func() {
+		traffic(send, stop)
+		close(trafficDone)
+	}()
+
+	// Let the pattern run and, for the idle case, the reader park again on an
+	// empty queue with the socket already having received a packet.
+	time.Sleep(500 * time.Millisecond)
+
+	start := time.Now()
+	if err := sc.Close(); err != nil {
+		t.Fatalf("Close: %v", err)
+	}
+	close(stop)
+
+	select {
+	case err := <-loopDone:
+		// Clean teardown surfaces as net.ErrClosed (propagated like the other
+		// platforms); the caller absorbs it via its closed flag.
+		if err != nil && !errors.Is(err, net.ErrClosed) {
+			t.Fatalf("%s: ListenOut returned unexpected error on teardown: %v", name, err)
+		}
+		t.Logf("%s: closed in %v (received %d packets)", name, time.Since(start), received.Load())
+	case <-time.After(3 * time.Second):
+		buf := make([]byte, 1<<20)
+		n := runtime.Stack(buf, true)
+		t.Fatalf("%s: HANG, ListenOut did not return within 3s of Close\n%s", name, buf[:n])
+	}
+	<-trafficDone
+}

udp/udp_tester.go

@@ -10,6 +10,7 @@ import (
 	"net/netip"
 	"os"
 	"sync"
+	"sync/atomic"
 
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
@@ -64,7 +65,9 @@ func acquirePacket() *Packet {
 }
 
 type TesterConn struct {
-	Addr netip.AddrPort
+	// addr is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
+	// move it mid-run to simulate roaming, so it is atomic rather than a plain field.
+	addr atomic.Pointer[netip.AddrPort]
 
 	RxPackets chan *Packet // Packets to receive into nebula
 	TxPackets chan *Packet // Packets transmitted outside by nebula
@@ -82,13 +85,24 @@ type TesterConn struct {
 }
 
 func NewListener(l *slog.Logger, ip netip.Addr, port int, _ bool, _ int) (Conn, error) {
-	return &TesterConn{
+	c := &TesterConn{
-		Addr:      netip.AddrPortFrom(ip, uint16(port)),
 		RxPackets: make(chan *Packet, 10),
 		TxPackets: make(chan *Packet, 10),
 		done:      make(chan struct{}),
 		l:         l,
-	}, nil
+	}
+	c.SetAddr(netip.AddrPortFrom(ip, uint16(port)))
+	return c, nil
+}
+
+// GetAddr returns the underlay address this conn currently sends from.
+func (u *TesterConn) GetAddr() netip.AddrPort {
+	return *u.addr.Load()
+}
+
+// SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network.
+func (u *TesterConn) SetAddr(addr netip.AddrPort) {
+	u.addr.Store(&addr)
 }
 
 // Send will place a UdpPacket onto the receive queue for nebula to consume
@@ -147,7 +161,7 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
 		p.Data = p.Data[:len(b)]
 	}
 	copy(p.Data, b)
-	p.From = u.Addr
+	p.From = u.GetAddr()
 	p.To = addr
 	select {
 	case <-u.done:
@@ -178,7 +192,7 @@ func NewUDPStatsEmitter(_ []Conn) func() {
 }
 
 func (u *TesterConn) LocalAddr() (netip.AddrPort, error) {
-	return u.Addr, nil
+	return u.GetAddr(), nil
 }
 
 func (u *TesterConn) SupportsMultipleReaders() bool {

util/cpupin_linux.go

@@ -1,43 +0,0 @@
-//go:build linux && !android && !e2e_testing
-
-package util
-
-import (
-	"runtime"
-
-	"golang.org/x/sys/unix"
-)
-
-// PinThreadToCPU restricts the calling OS thread to the given CPU via
-// sched_setaffinity(2). Combined with runtime.LockOSThread on the
-// goroutine, this prevents the kernel from migrating us across CPUs and
-// in turn keeps every UDP send from this goroutine going through the
-// same XPS-selected TX ring, eliminating the wire-side reorder that
-// otherwise fragments one nebula flow across multiple rings.
-func PinThreadToCPU(cpu int) error {
-	runtime.LockOSThread()
-	var set unix.CPUSet
-	set.Zero()
-	set.Set(cpu)
-	return unix.SchedSetaffinity(0, &set)
-}
-
-// AllowedCPUs returns the CPU IDs the calling process is currently allowed to
-// run on, as reported by sched_getaffinity(2). Under a cgroup cpuset or a
-// `taskset` mask the allowed IDs are frequently not the contiguous range
-// 0..NumCPU-1 (e.g. pinned to CPUs 4-7: NumCPU reports 4 while the valid IDs
-// are 4,5,6,7). Callers that need a real CPU to pin to must choose from this
-// set rather than assuming i % NumCPU is runnable, or every pin fails.
-func AllowedCPUs() ([]int, error) {
-	var set unix.CPUSet
-	if err := unix.SchedGetaffinity(0, &set); err != nil {
-		return nil, err
-	}
-	cpus := make([]int, 0, set.Count())
-	for cpu := 0; cpu < len(set)*64; cpu++ {
-		if set.IsSet(cpu) {
-			cpus = append(cpus, cpu)
-		}
-	}
-	return cpus, nil
-}

util/cpupin_other.go

@@ -1,18 +0,0 @@
-//go:build !linux || android || e2e_testing
-
-package util
-
-// PinThreadToCPU is a no-op outside Linux: only Linux exposes a stable
-// per-thread CPU affinity API and only Linux has XPS-driven TX ring
-// selection in the first place. On every other platform there's nothing
-// to fix here.
-func PinThreadToCPU(_ int) error {
-	return nil
-}
-
-// AllowedCPUs has no meaningful answer off Linux (no sched_getaffinity), so it
-// reports "unknown" by returning a nil slice and nil error. Callers treat an
-// empty result as "fall back to the default CPU choice".
-func AllowedCPUs() ([]int, error) {
-	return nil, nil
-}