warn on uselessly low MTU

.github/actions/code-sign/action.yml

@@ -25,9 +25,9 @@ inputs:
     required: false
     default: "code-signer"
   key-prefix:
-    description: "S3 key prefix to write under; defaults to code-signing/<owner>/<repo> of the calling repo"
+    description: "S3 key prefix the caller is authorized to write under"
     required: false
-    default: ""
+    default: "code-signing/slackhq/nebula"
 
 runs:
   using: composite
@@ -57,9 +57,6 @@ runs:
         KEY_PREFIX: ${{ inputs.key-prefix }}
       run: |
         set -eu
-        # Default the prefix to this repo so the S3 key attributes the sign correctly.
-        # nebula-nightly runs this same action but writes under its own repo's prefix.
-        KEY_PREFIX="${KEY_PREFIX:-code-signing/$GITHUB_REPOSITORY}"
         RUN="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
 
         find "$SIGN_PATH" -name '*.exe' -print | while read -r path

.github/workflows/release.yml

@@ -12,9 +12,9 @@ jobs:
     steps:
       - uses: actions/checkout@v7
 
-      - uses: actions/setup-go@v7
+      - uses: actions/setup-go@v6
         with:
-          go-version: '1.26'
+          go-version: '1.25'
           check-latest: true
 
       - name: Build
@@ -38,9 +38,9 @@ jobs:
     steps:
       - uses: actions/checkout@v7
 
-      - uses: actions/setup-go@v7
+      - uses: actions/setup-go@v6
         with:
-          go-version: '1.26'
+          go-version: '1.25'
           check-latest: true
 
       - name: Build
@@ -78,9 +78,9 @@ jobs:
     steps:
       - uses: actions/checkout@v7
 
-      - uses: actions/setup-go@v7
+      - uses: actions/setup-go@v6
         with:
-          go-version: '1.26'
+          go-version: '1.25'
           check-latest: true
 
       - name: Import certificates

.github/workflows/smoke-extra.yml

@@ -32,9 +32,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v7
+    - uses: actions/setup-go@v6
       with:
-        go-version: '1.26'
+        go-version: '1.25'
         check-latest: true
 
     - name: add hashicorp source
@@ -64,9 +64,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v7
+    - uses: actions/setup-go@v6
       with:
-        go-version: '1.26'
+        go-version: '1.25'
         check-latest: true
 
     - name: add hashicorp source
@@ -90,9 +90,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v7
+    - uses: actions/setup-go@v6
       with:
-        go-version: '1.26'
+        go-version: '1.25'
         check-latest: true
 
     # WSL2 + Ubuntu so the smoke can run a real linux peer with its own

.github/workflows/smoke.yml

@@ -20,9 +20,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v7
+    - uses: actions/setup-go@v6
       with:
-        go-version: '1.26'
+        go-version: '1.25'
         check-latest: true
 
     - name: build

.github/workflows/test.yml

@@ -20,9 +20,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v7
+    - uses: actions/setup-go@v6
       with:
-        go-version: '1.26'
+        go-version: '1.25'
         check-latest: true
 
     - name: Install goimports
@@ -42,7 +42,7 @@ jobs:
     - name: golangci-lint
       uses: golangci/golangci-lint-action@v9
       with:
-        version: v2.12
+        version: v2.5
 
   test:
     name: Test ${{ matrix.name }}
@@ -80,9 +80,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v7
+    - uses: actions/setup-go@v6
       with:
-        go-version: '1.26'
+        go-version: '1.25'
         check-latest: true
 
     - name: Build
@@ -125,9 +125,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v7
+    - uses: actions/setup-go@v6
       with:
-        go-version: '1.26'
+        go-version: '1.25'
         check-latest: true
 
     - name: Build ${{ matrix.name }}

CHANGELOG.md

@@ -7,88 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-## [1.11.0] - 2026-07-23
-
-See the [v1.11.0](https://github.com/slackhq/nebula/milestone/25?closed=1) milestone for a complete list of changes.
-
-### Breaking
-
-- Logging has switched from logrus to Go's structured `slog`. Log output changes: levels are upper case
-  (`level=INFO`), trace prints as `level=DEBUG-4`, timestamps are always RFC3339Nano and `logging.timestamp_format`
-  is ignored, and some messages were reworded. Review any log parsing before upgrading. This is also an API break
-  for embedders, as constructors now take a `*slog.Logger`. (#1672, #1734, #1621)
-- `firewall.inbound_action` and `firewall.outbound_action` (used to set reject vs. drop policy) were each being
-  applied to the opposite direction, that is now corrected. This only affects how blocked packets are answered, not
-  which packets the firewall allows or denies. If you set either of these you are getting the behavior of the other
-  one today and likely want to swap them before upgrading. (#1798)
-- On Windows, Nebula now installs WFP PERMIT filters for the nebula adapter and the listener port by default. WFP
-  sits below Windows Defender Firewall, so any WDF inbound rules you rely on for either will no longer apply. Set
-  `tun.windows_bypass_wdf` and `listen.windows_bypass_wdf` to false to leave WDF in charge. (#1710)
-- On Windows, the nebula device is now set to the `private` network category instead of whatever Windows decided,
-  which is usually `Public`. This makes the host firewall less restrictive on the overlay. Set
-  `tun.network_category` to `unset` to keep the old behavior. (#1710)
-- Reject packets for non-TCP now use ICMP code 13, communication administratively prohibited, instead of code 3,
-  port unreachable. Anything keying off the old code needs updating. (#1766, #1768)
-- The SSH debug server's profiling commands are now confined to `sshd.sandbox_dir`, which defaults to
-  `$TMP/nebula-debug`. Relative paths resolve inside it and absolute paths outside it are rejected, so anything
-  scripting `start-cpu-profile`, `save-heap-profile`, or `save-mutex-profile` with a path elsewhere needs the
-  directory set. The directory is not created for you. (#1622)
-
-### Added
-
-- Sign the Windows release binaries. (#1718)
-- Generate IPv6 reject packets, matching the existing IPv4 behavior. (#1766, #1767, #1768)
-- Accept `-` in `nebula-cert` to read from stdin or write to stdout. (#1714)
-- Search for both `config.yml` and `config.yaml` in service and command line modes. (#1717)
-- Add version labels to the Docker/OCI images. (#1772)
-- Rebind the listener and re-query lighthouses on macOS when the underlay network changes, so devices moving
-  between wifi and wired or between networks recover without waiting for dead tunnel detection. Controlled by
-  `listen.rebind_on_network_change` (default `true`, not reloadable). (#1816)
-
-### Changed
-
-- Reload the firewall when the unsafe networks in the certificate change. (#1719)
-- Reconfigure, start, and stop the stats listener on a config reload instead of requiring a restart. (#1670)
-- Update a static host's addresses when they change on reload. (#1713)
-- Don't require a port on ICMP firewall rules. (#1609)
-- Connection track ICMP traffic. (#1602)
-- Return `NODATA` instead of `NXDOMAIN` from the DNS server for a name that exists but has no record of the
-  requested type, so clients that query `AAAA` first (busybox/Alpine) fall through to `A`. (#1668)
-- Record the local host's details in the DNS server. (#1716)
-- Install Windows unsafe routes as link routes. (#1709)
-- Reduce relay handshake log spam, and only log a handshake send error at error level when the remote list
-  changes. (#1733, #1765, #1810)
-- Start, stop, and reload subsystems (DNS, stats, conntrack, ssh, punchy) cleanly without leaking goroutines. (#1640, #1654, #1661, #1667, #1669, #1708, #1806, #1815)
-- `Control` is now safe to stop and wait on from any lifecycle state, and a new `Control.Wait` blocks until nebula
-  has fully stopped and returns the first fatal reader error. Failed starts release the udp sockets and tun fd
-  instead of leaking them. (#1794)
-- Trigger an immediate lighthouse update when reconnecting to or adding a lighthouse instead of waiting for the next update tick. (#1645)
-- Bring the Darwin and OpenBSD tun implementations in line with the other BSDs. (#1703)
-- Update to build against go v1.26. (#1818)
-- Various dependency updates. (#1586, #1587, #1604, #1617, #1618, #1627, #1628, #1629, #1652, #1664, #1665, #1697, #1721, #1732, #1742, #1743, #1750, #1763, #1771, #1782, #1800, #1807)
-
-### Fixed
-
-- Fix a data race on a host's remote address that could send packets to the wrong address during a roam. (#1773)
-- Fix tunnels that could permanently escape connection manager monitoring. (#1752)
-- Fix a crash when reloading the SSH server's trusted keys. (#1787)
-- Fix hostmap corruption when a host has multiple overlay addresses. Each address now gets its own list instead of
-  a single shared chain, which also fixes two latent bugs on the add and makePrimary paths. (#1788, #1790)
-- Apply `remote_allow_list` IPv4 rules to 4-in-6 mapped addresses. (#1786)
-- Don't panic in the DNS server on a short or empty query name. (#1635)
-- Advance the replay window on relayed packets so a relay drops replayed frames instead of re-forwarding them. (#1751)
-- Fix a race in relay state handling. (#1753)
-- Lock replay window updates so concurrent readers can't corrupt it. (#1802)
-- Reject malformed handshakes more reliably, including invalid ed25519 key lengths. (#1601, #1756)
-- Properly handle `closetunnel` packets. (#1638)
-- Fix an IPv6 extension-header length overflow that could make the firewall parse the wrong protocol and ports. (#1789)
-- Fix relay re-establishment when a handshake arrives over a relay entry that a one-sided teardown left
-  `Disestablished`, which silently dropped every send until dead tunnel detection forced a re-handshake. (#1805)
-- Don't build new relay state on a tunnel that was just discarded. (#1796)
-- Don't delete the wrong pending hostinfo in the handshake manager. (#1811)
-- Don't call the packet reader after a UDP error on Darwin. (#1755)
-- Open the FreeBSD tun device non blocking. (#1666)
-
 ## [1.10.3] - 2026-02-06
 
 ### Security

cmd/nebula/close_on_timer_test.go

@@ -1,96 +0,0 @@
-//go:build linux && !android && !e2e_testing
-
-package main
-
-import (
-	"fmt"
-	"net/netip"
-	"os"
-	"path/filepath"
-	"runtime"
-	"testing"
-	"time"
-
-	"github.com/slackhq/nebula"
-	"github.com/slackhq/nebula/cert"
-	cert_test "github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/test"
-	"github.com/stretchr/testify/require"
-)
-
-// TestControlStopClosesOnTimer reproduces the dnclient lifecycle: nebula runs as
-// a library, and on a config update dnclient calls Stop() in-process to tear the
-// old instance down before starting a new one. This boots a real nebula (real
-// blocking UDP sockets, tun disabled), lets it run, then Stop()s it on a timer
-// and asserts it actually closes. If the reader goroutines parked in recvmmsg
-// don't wake on Close(), Wait() blocks forever and this fails with a goroutine
-// dump instead of relying on a process signal to unstick them.
-func TestControlStopClosesOnTimer(t *testing.T) {
-	l := test.NewLogger()
-	dir := t.TempDir()
-
-	before := time.Now().Add(-time.Hour)
-	after := time.Now().Add(time.Hour)
-	ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, before, after, nil, nil, nil)
-	networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
-	_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "close-on-timer", before, after, networks, nil, nil)
-
-	caPath := filepath.Join(dir, "ca.pem")
-	certPath := filepath.Join(dir, "cert.pem")
-	keyPath := filepath.Join(dir, "key.pem")
-	require.NoError(t, os.WriteFile(caPath, caPEM, 0o600))
-	require.NoError(t, os.WriteFile(certPath, certPEM, 0o600))
-	require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600))
-
-	// tun disabled so no device/root is needed; routines: 2 so we exercise the
-	// multi-socket (SO_REUSEPORT) teardown, which is where dnclient runs.
-	configBody := fmt.Sprintf(`
-pki:
-  ca: %s
-  cert: %s
-  key: %s
-listen:
-  host: 127.0.0.1
-  port: 0
-tun:
-  disabled: true
-firewall:
-  outbound:
-    - port: any
-      proto: any
-      host: any
-  inbound:
-    - port: any
-      proto: any
-      host: any
-routines: 2
-`, caPath, certPath, keyPath)
-	require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(configBody), 0o600))
-
-	c := config.NewC(l)
-	require.NoError(t, c.Load(dir))
-
-	ctrl, err := nebula.Main(c, false, "close-on-timer", l, nil)
-	require.NoError(t, err)
-	require.NoError(t, ctrl.Start())
-
-	// Run like a live nebula, then close on a timer, exactly as dnclient does.
-	<-time.NewTimer(5 * time.Second).C
-
-	stopped := make(chan struct{})
-	go func() {
-		ctrl.Stop() // closes the udp sockets (shutdown(2)) and the tun
-		ctrl.Wait() // blocks until every reader goroutine has returned
-		close(stopped)
-	}()
-
-	select {
-	case <-stopped:
-		t.Log("nebula closed cleanly on timer")
-	case <-time.After(10 * time.Second):
-		buf := make([]byte, 1<<20)
-		n := runtime.Stack(buf, true)
-		t.Fatalf("nebula did NOT close within 10s of Stop(): a blocking reader never woke\n%s", buf[:n])
-	}
-}

connection_manager_test.go

@@ -25,7 +25,6 @@ func newTestLighthouse() *LightHouse {
 	lighthouses := []netip.Addr{}
 	staticList := map[netip.Addr]struct{}{}
 
-	lh.localAddrsFn = func(*LocalAllowList) []netip.Addr { return nil }
 	lh.lighthouses.Store(&lighthouses)
 	lh.staticList.Store(&staticList)
 

connection_state.go

@@ -2,13 +2,11 @@ package nebula
 
 import (
 	"encoding/json"
-	"log/slog"
 	"sync"
 	"sync/atomic"
 
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/handshake"
-	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/noiseutil"
 )
 
@@ -22,7 +20,6 @@ type ConnectionState struct {
 	initiator      bool
 	messageCounter atomic.Uint64
 	window         *Bits
-	decryptLock    sync.Mutex
 	writeLock      sync.Mutex
 }
 
@@ -57,52 +54,3 @@ func (cs *ConnectionState) MarshalJSON() ([]byte, error) {
 func (cs *ConnectionState) Curve() cert.Curve {
 	return cs.myCert.Curve()
 }
-
-func (cs *ConnectionState) Decrypt(l *slog.Logger, messageCounter uint64, out []byte, packet []byte, nb []byte) ([]byte, error) {
-	var err error
-	cs.decryptLock.Lock()
-	result := cs.window.Check(l, messageCounter)
-	cs.decryptLock.Unlock()
-	if !result {
-		return nil, ErrAlreadySeen
-	}
-
-	out, err = cs.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
-	if err != nil {
-		return nil, err
-	}
-
-	cs.decryptLock.Lock()
-	result = cs.window.Update(l, messageCounter)
-	cs.decryptLock.Unlock()
-	if !result {
-		return nil, ErrAlreadySeen
-	}
-	return out, nil
-}
-
-// VerifyRelay verifies AEAD protected (but not encrypted) relay frames. packet must be length-checked by the caller.
-func (cs *ConnectionState) VerifyRelay(l *slog.Logger, messageCounter uint64, packet []byte, nb []byte) error {
-	cs.decryptLock.Lock()
-	result := cs.window.Check(l, messageCounter)
-	cs.decryptLock.Unlock()
-	if !result {
-		return ErrAlreadySeen
-	}
-
-	signedPayload := packet[:len(packet)-cs.dKey.Overhead()]
-	signatureValue := packet[len(packet)-cs.dKey.Overhead():]
-	_, err := cs.dKey.DecryptDanger(nil, signedPayload, signatureValue, messageCounter, nb)
-	if err != nil {
-		return err
-	}
-
-	cs.decryptLock.Lock()
-	result = cs.window.Update(l, messageCounter)
-	cs.decryptLock.Unlock()
-	if !result {
-		return ErrAlreadySeen
-	}
-
-	return nil
-}

control.go

@@ -52,9 +52,7 @@ type Control struct {
 	sshStart               func()
 	statsStart             func()
 	dnsStart               func()
-	infoAPIStart           func()
 	lighthouseStart        func()
-	networkChangeStart     func(rebind func())
 	connectionManagerStart func(context.Context)
 }
 
@@ -106,12 +104,6 @@ func (c *Control) Start() error {
 	if c.dnsStart != nil {
 		go c.dnsStart()
 	}
-	if c.networkChangeStart != nil {
-		go c.networkChangeStart(c.RebindUDPServer)
-	}
-	if c.infoAPIStart != nil {
-		go c.infoAPIStart()
-	}
 	if c.connectionManagerStart != nil {
 		go c.connectionManagerStart(c.ctx)
 	}
@@ -206,11 +198,7 @@ func (c *Control) RebindUDPServer() {
 		return
 	}
 
-	// A failure here means we are likely still pinned to the interface we came up on, so the rest of this is
+	_ = c.f.outside.Rebind()
-	// unlikely to help. Say so instead of silently carrying on as if we rebound.
-	if err := c.f.outside.Rebind(); err != nil {
-		c.l.Error("Failed to rebind udp socket", "error", err)
-	}
 
 	// Trigger a lighthouse update, useful for mobile clients that should have an update interval of 0
 	c.f.lightHouse.SendUpdate()

control_tester.go

@@ -108,19 +108,7 @@ func (c *Control) GetVpnAddrs() []netip.Addr {
 }
 
 func (c *Control) GetUDPAddr() netip.AddrPort {
-	return c.f.outside.(*udp.TesterConn).GetAddr()
+	return c.f.outside.(*udp.TesterConn).Addr
-}
-
-// SetUDPAddr moves this node to a new underlay address, standing in for a laptop waking up on a different
-// network. Register the new address with the router as well or nothing will route back.
-func (c *Control) SetUDPAddr(addr netip.AddrPort) {
-	c.f.outside.(*udp.TesterConn).SetAddr(addr)
-}
-
-// SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of
-// whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker.
-func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {
-	c.f.lightHouse.localAddrsFn = fn
 }
 
 func (c *Control) KillPendingTunnel(vpnIp netip.Addr) bool {

dns_server.go

@@ -97,7 +97,8 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
 	newAddr := getDnsServerAddr(c)
 
 	d.serverMu.Lock()
-	running := d.server != nil
+	running := d.server
+	runningStarted := d.started
 	sameAddr := d.addr == newAddr
 	d.addr = newAddr
 	d.enabled.Store(enabled)
@@ -111,7 +112,7 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
 	}
 
 	if !enabled {
-		if running {
+		if running != nil {
 			d.Stop()
 		}
 		// Drop any records that accumulated while enabled; a later re-enable
@@ -120,12 +121,12 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
 		return nil
 	}
 
-	if !running {
+	if running == nil {
 		// Was disabled (or never started); bring it up now.
 		go d.Start()
 	} else if !sameAddr {
-		// Stop clears the slot before shutting down, otherwise the Start below can find the dying server and refuse
+		d.shutdownServer(running, runningStarted, "reload")
-		d.Stop()
+		// Old Start goroutine has now exited; bring up a fresh listener on the new address.
 		go d.Start()
 	}
 
@@ -161,9 +162,7 @@ func (d *dnsServer) Start() {
 
 	started := make(chan struct{})
 	d.serverMu.Lock()
-	// Re-check enabled under the lock, a disable that raced our check above snapshots the slot under it too.
+	if d.ctx.Err() != nil {
-	// Two reloads in quick succession can both spawn a Start, the loser would orphan the live listener past Stop
-	if d.ctx.Err() != nil || d.server != nil || !d.enabled.Load() {
 		d.serverMu.Unlock()
 		return
 	}
@@ -201,14 +200,6 @@ func (d *dnsServer) Start() {
 		close(started)
 	}
 
-	// Release our slot, unless a reload already replaced us, so a dead listener can't block a future Start
-	d.serverMu.Lock()
-	if d.server == server {
-		d.server = nil
-		d.started = nil
-	}
-	d.serverMu.Unlock()
-
 	if err != nil {
 		d.l.Warn("Failed to run the DNS responder", "error", err)
 	}
@@ -258,12 +249,31 @@ func (d *dnsServer) QueryCert(data string) string {
 		return ""
 	}
 
-	crt := findCertificateForVpnAddr(d.certState(), d.hostMap, ip)
+	// The hostmap only ever contains peers we have handshaked with, so it never carries an entry for ourselves.
-	if crt == nil {
+	// Answer self lookups straight from the local cert state.
+	if cs := d.certState(); cs != nil && cs.myVpnAddrsTable != nil && cs.myVpnAddrsTable.Contains(ip) {
+		c := cs.GetDefaultCertificate()
+		if c == nil {
+			return ""
+		}
+		b, err := c.MarshalJSON()
+		if err != nil {
+			return ""
+		}
+		return string(b)
+	}
+
+	hostinfo := d.hostMap.QueryVpnAddr(ip)
+	if hostinfo == nil {
 		return ""
 	}
 
-	b, err := crt.MarshalJSON()
+	q := hostinfo.GetCert()
+	if q == nil {
+		return ""
+	}
+
+	b, err := q.Certificate.MarshalJSON()
 	if err != nil {
 		return ""
 	}

dns_server_test.go

@@ -194,51 +194,14 @@ func TestDnsServer_reload_initial_serveDnsWithoutLighthouse(t *testing.T) {
 }
 
 func TestDnsServer_reload_sameAddr_noOp(t *testing.T) {
-	port := freeUDPPort(t)
 	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", port, true, true)
+	setDnsConfig(c, "127.0.0.1", "0", true, true)
+
 	require.NoError(t, ds.reload(c, true))
-
+	// No server running yet, no addr change. Reload should not spawn anything.
-	go ds.Start()
-	waitForBind(t, ds)
-
-	ds.serverMu.Lock()
-	before := ds.server
-	ds.serverMu.Unlock()
-	require.NotNil(t, before)
-
-	// Same address, so the running listener must be left alone rather than rebuilt under live queries
 	require.NoError(t, ds.reload(c, false))
 	assert.True(t, ds.enabled.Load())
-
+	assert.Nil(t, ds.server)
-	ds.serverMu.Lock()
-	after := ds.server
-	ds.serverMu.Unlock()
-	assert.Same(t, before, after, "a same-address reload must not restart the listener")
-
-	ds.Stop()
-}
-
-// The branch the old sameAddr test was accidentally hitting: enabled with nothing running means reload starts it.
-func TestDnsServer_reload_whenNotRunning_starts(t *testing.T) {
-	port := freeUDPPort(t)
-	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", port, true, true)
-
-	// initial only records config, it never starts anything
-	require.NoError(t, ds.reload(c, true))
-	ds.serverMu.Lock()
-	assert.Nil(t, ds.server, "the initial reload must not start a listener")
-	ds.serverMu.Unlock()
-
-	require.NoError(t, ds.reload(c, false))
-	waitForBind(t, ds)
-
-	ds.serverMu.Lock()
-	assert.NotNil(t, ds.server, "a reload with nothing running should bring DNS up")
-	ds.serverMu.Unlock()
-
-	ds.Stop()
 }
 
 func TestDnsServer_StartStop_lifecycle(t *testing.T) {
@@ -464,168 +427,3 @@ func waitFor(t *testing.T, cond func() bool) {
 	}
 	t.Fatal("timed out waiting for condition")
 }
-
-// Two reloads in quick succession, or a HUP before Control.Start, can race two Starts at the same listener.
-func TestDnsServer_Start_isIdempotent(t *testing.T) {
-	port := freeUDPPort(t)
-	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", port, true, true)
-	require.NoError(t, ds.reload(c, true))
-
-	go ds.Start()
-	waitForBind(t, ds)
-
-	ds.serverMu.Lock()
-	first := ds.server
-	ds.serverMu.Unlock()
-	require.NotNil(t, first)
-
-	// If the second Start replaces the tracked server, Stop kills the wrong one and the port leaks
-	done := make(chan struct{})
-	go func() {
-		ds.Start()
-		close(done)
-	}()
-	select {
-	case <-done:
-	case <-time.After(time.Second * 5):
-		t.Fatal("second Start never returned")
-	}
-
-	ds.serverMu.Lock()
-	second := ds.server
-	ds.serverMu.Unlock()
-	assert.Same(t, first, second, "a second Start must not replace the running server")
-
-	// The real proof, after Stop the port must actually be free
-	ds.Stop()
-	waitFor(t, func() bool {
-		pc, err := net.ListenPacket("udp", "127.0.0.1:"+port)
-		if err != nil {
-			return false
-		}
-		_ = pc.Close()
-		return true
-	})
-}
-
-// An address change must actually end up listening on the new port. Start's guard refuses when a server is already
-// installed, so reload has to clear the slot before shutting the old one down.
-func TestDnsServer_reload_addrChange_restarts(t *testing.T) {
-	first := freeUDPPort(t)
-	second := freeUDPPort(t)
-
-	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", first, true, true)
-	require.NoError(t, ds.reload(c, true))
-
-	go ds.Start()
-	waitForBind(t, ds)
-
-	// Cycle a few times, the failure this guards against depends on which goroutine wins serverMu
-	for i := range 8 {
-		want := second
-		if i%2 == 1 {
-			want = first
-		}
-		setDnsConfig(c, "127.0.0.1", want, true, true)
-		require.NoError(t, ds.reload(c, false))
-		waitForBind(t, ds)
-
-		ds.serverMu.Lock()
-		srv := ds.server
-		ds.serverMu.Unlock()
-		require.NotNil(t, srv, "reload left DNS down instead of restarting it")
-		require.Equal(t, "127.0.0.1:"+want, srv.Addr, "reload should be serving the new address")
-	}
-
-	// Land back on second so the port assertions below are meaningful
-	setDnsConfig(c, "127.0.0.1", second, true, true)
-	require.NoError(t, ds.reload(c, false))
-	waitForBind(t, ds)
-
-	// The old port must be released and the new one actually held
-	waitFor(t, func() bool {
-		pc, err := net.ListenPacket("udp", "127.0.0.1:"+first)
-		if err != nil {
-			return false
-		}
-		_ = pc.Close()
-		return true
-	})
-	_, err := net.ListenPacket("udp", "127.0.0.1:"+second)
-	require.Error(t, err, "the new address should be bound by the DNS responder")
-
-	ds.Stop()
-}
-
-// A listener that dies on its own must release the slot, or a later same-addr reload sees it as running and no-ops.
-func TestDnsServer_Start_bindFailure_releasesSlot(t *testing.T) {
-	port := freeUDPPort(t)
-	blocker, err := net.ListenPacket("udp", "127.0.0.1:"+port)
-	require.NoError(t, err)
-
-	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", port, true, true)
-	require.NoError(t, ds.reload(c, true))
-
-	ds.Start() // returns once the bind fails
-
-	ds.serverMu.Lock()
-	assert.Nil(t, ds.server, "a listener that failed to bind must not stay parked in the slot")
-	ds.serverMu.Unlock()
-
-	// With the slot released, a reload can retry once the port frees up
-	require.NoError(t, blocker.Close())
-	require.NoError(t, ds.reload(c, false))
-	waitForBind(t, ds)
-
-	ds.serverMu.Lock()
-	assert.NotNil(t, ds.server, "a same-addr reload should retry after a failed bind")
-	ds.serverMu.Unlock()
-
-	ds.Stop()
-}
-
-// A disable that lands while Start is between its unlocked check and the guard must not leave a listener behind.
-func TestDnsServer_Start_refusesWhenDisabledUnderLock(t *testing.T) {
-	port := freeUDPPort(t)
-	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", port, true, true)
-	require.NoError(t, ds.reload(c, true))
-	require.True(t, ds.enabled.Load())
-
-	// Holding serverMu parks Start on the lock, the only way to land the disable in that window on purpose
-	ds.serverMu.Lock()
-
-	done := make(chan struct{})
-	go func() {
-		ds.Start()
-		close(done)
-	}()
-
-	select {
-	case <-done:
-		ds.serverMu.Unlock()
-		t.Fatal("Start returned early, the test never exercised the window")
-	case <-time.After(time.Millisecond * 100):
-	}
-
-	// The disable reload's critical section. It sees nothing running, so it never calls Stop.
-	ds.enabled.Store(false)
-	ds.serverMu.Unlock()
-
-	select {
-	case <-done:
-	case <-time.After(time.Second * 5):
-		t.Fatal("Start never returned")
-	}
-
-	ds.serverMu.Lock()
-	assert.Nil(t, ds.server, "Start must not install a listener a disable already cancelled")
-	ds.serverMu.Unlock()
-
-	pc, err := net.ListenPacket("udp", "127.0.0.1:"+port)
-	require.NoError(t, err, "an orphaned listener is still holding the port")
-	_ = pc.Close()
-}

e2e/handshakes_test.go

@@ -725,70 +725,6 @@ func TestReestablishRelays(t *testing.T) {
 
 }
 
-func TestRelayHandshakeOverDisestablishedEntry(t *testing.T) {
-	t.Parallel()
-	// If them tears down the tunnel while me keeps Established relay state, me's next
-	// handshake flows through the relay with no fresh CreateRelayRequest and lands on
-	// them's Disestablished terminal relay entry. them must re-establish that entry, or
-	// its first transmit deletes its only relay and the tunnel is born transmit-dead:
-	// them can receive but every send is silently dropped.
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
-	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay  ", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them   ", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
-
-	// Teach my how to get to the relay and that their can be reached via the relay
-	myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
-	myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
-	relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	// Build a router so we don't have to reason who gets which packet
-	r := router.NewR(t, myControl, relayControl, theirControl)
-	defer r.RenderFlow()
-
-	// Start the servers
-	myControl.Start()
-	relayControl.Start()
-	theirControl.Start()
-
-	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
-
-	p := r.RouteForAllUntilTxTun(theirControl)
-	assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
-	oldIdx := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false).LocalIndex
-
-	t.Log("Close the tunnel on them only, marking their relay entry Disestablished")
-	theirControl.CloseTunnel(myVpnIpNet[0].Addr(), true)
-
-	t.Log("Re-handshake from me, riding the still-Established relay state")
-	myControl.ReHandshake(theirVpnIpNet[0].Addr())
-	for {
-		h := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
-		if h != nil && h.LocalIndex != oldIdx && h.RemoteIndex != 0 {
-			break
-		}
-		r.RouteForAllExitFunc(func(*udp.Packet, *nebula.Control) router.ExitType {
-			return router.RouteAndExit
-		})
-	}
-
-	hAtThem := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-	require.NotNil(t, hAtThem, "them should have completed the relayed handshake")
-	require.Equal(t, []netip.Addr{relayVpnIpNet[0].Addr()}, hAtThem.CurrentRelaysToMe, "them should know a relay for the new tunnel")
-
-	t.Log("Send from them to me; their only relay entry must survive the transmit")
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them")))
-	require.Never(t, func() bool {
-		h := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-		return h == nil || len(h.CurrentRelaysToMe) == 0
-	}, time.Second, 10*time.Millisecond, "them deleted its only relay entry; the tunnel is permanently transmit-dead")
-
-	p = r.RouteForAllUntilTxTun(myControl)
-	assertUdpPacket(t, []byte("Hi from them"), p, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), 80, 80)
-	r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
-}
-
 func TestStage1RaceRelays(t *testing.T) {
 	t.Parallel()
 	//NOTE: this is a race between me and relay resulting in a full tunnel from me to them via relay

e2e/rebind_test.go

@@ -1,225 +0,0 @@
-//go:build e2e_testing
-// +build e2e_testing
-
-package e2e
-
-import (
-	"net/netip"
-	"testing"
-	"time"
-
-	"github.com/slackhq/nebula"
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/e2e/router"
-	"github.com/slackhq/nebula/header"
-	"github.com/slackhq/nebula/udp"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// reportedAddrs is what the lighthouse would hand a peer asking where vpnAddr is.
-func reportedAddrs(t *testing.T, lh *nebula.Control, vpnAddr netip.Addr) []netip.AddrPort {
-	t.Helper()
-	cm := lh.QueryLighthouse(vpnAddr)
-	if cm == nil {
-		return nil
-	}
-	var out []netip.AddrPort
-	for _, c := range *cm {
-		out = append(out, c.Reported...)
-		out = append(out, c.Learned...)
-	}
-	return out
-}
-
-// waitForLighthouseMsg routes until a lighthouse message lands on lh, or gives up. Reports whether one arrived.
-func waitForLighthouseMsg(t *testing.T, r *router.R, lh *nebula.Control, wait time.Duration) bool {
-	t.Helper()
-	h := &header.H{}
-	return r.RouteForAllExitFuncOrTimeout(wait, func(p *udp.Packet, c *nebula.Control) router.ExitType {
-		if c != lh {
-			return router.KeepRouting
-		}
-		// Punches are a single byte and never parse, they are just not what we are after
-		if err := h.Parse(p.Data); err != nil {
-			return router.KeepRouting
-		}
-		if h.Type == header.LightHouse {
-			return router.RouteAndExit
-		}
-		return router.KeepRouting
-	})
-}
-
-// A laptop that changes networks has to tell the lighthouse promptly, otherwise the lighthouse keeps handing peers
-// the old address and their punches land nowhere. On a long lighthouse interval the only thing that closes that
-// window is the rebind, which on darwin the network change monitor drives. The e2e build compiles the monitor out,
-// so we call RebindUDPServer directly, which is the same thing the monitor does.
-func TestRebindSendsLighthouseUpdate(t *testing.T) {
-	t.Parallel()
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
-		"lighthouse": m{"am_lighthouse": true},
-	})
-
-	// 600s interval, so nothing scheduled can send an update during this test. A rebind is the only thing that can.
-	myControl, _, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{
-		"lighthouse": m{
-			"hosts":    []any{lhVpnIpNet[0].Addr().String()},
-			"interval": 600,
-		},
-		"static_host_map": m{
-			lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
-		},
-	})
-
-	r := router.NewR(t, lhControl, myControl)
-	defer r.RenderFlow()
-
-	lhControl.Start()
-	myControl.Start()
-
-	// Let the startup registration finish, then clear everything it left behind
-	require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an initial registration")
-	r.RouteFor(time.Millisecond * 400)
-
-	// Nothing should be talking to the lighthouse on its own now
-	require.False(t, waitForLighthouseMsg(t, r, lhControl, time.Millisecond*200),
-		"nothing should reach the lighthouse before the rebind")
-
-	myControl.RebindUDPServer()
-
-	assert.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5),
-		"a rebind should push an update to the lighthouse rather than waiting out the interval")
-
-	lhControl.Stop()
-	myControl.Stop()
-}
-
-// The other half of a rebind: every live tunnel requeries the lighthouse on its next send. That query is what makes
-// the lighthouse tell the peer to punch toward our new address, which is the part that actually revives a tunnel
-// whose remote NAT state died while we were on a different network.
-func TestRebindRequeriesPeersOnNextSend(t *testing.T) {
-	t.Parallel()
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
-		"lighthouse": m{"am_lighthouse": true},
-	})
-
-	lhCfg := m{
-		"lighthouse": m{
-			"hosts":    []any{lhVpnIpNet[0].Addr().String()},
-			"interval": 600,
-			// Without this the peers advertise this machine's real addresses and then try to punch at them,
-			// which the router has no route for.
-			"local_allow_list": m{
-				"10.0.0.0/24": true,
-				"::/0":        false,
-			},
-		},
-		"static_host_map": m{
-			lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
-		},
-	}
-
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", lhCfg)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.3/24", lhCfg)
-
-	r := router.NewR(t, lhControl, myControl, theirControl)
-	defer r.RenderFlow()
-
-	lhControl.Start()
-	myControl.Start()
-	theirControl.Start()
-	r.RouteFor(time.Millisecond * 500)
-
-	// Point the peers at each other directly, this test is about the rebind and not about lighthouse discovery
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("initial")))
-	r.RouteFor(time.Second)
-	require.NotNil(t, myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false), "expected a tunnel to them")
-	r.RouteFor(time.Millisecond * 300)
-
-	// Assert on what the peer sees rather than on lighthouse traffic. A query for them makes the lighthouse send
-	// them a punch notification, which is the whole point. Our own update to the lighthouse sends them nothing,
-	// so this cannot be satisfied by the update the rebind itself pushes.
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("quiet")))
-	require.False(t, waitForLighthouseMsg(t, r, theirControl, time.Millisecond*300),
-		"an ordinary send should not requery the lighthouse")
-
-	myControl.RebindUDPServer()
-	r.RouteFor(time.Millisecond * 300) // let the update the rebind itself sends pass by
-
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("after rebind")))
-	assert.True(t, waitForLighthouseMsg(t, r, theirControl, time.Second*5),
-		"the first send after a rebind should requery the lighthouse, which then tells the peer to punch at us")
-
-	lhControl.Stop()
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-// The scenario this whole thing exists for: a laptop sleeps at the office and wakes up at home on a new address.
-// Until it tells the lighthouse, the lighthouse keeps handing peers the office address, so their punches land
-// nowhere and the tunnel stays dead. On a long interval the rebind is the only thing that closes that window.
-func TestRebindAdvertisesNewAddressAfterMove(t *testing.T) {
-	t.Parallel()
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
-		"lighthouse": m{"am_lighthouse": true},
-	})
-
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{
-		"lighthouse": m{
-			"hosts":    []any{lhVpnIpNet[0].Addr().String()},
-			"interval": 600,
-		},
-		"static_host_map": m{
-			lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
-		},
-	})
-
-	// Advertise wherever we currently are rather than this machine's real NICs, read fresh each time so a move
-	// is picked up.
-	myControl.SetLocalAddrsFn(func(*nebula.LocalAllowList) []netip.Addr {
-		return []netip.Addr{myControl.GetUDPAddr().Addr()}
-	})
-
-	r := router.NewR(t, lhControl, myControl)
-	defer r.RenderFlow()
-
-	lhControl.Start()
-	myControl.Start()
-
-	require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an initial registration")
-	r.RouteFor(time.Millisecond * 400)
-
-	require.Contains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), myUdpAddr,
-		"the lighthouse should know the address we started on")
-
-	// Wake up somewhere else
-	newAddr := netip.MustParseAddrPort("10.0.0.99:4242")
-	myControl.SetUDPAddr(newAddr)
-	r.AddRoute(newAddr.Addr(), newAddr.Port(), myControl)
-
-	// Nothing has told the lighthouse, and with interval 600 nothing scheduled will
-	r.RouteFor(time.Millisecond * 400)
-	require.NotContains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), newAddr,
-		"the lighthouse should still be handing out the old address before the rebind")
-
-	myControl.RebindUDPServer()
-	require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an update after the rebind")
-	r.RouteFor(time.Millisecond * 400)
-
-	assert.Contains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), newAddr,
-		"after the rebind the lighthouse should hand peers our new address")
-
-	lhControl.Stop()
-	myControl.Stop()
-}

e2e/recovery_timing_test.go

@@ -1,136 +0,0 @@
-//go:build e2e_testing
-// +build e2e_testing
-
-package e2e
-
-import (
-	"testing"
-	"time"
-
-	"github.com/slackhq/nebula"
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/e2e/router"
-	"github.com/slackhq/nebula/udp"
-)
-
-// TestRecoveryTiming measures how long a tunnel takes to come back after the peer stops accepting our traffic,
-// which is what a laptop waking on a new network looks like from the peer's side: its NAT has no state for where
-// we are now, so everything we send disappears.
-//
-// It is a measurement, not a pass/fail assertion. Recovery is timed to the moment the peer punches back at us,
-// since that is when its NAT opens and the tunnel is usable again.
-//
-//	go test -tags e2e_testing -v -run TestRecoveryTiming ./e2e/
-func TestRecoveryTiming(t *testing.T) {
-	for _, tc := range []struct {
-		name   string
-		rebind bool
-	}{
-		{"no trigger", false},
-		{"rebind counter", true},
-	} {
-		t.Run(tc.name, func(t *testing.T) {
-			d, lost := measureRecovery(t, tc.rebind)
-			t.Logf("RESULT %-16s recovered in %-9v (%d packets lost)", tc.name, d.Round(time.Millisecond), lost)
-		})
-	}
-}
-
-// measureRecovery returns how long until the peer punched back, and how many of our packets died meanwhile. When
-// rebind is true we call RebindUDPServer once the tunnel goes dark, which is what the darwin network change
-// monitor does and what iOS has always done. When false, nothing tells nebula anything is wrong.
-func measureRecovery(t *testing.T, rebind bool) (time.Duration, int) {
-	t.Helper()
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
-		"lighthouse": m{"am_lighthouse": true},
-	})
-
-	peerCfg := m{
-		"lighthouse": m{
-			"hosts":    []any{lhVpnIpNet[0].Addr().String()},
-			"interval": 600,
-			"local_allow_list": m{
-				"10.0.0.0/24": true,
-				"::/0":        false,
-			},
-		},
-		"static_host_map": m{
-			lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
-		},
-	}
-
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", peerCfg)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.3/24", peerCfg)
-
-	r := router.NewR(t, lhControl, myControl, theirControl)
-	defer r.RenderFlow()
-	defer func() {
-		lhControl.Stop()
-		myControl.Stop()
-		theirControl.Stop()
-	}()
-
-	lhControl.Start()
-	myControl.Start()
-	theirControl.Start()
-	r.RouteFor(time.Millisecond * 500)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("establish")))
-	r.RouteFor(time.Second)
-	if myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false) == nil {
-		t.Fatal("failed to establish the tunnel we are measuring")
-	}
-	r.RouteFor(time.Millisecond * 500)
-
-	// From here the peer's NAT has no state for us, everything we send it disappears
-	start := time.Now()
-	blackholed := 0
-	var recovered time.Duration
-
-	if rebind {
-		myControl.RebindUDPServer()
-	}
-
-	// Keep the tun busy the way someone retrying a stalled connection would
-	stop := make(chan struct{})
-	defer close(stop)
-	go func() {
-		tick := time.NewTicker(time.Millisecond * 200)
-		defer tick.Stop()
-		for {
-			select {
-			case <-stop:
-				return
-			case <-tick.C:
-				myControl.InjectTunPacket(BuildTunUDPPacket(
-					theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("retry")))
-			}
-		}
-	}()
-
-	r.RouteForAllExitFuncOrTimeout(time.Second*30, func(p *udp.Packet, c *nebula.Control) router.ExitType {
-		if c == theirControl && p.From == myControl.GetUDPAddr() {
-			blackholed++
-			return router.Drop
-		}
-
-		// The peer reaching us directly is the moment its NAT opened, whether that is a punch or a handshake
-		if c == myControl && p.From == theirUdpAddr {
-			recovered = time.Since(start)
-			return router.RouteAndExit
-		}
-
-		return router.KeepRouting
-	})
-
-	if recovered == 0 {
-		t.Fatalf("no recovery within 30s (%d packets blackholed)", blackholed)
-	}
-	return recovered, blackholed
-}

e2e/router/router.go

@@ -114,28 +114,6 @@ type packet struct {
 	packet *udp.Packet
 	tun    bool // a packet pulled off a tun device
 	rx     bool // the packet was received by a udp device
-
-	// h is the nebula header, parsed once when the packet is recorded. parseErr says why there isn't one, which
-	// the flow log reports rather than hiding. Punchy sends a single byte, so an unparseable packet is normal.
-	h        header.H
-	parseErr error
-}
-
-// fromAddr and toAddr are the addresses this packet actually travelled between. Reading them off the control
-// instead would misreport the whole history once a test moves a node. Tun packets are synthesized without
-// addresses, so they fall back to the control.
-func (p *packet) fromAddr() netip.AddrPort {
-	if p.tun || !p.packet.From.IsValid() {
-		return p.from.GetUDPAddr()
-	}
-	return p.packet.From
-}
-
-func (p *packet) toAddr() netip.AddrPort {
-	if p.tun || !p.packet.To.IsValid() {
-		return p.to.GetUDPAddr()
-	}
-	return p.packet.To
 }
 
 func (p *packet) WasReceived() {
@@ -153,9 +131,6 @@ const (
 	ExitNow ExitType = 1
 	// RouteAndExit routes this packet and exits immediately afterwards
 	RouteAndExit ExitType = 2
-	// Drop discards this packet without delivering it and keeps routing. Use it to simulate a blackhole, such as
-	// a restrictive NAT refusing traffic from an address it has not seen.
-	Drop ExitType = 3
 )
 
 type ExitFunc func(packet *udp.Packet, receiver *nebula.Control) ExitType
@@ -166,9 +141,7 @@ type ExitFunc func(packet *udp.Packet, receiver *nebula.Control) ExitType
 func NewR(t testing.TB, controls ...*nebula.Control) *R {
 	ctx, cancel := context.WithCancel(context.Background())
 
-	// t.Name() contains a slash for subtests, so the flow log can land in a nested directory
+	if err := os.MkdirAll("mermaid", 0755); err != nil {
-	fn := filepath.Join("mermaid", fmt.Sprintf("%s.md", t.Name()))
-	if err := os.MkdirAll(filepath.Dir(fn), 0755); err != nil {
 		panic(err)
 	}
 
@@ -179,7 +152,7 @@ func NewR(t testing.TB, controls ...*nebula.Control) *R {
 		outNat:       make(map[outNatKey]netip.AddrPort),
 		flow:         []flowEntry{},
 		ignoreFlows:  []ignoreFlow{},
-		fn:           fn,
+		fn:           filepath.Join("mermaid", fmt.Sprintf("%s.md", t.Name())),
 		t:            t,
 		cancelRender: cancel,
 	}
@@ -276,7 +249,7 @@ func (r *R) renderFlow() {
 			continue
 		}
 
-		addr := e.packet.fromAddr()
+		addr := e.packet.from.GetUDPAddr()
 		if _, ok := participants[addr]; ok {
 			continue
 		}
@@ -295,6 +268,7 @@ func (r *R) renderFlow() {
 	}
 
 	// Print packets
+	h := &header.H{}
 	for _, e := range r.flow {
 		if e.packet == nil {
 			//fmt.Fprintf(f, "    note over %s: %s\n", strings.Join(participantsVals, ", "), e.note)
@@ -306,22 +280,21 @@ func (r *R) renderFlow() {
 			fmt.Fprintln(f, r.formatUdpPacket(p))
 
 		} else {
+			if err := h.Parse(p.packet.Data); err != nil {
+				panic(err)
+			}
+
 			line := "--x"
 			if p.rx {
 				line = "->>"
 			}
 
-			detail := fmt.Sprintf("%s(%s), index %v, counter: %v",
+			fmt.Fprintf(f,
-				p.h.TypeName(), p.h.SubTypeName(), p.h.RemoteIndex, p.h.MessageCounter)
+				"    %s%s%s: %s(%s), index %v, counter: %v\n",
-			if p.parseErr != nil {
+				normalizeName(p.from.GetUDPAddr().String()),
-				detail = fmt.Sprintf("unparsed, %v (%d bytes)", p.parseErr, len(p.packet.Data))
-			}
-
-			fmt.Fprintf(f, "    %s%s%s: %s\n",
-				normalizeName(p.fromAddr().String()),
 				line,
-				normalizeName(p.toAddr().String()),
+				normalizeName(p.to.GetUDPAddr().String()),
-				detail,
+				h.TypeName(), h.SubTypeName(), h.RemoteIndex, h.MessageCounter,
 			)
 		}
 	}
@@ -435,34 +408,29 @@ func (r *R) unlockedInjectFlow(from, to *nebula.Control, p *udp.Packet, tun bool
 
 	r.renderHostmaps(fmt.Sprintf("Packet %v", len(r.flow)))
 
-	var h header.H
+	if len(r.ignoreFlows) > 0 {
-	var parseErr error
+		var h header.H
-	if !tun {
+		err := h.Parse(p.Data)
-		parseErr = h.Parse(p.Data)
+		if err != nil {
-	}
+			panic(err)
-
-	// Decide before copying, the copy comes from a freelist and an ignored packet would never be released
-	for _, i := range r.ignoreFlows {
-		if tun {
-			if i.tun.HasValue && i.tun.IsTrue {
-				return nil
-			}
-			continue
 		}
 
-		// A packet we could not parse has no type to match against, so no rule can ignore it
+		for _, i := range r.ignoreFlows {
-		if parseErr == nil && i.messageType == h.Type && i.subType == h.Subtype {
+			if !tun {
-			return nil
+				if i.messageType == h.Type && i.subType == h.Subtype {
+					return nil
+				}
+			} else if i.tun.HasValue && i.tun.IsTrue {
+				return nil
+			}
 		}
 	}
 
 	fp := &packet{
-		from:     from,
+		from:   from,
-		to:       to,
+		to:     to,
-		packet:   p.Copy(),
+		packet: p.Copy(),
-		tun:      tun,
+		tun:    tun,
-		h:        h,
-		parseErr: parseErr,
 	}
 
 	r.flow = append(r.flow, flowEntry{packet: fp})
@@ -692,10 +660,6 @@ func (r *R) RouteExitFunc(sender *nebula.Control, whatDo ExitFunc) {
 			p.Release()
 			return
 
-		case Drop:
-			// Record it so the flow log shows the attempt, but never hand it to the receiver
-			r.unlockedInjectFlow(sender, receiver, p, false)
-
 		case KeepRouting:
 			fp := r.unlockedInjectFlow(sender, receiver, p, false)
 			receiver.InjectUDPPacket(p)
@@ -726,85 +690,6 @@ func (r *R) RouteUntilAfterMsgType(sender *nebula.Control, msgType header.Messag
 	})
 }
 
-// RouteFor routes everything that shows up for the given duration and then returns. Use it to let a test settle
-// deterministically rather than sleeping and hoping: a single FlushAll races a completing handshake, which queues
-// more packets right behind it.
-func (r *R) RouteFor(d time.Duration) {
-	r.RouteForAllExitFuncOrTimeout(d, func(*udp.Packet, *nebula.Control) ExitType {
-		return KeepRouting
-	})
-}
-
-// RouteForAllExitFuncOrTimeout is RouteForAllExitFunc with a deadline, reporting whether whatDo asked to exit
-// before time ran out. The unbounded version blocks forever on a quiet network, so this is what a test needs to
-// assert that something does NOT happen, or to route for a fixed settling period.
-func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc) bool {
-	sc := make([]reflect.SelectCase, 0, len(r.controls)+1)
-	cm := make([]*nebula.Control, 0, len(r.controls))
-
-	for _, c := range r.controls {
-		sc = append(sc, reflect.SelectCase{
-			Dir:  reflect.SelectRecv,
-			Chan: reflect.ValueOf(c.GetUDPTxChan()),
-			Send: reflect.Value{},
-		})
-		cm = append(cm, c)
-	}
-
-	timer := time.NewTimer(timeout)
-	defer timer.Stop()
-	sc = append(sc, reflect.SelectCase{
-		Dir:  reflect.SelectRecv,
-		Chan: reflect.ValueOf(timer.C),
-		Send: reflect.Value{},
-	})
-
-	for {
-		x, rx, _ := reflect.Select(sc)
-		if x == len(cm) {
-			return false
-		}
-
-		r.Lock()
-		p := rx.Interface().(*udp.Packet)
-		receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
-		if receiver == nil {
-			r.Unlock()
-			panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String())
-		}
-
-		e := whatDo(p, receiver)
-		switch e {
-		case ExitNow:
-			r.Unlock()
-			p.Release()
-			return true
-
-		case RouteAndExit:
-			fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
-			receiver.InjectUDPPacket(p)
-			fp.WasReceived()
-			r.Unlock()
-			p.Release()
-			return true
-
-		case Drop:
-			// Record it so the flow log shows the attempt, but never hand it to the receiver
-			r.unlockedInjectFlow(cm[x], receiver, p, false)
-
-		case KeepRouting:
-			fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
-			receiver.InjectUDPPacket(p)
-			fp.WasReceived()
-
-		default:
-			panic(fmt.Sprintf("Unknown exitFunc return: %v", e))
-		}
-		r.Unlock()
-		p.Release()
-	}
-}
-
 func (r *R) RouteForAllUntilAfterMsgTypeTo(receiver *nebula.Control, msgType header.MessageType, subType header.MessageSubType) {
 	h := &header.H{}
 	r.RouteForAllExitFunc(func(p *udp.Packet, r *nebula.Control) ExitType {
@@ -897,10 +782,6 @@ func (r *R) RouteForAllExitFunc(whatDo ExitFunc) {
 			p.Release()
 			return
 
-		case Drop:
-			// Record it so the flow log shows the attempt, but never hand it to the receiver
-			r.unlockedInjectFlow(cm[x], receiver, p, false)
-
 		case KeepRouting:
 			fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
 			receiver.InjectUDPPacket(p)

examples/config.yml

@@ -110,6 +110,15 @@ lighthouse:
     #- "1.1.1.1:4242"
     #- "1.2.3.4:0" # port will be replaced with the real listening port
 
+  # Locally discovered addresses are checked against the MTU of the link they were found on. If the link cannot fit
+  # a full-size packet from the nebula tun device (`tun.mtu` plus encapsulation overhead, which is larger for relayed
+  # traffic) without fragmenting, a warning is logged.
+  # When omit_low_mtu_addrs is true, addresses whose links cannot fit normal nebula traffic are dropped from
+  # lighthouse reports entirely.
+  # Addresses that can fit normal nebula traffic but not relayed traffic are always still advertised.
+  # This does not apply to addresses listed in advertise_addrs.
+  #omit_low_mtu_addrs: false
+
   # EXPERIMENTAL: This option may change or disappear in the future.
   # This setting allows us to "guess" what the remote might be for a host
   # while we wait for the lighthouse response.
@@ -146,14 +155,6 @@ listen:
   # Default true; set to false to leave WDF in charge of inbound decisions on the listener port. Not reloadable.
   #windows_bypass_wdf: true
 
-  # On macOS only
-  # macOS scopes the udp socket to the interface it was created on, so moving between networks (wifi to wired,
-  # office to home) leaves Nebula sending out an interface that no longer has a route. When true, Nebula watches
-  # the routing socket and rebinds the listener once the change settles.
-  # iOS does not use this, the host app drives the same rebind itself.
-  # Default true. Not reloadable.
-  #rebind_on_network_change: true
-
   # By default, Nebula replies to packets it has no tunnel for with a "recv_error" packet. This packet helps speed up reconnection
   # in the case that Nebula on either side did not shut down cleanly. This response can be abused as a way to discover if Nebula is running
   # on a host though. This option lets you configure if you want to send "recv_error" packets always, never, or only to private network remotes.
@@ -231,35 +232,6 @@ punchy:
   # Overriding this to "" is the same as "/" and will allow overwriting any path on the host.
   #sandbox_dir: /var/tmp/nebula-debug
 
-# EXPERIMENTAL: this feature may change or disappear in the future.
-# info_api exposes a small local HTTP+JSON API that lets other programs on
-# this machine resolve a vpn address to its certificate identity (name, vpn
-# addresses, groups, fingerprint, validity), e.g. for making authorization
-# decisions about an inbound connection:
-#   GET /v1/host?addr=<vpn addr>  - identity of the host owning the address: a
-#     peer with an active tunnel, or this node itself. `addr` may include a
-#     port (`192.168.100.7:54321`), which is ignored, so a connection's remote
-#     address can be passed through as is. Returns 404 when the address is
-#     unknown or has no active tunnel.
-#   GET /v1/self                  - this node's own identity.
-# Identity answers can be trusted because nebula drops inbound packets whose
-# source vpn address is not contained in the sender's certificate, so the
-# source address of a connection arriving over the nebula interface is
-# guaranteed to map to the certificate reported here.
-# There is no authentication in this API; restrict access with unix socket
-# file permissions.
-# This whole section is reloadable.
-#info_api:
-  # Toggles the feature
-  #enabled: false
-  # listen accepts a unix socket path as a unix:// URL with an absolute path:
-  #listen: unix:///var/run/nebula-info-api.sock
-  # File mode for the unix socket, as an octal string.
-  # The socket is created by nebula's user; to grant a group of local services
-  # access, place the socket in a directory with appropriate permissions
-  # (e.g. a systemd RuntimeDirectory) and relax this to "0660".
-  #socket_mode: "0600"
-
 # EXPERIMENTAL: relay support for networks that can't establish direct connections.
 relay:
   # Relays are a list of Nebula IP's that peers can use to relay packets to me.

examples/service_scripts/nebula.service

@@ -8,15 +8,6 @@ Before=sshd.service
 Type=notify
 NotifyAccess=main
 SyslogIdentifier=nebula
-
-# Uncomment to run as an unprivileged user with only CAP_NET_ADMIN. Requires a
-# nebula user that owns the config directory. Add CAP_NET_BIND_SERVICE to both
-# lines if any listener (lighthouse DNS, listen.port, stats, sshd) binds <1024.
-#User=nebula
-#Group=nebula
-#CapabilityBoundingSet=CAP_NET_ADMIN
-#AmbientCapabilities=CAP_NET_ADMIN
-
 ExecReload=/bin/kill -HUP $MAINPID
 ExecStart=/usr/local/bin/nebula -config /etc/nebula/config.yml
 Restart=always

firewall.go

@@ -44,8 +44,8 @@ type Firewall struct {
 	InRules  *FirewallTable
 	OutRules *FirewallTable
 
-	InboundSendReject  bool
+	InSendReject  bool
-	OutboundSendReject bool
+	OutSendReject bool
 
 	//TODO: we should have many more options for TCP, an option for ICMP, and mimic the kernel a bit better
 	// https://www.kernel.org/doc/Documentation/networking/nf_conntrack-sysctl.txt
@@ -216,23 +216,23 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
 	inboundAction := c.GetString("firewall.inbound_action", "drop")
 	switch inboundAction {
 	case "reject":
-		fw.InboundSendReject = true
+		fw.InSendReject = true
 	case "drop":
-		fw.InboundSendReject = false
+		fw.InSendReject = false
 	default:
 		l.Warn("invalid firewall.inbound_action, defaulting to `drop`", "action", inboundAction)
-		fw.InboundSendReject = false
+		fw.InSendReject = false
 	}
 
 	outboundAction := c.GetString("firewall.outbound_action", "drop")
 	switch outboundAction {
 	case "reject":
-		fw.OutboundSendReject = true
+		fw.OutSendReject = true
 	case "drop":
-		fw.OutboundSendReject = false
+		fw.OutSendReject = false
 	default:
 		l.Warn("invalid firewall.outbound_action, defaulting to `drop`", "action", outboundAction)
-		fw.OutboundSendReject = false
+		fw.OutSendReject = false
 	}
 
 	err := AddFirewallRulesFromConfig(l, false, c, fw)

go.mod

@@ -1,6 +1,6 @@
 module github.com/slackhq/nebula
 
-go 1.26.0
+go 1.25.0
 
 require (
 	dario.cat/mergo v1.0.2
@@ -24,12 +24,12 @@ require (
 	github.com/vishvananda/netlink v1.3.1
 	go.uber.org/goleak v1.3.0
 	go.yaml.in/yaml/v3 v3.0.4
-	golang.org/x/crypto v0.54.0
+	golang.org/x/crypto v0.53.0
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
-	golang.org/x/net v0.57.0
+	golang.org/x/net v0.56.0
-	golang.org/x/sync v0.22.0
+	golang.org/x/sync v0.21.0
-	golang.org/x/sys v0.47.0
+	golang.org/x/sys v0.46.0
-	golang.org/x/term v0.45.0
+	golang.org/x/term v0.44.0
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
 	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
 	golang.zx2c4.com/wireguard/windows v1.0.1

go.sum

@@ -162,8 +162,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
+golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
-golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
+golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
@@ -182,8 +182,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
+golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
-golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
+golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -191,8 +191,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
+golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
-golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -208,11 +208,11 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
+golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
-golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
+golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
-golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
+golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

handshake_manager.go

@@ -295,13 +295,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 		hm.messageMetrics.Tx(header.Handshake, hh.machine.Subtype(), 1)
 		err := hm.outside.WriteTo(stage0, addr)
 		if err != nil {
-			// These repeat every attempt, so match the success log below and only shout when the remotes changed
+			hostinfo.logger(hm.l).Error("Failed to send handshake message",
-			level := slog.LevelDebug
-			if remotesHaveChanged {
-				level = slog.LevelError
-			}
-
-			hostinfo.logger(hm.l).Log(context.Background(), level, "Failed to send handshake message",
 				"udpAddr", addr,
 				"initiatorIndex", hostinfo.localIndexId,
 				"handshake", hsFields,
@@ -535,9 +529,7 @@ func (hm *HandshakeManager) DeleteHostInfo(hostinfo *HostInfo) {
 
 func (hm *HandshakeManager) unlockedDeleteHostInfo(hostinfo *HostInfo) {
 	for _, addr := range hostinfo.vpnAddrs {
-		if cur, ok := hm.vpnIps[addr]; ok && cur.hostinfo == hostinfo {
+		delete(hm.vpnIps, addr)
-			delete(hm.vpnIps, addr)
-		}
 	}
 
 	if len(hm.vpnIps) == 0 {
@@ -1085,7 +1077,7 @@ func (hm *HandshakeManager) sendHandshakeResponse(via ViaSender, msg []byte, hos
 		hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
 		// We received a valid handshake on this relay, so make sure the relay
 		// state reflects that, in case it had been marked Disestablished.
-		via.relayHI.relayState.UpdateRelayForByIdxState(via.relay.LocalIndex, Established)
+		via.relayHI.relayState.UpdateRelayForByIdxState(via.remoteIdx, Established)
 		f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false)
 		f.l.Info("Handshake message sent", append(logFields, "relay", via.relayHI.vpnAddrs[0])...)
 	}

hostmap.go

@@ -287,6 +287,7 @@ type HostInfo struct {
 type ViaSender struct {
 	UdpAddr   netip.AddrPort
 	relayHI   *HostInfo // relayHI is the host info object of the relay
+	remoteIdx uint32    // remoteIdx is the index included in the header of the received packet
 	relay     *Relay    // relay contains the rest of the relay information, including the PeerIP of the host trying to communicate with us.
 	IsRelayed bool      // IsRelayed is true if the packet was sent through a relay
 }
@@ -868,9 +869,17 @@ func (i *HostInfo) logger(l *slog.Logger) *slog.Logger {
 
 // Utility functions
 
-func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
+// localAddr is a locally discovered address candidate for lighthouse
+// advertisement, along with details about the link it was found on.
+type localAddr struct {
+	addr    netip.Addr
+	ifName  string
+	linkMTU int // MTU reported for the link, or <= 0 if unknown
+}
+
+func localAddrs(l *slog.Logger, allowList *LocalAllowList) []localAddr {
 	//FIXME: This function is pretty garbage
-	var finalAddrs []netip.Addr
+	var finalAddrs []localAddr
 	ifaces, _ := net.Interfaces()
 	for _, i := range ifaces {
 		allow := allowList.AllowName(i.Name)
@@ -915,7 +924,7 @@ func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
 					continue
 				}
 
-				finalAddrs = append(finalAddrs, addr)
+				finalAddrs = append(finalAddrs, localAddr{addr: addr, ifName: i.Name, linkMTU: i.MTU})
 			}
 		}
 	}

info_api_server.go

@@ -1,409 +0,0 @@
-package nebula
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io/fs"
-	"log/slog"
-	"net"
-	"net/http"
-	"net/netip"
-	"os"
-	"path/filepath"
-	"strconv"
-	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
-
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/config"
-)
-
-// infoAPIServer is a small http+json listener on a unix socket that lets other
-// programs on this machine resolve a vpn address to its certificate identity (name, groups, networks)
-// for making authorization decisions. Lifecycle works like statsServer: the constructor wires the
-// reload callback, reload records config, Start runs the runtime, Stop tears it down
-type infoAPIServer struct {
-	l       *slog.Logger
-	ctx     context.Context
-	hostMap *HostMap
-	pki     *PKI
-
-	// enabled mirrors `info_api.enabled` so callers of Start don't need to know the gating rules
-	enabled atomic.Bool
-
-	runMu  sync.Mutex
-	runCfg *infoAPIConfig
-	run    *infoAPIRuntime // non-nil while a runtime is live
-}
-
-// infoAPIRuntime is the live state owned by a single Start invocation. Stop and Start's exit path
-// use pointer equality to tell "my runtime" apart from one that replaced it after a reload
-type infoAPIRuntime struct {
-	server   *http.Server
-	listener net.Listener
-}
-
-// infoAPIConfig is a snapshot of the info_api config section, comparable with == so reload can
-// detect "no change" cheaply
-type infoAPIConfig struct {
-	enabled bool
-	listen  string // raw config value, for error messages
-	addr    string // unix socket path
-	// file mode applied to the unix socket after bind
-	socketMode fs.FileMode
-}
-
-// newInfoAPIServerFromConfig builds a infoAPIServer and applies the initial config. The reload
-// callback is registered first so a SIGHUP can later enable, fix, or disable the listener even if
-// the initial config was bad. Nothing binds until Start, so config tests are side effect free.
-// A bad config is logged rather than returned: it must not stop nebula from starting, the feature
-// just stays disabled until a reload provides a valid config
-func newInfoAPIServerFromConfig(ctx context.Context, l *slog.Logger, pki *PKI, hostMap *HostMap, c *config.C) *infoAPIServer {
-	h := &infoAPIServer{
-		l:       l,
-		ctx:     ctx,
-		hostMap: hostMap,
-		pki:     pki,
-	}
-
-	c.RegisterReloadCallback(func(c *config.C) {
-		if err := h.reload(c, false); err != nil {
-			h.l.Warn("Failed to reload info API from config", "error", err)
-		}
-	})
-
-	if err := h.reload(c, true); err != nil {
-		h.l.Warn("Failed to apply info API config; it will stay disabled until the config is fixed and reloaded", "error", err)
-	}
-	return h
-}
-
-// reload records the latest config. The initial call only records it, Control.Start launches the
-// first runtime via infoAPIStart. Later calls reconcile the running listener with the new config:
-// enable, disable, or restart when the listen config changed
-func (h *infoAPIServer) reload(c *config.C, initial bool) error {
-	newCfg, err := loadInfoAPIConfig(c)
-	if err != nil {
-		return err
-	}
-
-	h.runMu.Lock()
-	sameCfg := h.runCfg != nil && *h.runCfg == newCfg
-	h.runCfg = &newCfg
-	running := h.run != nil
-	h.runMu.Unlock()
-
-	h.enabled.Store(newCfg.enabled)
-
-	if initial || sameCfg {
-		return nil
-	}
-
-	if running {
-		h.Stop()
-	}
-	if newCfg.enabled {
-		go h.Start()
-	}
-	return nil
-}
-
-// Start binds the listener from the latest config and serves until Stop is called or ctx fires.
-// Safe to call when disabled or already running (both no-op)
-func (h *infoAPIServer) Start() {
-	if !h.enabled.Load() {
-		return
-	}
-
-	h.runMu.Lock()
-	if h.ctx.Err() != nil || h.run != nil || h.runCfg == nil {
-		h.runMu.Unlock()
-		return
-	}
-	cfg := *h.runCfg
-	ln, err := h.listen(cfg)
-	if err != nil {
-		// drop the cached config so a SIGHUP with the same config retries the bind
-		h.runCfg = nil
-		h.runMu.Unlock()
-		h.l.Error("Failed to start info API listener", "listen", cfg.listen, "error", err)
-		return
-	}
-
-	mux := http.NewServeMux()
-	mux.HandleFunc("GET /v1/host", h.handleHost)
-	mux.HandleFunc("GET /v1/self", h.handleSelf)
-	srv := &http.Server{Handler: mux, ReadHeaderTimeout: 5 * time.Second}
-	rt := &infoAPIRuntime{server: srv, listener: ln}
-	h.run = rt
-	h.runMu.Unlock()
-
-	h.l.Info("Starting info API listener", "addr", ln.Addr())
-	cleanExit := h.serve(srv, ln)
-
-	// A Stop that raced our bind shut the server down before Serve could adopt the listener;
-	// closing it again is harmless and guarantees a unix socket file gets unlinked
-	_ = ln.Close()
-
-	// Clear our runtime only if nothing has replaced it. Stop races through here too but leaves
-	// h.run == nil, so the pointer check skips
-	h.runMu.Lock()
-	if h.run == rt {
-		h.run = nil
-		// an error exit leaves runCfg cached as if it were applied, drop it so a SIGHUP with the
-		// same config re-triggers Start once the user fixes the underlying problem
-		if !cleanExit {
-			h.runCfg = nil
-		}
-	}
-	h.runMu.Unlock()
-}
-
-// serve runs srv.Serve and ensures ctx cancellation unblocks it. Returns true if the listener
-// exited cleanly (Stop, ctx cancellation), false on an unexpected error
-func (h *infoAPIServer) serve(srv *http.Server, ln net.Listener) bool {
-	// ctx cancellation triggers a server shutdown which in turn unblocks Serve, closing `done` on
-	// exit keeps the watcher from outliving this call
-	done := make(chan struct{})
-	go func() {
-		select {
-		case <-h.ctx.Done():
-			shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-			defer cancel()
-			if err := srv.Shutdown(shutdownCtx); err != nil {
-				h.l.Warn("Failed to shut down info API listener", "error", err)
-			}
-		case <-done:
-		}
-	}()
-	defer close(done)
-
-	err := srv.Serve(ln)
-	if err == nil || errors.Is(err, http.ErrServerClosed) {
-		return true
-	}
-	h.l.Error("Info API listener exited", "error", err)
-	return false
-}
-
-// Stop tears down the active runtime, if any. Idempotent
-func (h *infoAPIServer) Stop() {
-	h.runMu.Lock()
-	rt := h.run
-	h.run = nil
-	h.runMu.Unlock()
-	if rt == nil {
-		return
-	}
-	shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-	if err := rt.server.Shutdown(shutdownCtx); err != nil {
-		h.l.Warn("Failed to shut down info API listener", "error", err)
-	}
-}
-
-// listen binds the configured unix socket. It also clears a stale socket file left by an unclean
-// exit and applies the configured file mode
-func (h *infoAPIServer) listen(cfg infoAPIConfig) (net.Listener, error) {
-	if fi, err := os.Stat(cfg.addr); err == nil {
-		if fi.Mode()&os.ModeSocket == 0 {
-			return nil, fmt.Errorf("info_api.listen path %s exists and is not a socket, refusing to replace it", cfg.addr)
-		}
-		// a normal shutdown unlinks the socket, so a file here means a previous process exited
-		// uncleanly, remove it so the bind below can succeed
-		if err = os.Remove(cfg.addr); err != nil {
-			return nil, fmt.Errorf("failed to remove stale socket %s: %w", cfg.addr, err)
-		}
-	}
-
-	ln, err := net.Listen("unix", cfg.addr)
-	if err != nil {
-		return nil, err
-	}
-	// The socket is briefly live with umask-derived permissions before this chmod lands, tolerated
-	// because connections accepted in that window still only reach this read-only API
-	if err = os.Chmod(cfg.addr, cfg.socketMode); err != nil {
-		_ = ln.Close()
-		return nil, fmt.Errorf("failed to set mode on socket %s: %w", cfg.addr, err)
-	}
-	return ln, nil
-}
-
-func (h *infoAPIServer) certState() *CertState {
-	if h.pki == nil {
-		return nil
-	}
-	return h.pki.getCertState()
-}
-
-// handleHost serves GET /v1/host?addr=<vpn addr>, answering with the identity of the host that
-// owns the address: a peer with an active tunnel, or this node itself. addr may include a port,
-// which is ignored, so clients can pass a connection's remote address through without parsing it
-func (h *infoAPIServer) handleHost(w http.ResponseWriter, r *http.Request) {
-	q := r.URL.Query().Get("addr")
-	if q == "" {
-		writeJSONError(w, http.StatusBadRequest, "missing addr parameter")
-		return
-	}
-	ip, err := parseQueryAddrParam(q)
-	if err != nil {
-		writeJSONError(w, http.StatusBadRequest, "invalid address")
-		return
-	}
-
-	crt := findCertificateForVpnAddr(h.certState(), h.hostMap, ip)
-	if crt == nil {
-		writeJSONError(w, http.StatusNotFound, "no active tunnel for address")
-		return
-	}
-	h.writeHostIdentity(w, crt)
-}
-
-// handleSelf serves GET /v1/self, answering with this node's own identity
-func (h *infoAPIServer) handleSelf(w http.ResponseWriter, r *http.Request) {
-	var crt cert.Certificate
-	if cs := h.certState(); cs != nil {
-		crt = cs.getCertificate(cs.initiatingVersion)
-	}
-	if crt == nil {
-		writeJSONError(w, http.StatusInternalServerError, "no certificate available")
-		return
-	}
-	h.writeHostIdentity(w, crt)
-}
-
-func (h *infoAPIServer) writeHostIdentity(w http.ResponseWriter, crt cert.Certificate) {
-	id, err := newHostIdentity(crt)
-	if err != nil {
-		writeJSONError(w, http.StatusInternalServerError, "failed to fingerprint certificate")
-		return
-	}
-	w.Header().Set("Content-Type", "application/json")
-	if err = json.NewEncoder(w).Encode(id); err != nil {
-		h.l.Debug("Failed to write info API response", "error", err)
-	}
-}
-
-// findCertificateForVpnAddr answers "who owns this vpn address": ourselves (from local cert state,
-// the hostmap never carries an entry for this node) or a peer with an active tunnel. Returns nil
-// when the address is unknown or the tunnel is mid-teardown
-func findCertificateForVpnAddr(cs *CertState, hostMap *HostMap, ip netip.Addr) cert.Certificate {
-	if cs != nil && cs.myVpnAddrsTable != nil && cs.myVpnAddrsTable.Contains(ip) {
-		return cs.getCertificate(cs.initiatingVersion)
-	}
-
-	hostinfo := hostMap.QueryVpnAddr(ip)
-	if hostinfo == nil {
-		return nil
-	}
-	cc := hostinfo.GetCert()
-	if cc == nil {
-		return nil
-	}
-	return cc.Certificate
-}
-
-// hostIdentity is the json document served for both /v1/host and /v1/self, every field is derived
-// from the authenticated certificate alone
-type hostIdentity struct {
-	Name           string         `json:"name"`
-	VpnAddrs       []netip.Addr   `json:"vpnAddrs"`
-	Networks       []netip.Prefix `json:"networks"`
-	UnsafeNetworks []netip.Prefix `json:"unsafeNetworks"`
-	Groups         []string       `json:"groups"`
-	Fingerprint    string         `json:"fingerprint"`
-	Issuer         string         `json:"issuer"`
-	NotBefore      time.Time      `json:"notBefore"`
-	NotAfter       time.Time      `json:"notAfter"`
-	CertVersion    int            `json:"certVersion"`
-}
-
-func newHostIdentity(crt cert.Certificate) (hostIdentity, error) {
-	fp, err := crt.Fingerprint()
-	if err != nil {
-		return hostIdentity{}, err
-	}
-
-	// slices are always allocated so they marshal as [] rather than null
-	networks := crt.Networks()
-	id := hostIdentity{
-		Name:           crt.Name(),
-		VpnAddrs:       make([]netip.Addr, 0, len(networks)),
-		Networks:       append(make([]netip.Prefix, 0, len(networks)), networks...),
-		UnsafeNetworks: append(make([]netip.Prefix, 0, len(crt.UnsafeNetworks())), crt.UnsafeNetworks()...),
-		Groups:         append(make([]string, 0, len(crt.Groups())), crt.Groups()...),
-		Fingerprint:    fp,
-		Issuer:         crt.Issuer(),
-		NotBefore:      crt.NotBefore(),
-		NotAfter:       crt.NotAfter(),
-		CertVersion:    int(crt.Version()),
-	}
-	for _, n := range networks {
-		id.VpnAddrs = append(id.VpnAddrs, n.Addr())
-	}
-	return id, nil
-}
-
-func writeJSONError(w http.ResponseWriter, status int, msg string) {
-	w.Header().Set("Content-Type", "application/json")
-	w.WriteHeader(status)
-	_ = json.NewEncoder(w).Encode(map[string]string{"error": msg})
-}
-
-// parseQueryAddrParam parses the addr query parameter, accepting a bare address or an address with
-// a port (`192.168.100.7:54321`, `[fd00::1]:443`) so callers can pass a connection's RemoteAddr
-// straight through. The result is unmapped, 4in6 addresses (::ffff:a.b.c.d) become ipv4
-func parseQueryAddrParam(s string) (netip.Addr, error) {
-	if ip, err := netip.ParseAddr(s); err == nil {
-		return ip.Unmap(), nil
-	}
-	ap, err := netip.ParseAddrPort(s)
-	if err != nil {
-		return netip.Addr{}, err
-	}
-	return ap.Addr().Unmap(), nil
-}
-
-func loadInfoAPIConfig(c *config.C) (infoAPIConfig, error) {
-	cfg := infoAPIConfig{
-		enabled: c.GetBool("info_api.enabled", false),
-		listen:  c.GetString("info_api.listen", ""),
-	}
-	if !cfg.enabled {
-		return cfg, nil
-	}
-
-	if cfg.listen == "" {
-		return cfg, errors.New("info_api.listen can not be empty when info_api is enabled")
-	}
-	addr, err := parseInfoAPIListen(cfg.listen)
-	if err != nil {
-		return cfg, err
-	}
-	cfg.addr = addr
-
-	// read as a string so yaml can't reinterpret the octal literal
-	modeStr := c.GetString("info_api.socket_mode", "0600")
-	mode, err := strconv.ParseUint(modeStr, 8, 32)
-	if err != nil || fs.FileMode(mode)&^fs.ModePerm != 0 {
-		return cfg, fmt.Errorf("info_api.socket_mode was not a valid octal file mode: %s", modeStr)
-	}
-	cfg.socketMode = fs.FileMode(mode)
-	return cfg, nil
-}
-
-// parseInfoAPIListen extracts the unix socket path from the info_api.listen config value, which
-// must be a `unix://` URL with an absolute path, e.g. `unix:///var/run/nebula.sock`
-func parseInfoAPIListen(listen string) (addr string, err error) {
-	path, ok := strings.CutPrefix(listen, "unix://")
-	if !ok {
-		return "", fmt.Errorf("info_api.listen must be a unix:// socket path: %s", listen)
-	} else if !filepath.IsAbs(path) {
-		return "", fmt.Errorf("info_api.listen unix socket path must be absolute: %s", listen)
-	}
-	return path, nil
-}

info_api_server_test.go

@@ -1,448 +0,0 @@
-package nebula
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"io/fs"
-	"log/slog"
-	"net"
-	"net/http"
-	"net/http/httptest"
-	"net/netip"
-	"net/url"
-	"os"
-	"path/filepath"
-	"runtime"
-	"testing"
-	"time"
-
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/config"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func Test_parseInfoAPIListen(t *testing.T) {
-	type testCase struct {
-		listen  string
-		addr    string
-		wantErr bool
-	}
-	tests := []testCase{
-		{listen: "", wantErr: true},
-		{listen: "unix://", wantErr: true},
-		{listen: "unix://relative/path.sock", wantErr: true},
-		{listen: "not an address", wantErr: true},
-		// tcp host:port addresses are no longer accepted
-		{listen: "127.0.0.1:8085", wantErr: true},
-		{listen: "[::1]:8085", wantErr: true},
-		{listen: "localhost:8085", wantErr: true},
-	}
-
-	// A unix socket path must be absolute for the OS that will bind it, and filepath.IsAbs is
-	// GOOS-specific. CI runs the suite separately on each OS, so assert the platform's own native
-	// absolute path is accepted while the other platform's is rejected.
-	posixPath := "unix:///var/run/nebula.sock"
-	winPath := `unix://C:\nebula\hq.sock`
-	if runtime.GOOS == "windows" {
-		tests = append(tests,
-			testCase{listen: winPath, addr: `C:\nebula\hq.sock`},
-			testCase{listen: posixPath, wantErr: true},
-		)
-	} else {
-		tests = append(tests,
-			testCase{listen: posixPath, addr: "/var/run/nebula.sock"},
-			testCase{listen: winPath, wantErr: true},
-		)
-	}
-
-	for _, tt := range tests {
-		addr, err := parseInfoAPIListen(tt.listen)
-		if tt.wantErr {
-			require.Error(t, err, "listen=%q", tt.listen)
-			continue
-		}
-		require.NoError(t, err, "listen=%q", tt.listen)
-		assert.Equal(t, tt.addr, addr, "listen=%q", tt.listen)
-	}
-}
-
-func Test_loadInfoAPIConfig(t *testing.T) {
-	c := config.NewC(nil)
-
-	// the listen path must be absolute for the OS running the test (CI is per-OS)
-	listen, wantAddr := "unix:///tmp/hq.sock", "/tmp/hq.sock"
-	if runtime.GOOS == "windows" {
-		listen, wantAddr = `unix://C:\tmp\hq.sock`, `C:\tmp\hq.sock`
-	}
-
-	// absent section means disabled, no error
-	cfg, err := loadInfoAPIConfig(c)
-	require.NoError(t, err)
-	assert.False(t, cfg.enabled)
-
-	// enabled without a listen address is an error
-	setInfoAPIConfig(c, true, "", "")
-	_, err = loadInfoAPIConfig(c)
-	require.Error(t, err)
-
-	// a unix socket gets the default mode
-	setInfoAPIConfig(c, true, listen, "")
-	cfg, err = loadInfoAPIConfig(c)
-	require.NoError(t, err)
-	assert.Equal(t, wantAddr, cfg.addr)
-	assert.Equal(t, fs.FileMode(0o600), cfg.socketMode)
-
-	setInfoAPIConfig(c, true, listen, "0660")
-	cfg, err = loadInfoAPIConfig(c)
-	require.NoError(t, err)
-	assert.Equal(t, fs.FileMode(0o660), cfg.socketMode)
-
-	setInfoAPIConfig(c, true, listen, "withers")
-	_, err = loadInfoAPIConfig(c)
-	require.Error(t, err)
-
-	// mode bits beyond the permission bits are rejected
-	setInfoAPIConfig(c, true, listen, "10600")
-	_, err = loadInfoAPIConfig(c)
-	require.Error(t, err)
-
-	// tcp host:port listen addresses are no longer supported
-	setInfoAPIConfig(c, true, "127.0.0.1:8085", "")
-	_, err = loadInfoAPIConfig(c)
-	require.Error(t, err)
-}
-
-func TestInfoAPIServer_badConfigIsNonFatal(t *testing.T) {
-	// an enabled-but-invalid config must not stop construction; nebula keeps starting and the
-	// feature simply stays disabled until a reload supplies a valid config
-	c := config.NewC(nil)
-	setInfoAPIConfig(c, true, "not-a-unix-socket", "")
-	h := newInfoAPIServerFromConfig(context.Background(), slog.New(slog.DiscardHandler), nil, newHostMap(slog.New(slog.DiscardHandler)), c)
-	require.NotNil(t, h)
-	assert.False(t, h.enabled.Load())
-
-	// no config was recorded, so Start has nothing to bind and is a no-op
-	h.runMu.Lock()
-	assert.Nil(t, h.runCfg)
-	h.runMu.Unlock()
-	h.Start()
-	h.runMu.Lock()
-	assert.Nil(t, h.run)
-	h.runMu.Unlock()
-}
-
-func setInfoAPIConfig(c *config.C, enabled bool, listen, socketMode string) {
-	settings := map[string]any{
-		"enabled": enabled,
-		"listen":  listen,
-	}
-	if socketMode != "" {
-		settings["socket_mode"] = socketMode
-	}
-	c.Settings["info_api"] = settings
-}
-
-func newTestInfoAPIServer(t *testing.T) (*infoAPIServer, *config.C) {
-	t.Helper()
-	h := &infoAPIServer{
-		l:       slog.New(slog.DiscardHandler),
-		ctx:     context.Background(),
-		hostMap: newHostMap(slog.New(slog.DiscardHandler)),
-	}
-	h.hostMap.preferredRanges.Store(&[]netip.Prefix{})
-	return h, config.NewC(nil)
-}
-
-// addTestPeer creates a certificate for a peer owning each addr (as a /24 or /64) and inserts it
-// into the hostmap as an established tunnel
-func addTestPeer(t *testing.T, hm *HostMap, name string, addrs []netip.Addr, unsafeNetworks []netip.Prefix, groups []string) cert.Certificate {
-	t.Helper()
-	networks := make([]netip.Prefix, 0, len(addrs))
-	for _, a := range addrs {
-		bits := 24
-		if a.Is6() {
-			bits = 64
-		}
-		networks = append(networks, netip.PrefixFrom(a, bits))
-	}
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
-	crt, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, name, time.Time{}, time.Time{}, networks, unsafeNetworks, groups)
-	fp, err := crt.Fingerprint()
-	require.NoError(t, err)
-
-	hm.unlockedAddHostInfo(&HostInfo{
-		ConnectionState: &ConnectionState{
-			peerCert: &cert.CachedCertificate{Certificate: crt, Fingerprint: fp},
-		},
-		vpnAddrs: addrs,
-		relayState: RelayState{
-			relayForByAddr: map[netip.Addr]*Relay{},
-			relayForByIdx:  map[uint32]*Relay{},
-		},
-	}, &Interface{})
-	return crt
-}
-
-func getHost(t *testing.T, h *infoAPIServer, addrParam string) (int, map[string]any) {
-	t.Helper()
-	r := httptest.NewRequest(http.MethodGet, "/v1/host?addr="+url.QueryEscape(addrParam), nil)
-	w := httptest.NewRecorder()
-	h.handleHost(w, r)
-	return decodeResponse(t, w)
-}
-
-func decodeResponse(t *testing.T, w *httptest.ResponseRecorder) (int, map[string]any) {
-	t.Helper()
-	assert.Equal(t, "application/json", w.Header().Get("Content-Type"))
-	var body map[string]any
-	require.NoError(t, json.Unmarshal(w.Body.Bytes(), &body))
-	return w.Code, body
-}
-
-func TestInfoAPIServer_handleHost(t *testing.T) {
-	h, _ := newTestInfoAPIServer(t)
-	h.pki = newTestPKI(t, "self", []netip.Addr{netip.MustParseAddr("10.0.0.1")})
-
-	peerV4 := netip.MustParseAddr("10.0.0.99")
-	peerV6 := netip.MustParseAddr("fd00::99")
-	addTestPeer(t, h.hostMap, "laptop-alice", []netip.Addr{peerV4, peerV6},
-		[]netip.Prefix{netip.MustParsePrefix("192.168.50.0/24")}, []string{"eng", "ssh"})
-	addTestPeer(t, h.hostMap, "groupless", []netip.Addr{netip.MustParseAddr("10.0.0.77")}, nil, nil)
-
-	// an established peer comes back with its full identity
-	code, body := getHost(t, h, "10.0.0.99")
-	require.Equal(t, http.StatusOK, code)
-	assert.Equal(t, "laptop-alice", body["name"])
-	assert.Equal(t, []any{"10.0.0.99", "fd00::99"}, body["vpnAddrs"])
-	assert.Equal(t, []any{"10.0.0.99/24", "fd00::99/64"}, body["networks"])
-	assert.Equal(t, []any{"192.168.50.0/24"}, body["unsafeNetworks"])
-	assert.Equal(t, []any{"eng", "ssh"}, body["groups"])
-	assert.NotEmpty(t, body["fingerprint"])
-	assert.Equal(t, "2", fmt.Sprintf("%v", body["certVersion"]))
-	assert.NotEmpty(t, body["notBefore"])
-	assert.NotEmpty(t, body["notAfter"])
-
-	// empty cert slices marshal as [] rather than null
-	code, body = getHost(t, h, "10.0.0.77")
-	require.Equal(t, http.StatusOK, code)
-	require.NotNil(t, body["groups"])
-	assert.Empty(t, body["groups"])
-	require.NotNil(t, body["unsafeNetworks"])
-	assert.Empty(t, body["unsafeNetworks"])
-
-	// a port in addr is ignored so RemoteAddr can be passed through directly, including the
-	// bracketed v6 and 4in6 forms
-	for _, q := range []string{"10.0.0.99:54321", "[fd00::99]:443", "::ffff:10.0.0.99"} {
-		code, body = getHost(t, h, q)
-		require.Equal(t, http.StatusOK, code, "addr=%q", q)
-		assert.Equal(t, "laptop-alice", body["name"], "addr=%q", q)
-	}
-
-	// our own address answers from the local cert state
-	code, body = getHost(t, h, "10.0.0.1")
-	require.Equal(t, http.StatusOK, code)
-	assert.Equal(t, "self", body["name"])
-
-	code, body = getHost(t, h, "10.0.0.42")
-	assert.Equal(t, http.StatusNotFound, code)
-	assert.NotEmpty(t, body["error"])
-
-	// a tunnel mid-teardown (no peer cert) is treated as unknown
-	h.hostMap.unlockedAddHostInfo(&HostInfo{
-		ConnectionState: &ConnectionState{},
-		vpnAddrs:        []netip.Addr{netip.MustParseAddr("10.0.0.66")},
-		relayState: RelayState{
-			relayForByAddr: map[netip.Addr]*Relay{},
-			relayForByIdx:  map[uint32]*Relay{},
-		},
-	}, &Interface{})
-	code, _ = getHost(t, h, "10.0.0.66")
-	assert.Equal(t, http.StatusNotFound, code)
-
-	code, body = getHost(t, h, "not-an-address")
-	assert.Equal(t, http.StatusBadRequest, code)
-	assert.NotEmpty(t, body["error"])
-
-	r := httptest.NewRequest(http.MethodGet, "/v1/host", nil)
-	w := httptest.NewRecorder()
-	h.handleHost(w, r)
-	code, body = decodeResponse(t, w)
-	assert.Equal(t, http.StatusBadRequest, code)
-	assert.NotEmpty(t, body["error"])
-}
-
-func TestInfoAPIServer_handleSelf(t *testing.T) {
-	h, _ := newTestInfoAPIServer(t)
-	h.pki = newTestPKI(t, "lighthouse", []netip.Addr{netip.MustParseAddr("10.0.0.1")})
-
-	r := httptest.NewRequest(http.MethodGet, "/v1/self", nil)
-	w := httptest.NewRecorder()
-	h.handleSelf(w, r)
-	code, body := decodeResponse(t, w)
-	require.Equal(t, http.StatusOK, code)
-	assert.Equal(t, "lighthouse", body["name"])
-	assert.Equal(t, []any{"10.0.0.1"}, body["vpnAddrs"])
-
-	// no cert state available should be an error, not a panic
-	h.pki = nil
-	w = httptest.NewRecorder()
-	h.handleSelf(w, r)
-	code, body = decodeResponse(t, w)
-	assert.Equal(t, http.StatusInternalServerError, code)
-	assert.NotEmpty(t, body["error"])
-}
-
-func unixHTTPClient(path string) *http.Client {
-	return &http.Client{
-		Timeout: time.Second,
-		Transport: &http.Transport{
-			DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) {
-				return (&net.Dialer{}).DialContext(ctx, "unix", path)
-			},
-		},
-	}
-}
-
-// waitForServe polls until a GET /v1/self through client succeeds
-func waitForServe(t *testing.T, client *http.Client) {
-	t.Helper()
-	waitFor(t, func() bool {
-		resp, err := client.Get("http://hostquery/v1/self")
-		if err != nil {
-			return false
-		}
-		resp.Body.Close()
-		return resp.StatusCode == http.StatusOK
-	})
-}
-
-func skipIfNoUnixSockets(t *testing.T) {
-	t.Helper()
-	if runtime.GOOS == "windows" {
-		t.Skip("unix socket tests are not supported on windows CI")
-	}
-}
-
-func TestInfoAPIServer_unixLifecycle(t *testing.T) {
-	skipIfNoUnixSockets(t)
-	h, c := newTestInfoAPIServer(t)
-	h.pki = newTestPKI(t, "self", []netip.Addr{netip.MustParseAddr("10.0.0.1")})
-
-	sock := filepath.Join(t.TempDir(), "hq.sock")
-	setInfoAPIConfig(c, true, "unix://"+sock, "")
-	require.NoError(t, h.reload(c, true))
-
-	done := make(chan struct{})
-	go func() {
-		h.Start()
-		close(done)
-	}()
-
-	client := unixHTTPClient(sock)
-	waitForServe(t, client)
-
-	fi, err := os.Stat(sock)
-	require.NoError(t, err)
-	assert.Equal(t, fs.FileMode(0o600), fi.Mode().Perm())
-
-	resp, err := client.Get("http://hostquery/v1/host?addr=10.0.0.1")
-	require.NoError(t, err)
-	resp.Body.Close()
-	assert.Equal(t, http.StatusOK, resp.StatusCode)
-
-	h.Stop()
-	select {
-	case <-done:
-	case <-time.After(5 * time.Second):
-		t.Fatal("Start did not return after Stop")
-	}
-	_, err = os.Stat(sock)
-	assert.True(t, os.IsNotExist(err), "socket file should be unlinked on shutdown")
-}
-
-func TestInfoAPIServer_staleSocket(t *testing.T) {
-	skipIfNoUnixSockets(t)
-	h, _ := newTestInfoAPIServer(t)
-	sock := filepath.Join(t.TempDir(), "hq.sock")
-
-	// simulate an unclean exit, a leftover socket file with no listener
-	stale, err := net.ListenUnix("unix", &net.UnixAddr{Name: sock, Net: "unix"})
-	require.NoError(t, err)
-	stale.SetUnlinkOnClose(false)
-	require.NoError(t, stale.Close())
-	_, err = os.Stat(sock)
-	require.NoError(t, err, "stale socket file should exist")
-
-	cfg := infoAPIConfig{addr: sock, socketMode: 0o600}
-	ln, err := h.listen(cfg)
-	require.NoError(t, err, "a stale socket should be removed and rebound")
-	require.NoError(t, ln.Close())
-}
-
-func TestInfoAPIServer_existingFileNotReplaced(t *testing.T) {
-	skipIfNoUnixSockets(t)
-	h, _ := newTestInfoAPIServer(t)
-	path := filepath.Join(t.TempDir(), "hq.sock")
-	require.NoError(t, os.WriteFile(path, []byte("precious"), 0o600))
-
-	cfg := infoAPIConfig{addr: path, socketMode: 0o600}
-	_, err := h.listen(cfg)
-	require.Error(t, err, "a non-socket file at the listen path must not be replaced")
-
-	content, err := os.ReadFile(path)
-	require.NoError(t, err)
-	assert.Equal(t, "precious", string(content))
-}
-
-func TestInfoAPIServer_reload(t *testing.T) {
-	skipIfNoUnixSockets(t)
-	h, c := newTestInfoAPIServer(t)
-	h.pki = newTestPKI(t, "self", []netip.Addr{netip.MustParseAddr("10.0.0.1")})
-	dir := t.TempDir()
-	sock1 := filepath.Join(dir, "hq1.sock")
-	sock2 := filepath.Join(dir, "hq2.sock")
-
-	// initial reload only records config, Control.Start is what launches the runtime
-	setInfoAPIConfig(c, false, "unix://"+sock1, "")
-	require.NoError(t, h.reload(c, true))
-	assert.False(t, h.enabled.Load())
-	h.runMu.Lock()
-	assert.Nil(t, h.run)
-	h.runMu.Unlock()
-
-	// enabling via reload spawns the listener
-	setInfoAPIConfig(c, true, "unix://"+sock1, "")
-	require.NoError(t, h.reload(c, false))
-	waitForServe(t, unixHTTPClient(sock1))
-
-	// changing the listen path restarts on the new address
-	setInfoAPIConfig(c, true, "unix://"+sock2, "")
-	require.NoError(t, h.reload(c, false))
-	waitForServe(t, unixHTTPClient(sock2))
-	waitFor(t, func() bool {
-		_, err := os.Stat(sock1)
-		return os.IsNotExist(err)
-	})
-
-	// reloading an unchanged config does not restart the runtime
-	h.runMu.Lock()
-	rt := h.run
-	h.runMu.Unlock()
-	require.NoError(t, h.reload(c, false))
-	h.runMu.Lock()
-	assert.Same(t, rt, h.run)
-	h.runMu.Unlock()
-
-	// disabling stops the listener
-	setInfoAPIConfig(c, false, "unix://"+sock2, "")
-	require.NoError(t, h.reload(c, false))
-	assert.False(t, h.enabled.Load())
-	waitFor(t, func() bool {
-		h.runMu.Lock()
-		defer h.runMu.Unlock()
-		return h.run == nil
-	})
-}

inside.go

@@ -87,7 +87,7 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 }
 
 func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
-	if !f.firewall.OutboundSendReject {
+	if !f.firewall.InSendReject {
 		return
 	}
 
@@ -103,7 +103,7 @@ func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
 }
 
 func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *HostInfo, nb, out []byte, q int) {
-	if !f.firewall.InboundSendReject {
+	if !f.firewall.OutSendReject {
 		return
 	}
 
@@ -408,7 +408,7 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 		if err != nil {
 			hostinfo.logger(f.l).Error("Failed to write outgoing packet",
 				"error", err,
-				"udpAddr", hr,
+				"udpAddr", remote,
 			)
 		}
 	} else {

lighthouse.go

@@ -19,8 +19,11 @@ import (
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/logging"
+	"github.com/slackhq/nebula/overlay"
 	"github.com/slackhq/nebula/udp"
 	"github.com/slackhq/nebula/util"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
 )
 
 var ErrHostNotKnown = errors.New("host not known")
@@ -36,10 +39,6 @@ type LightHouse struct {
 	myVpnNetworksTable *bart.Lite
 	punchy             *Punchy
 
-	// localAddrsFn enumerates the underlay addresses we advertise. It is a field so tests can supply simulated
-	// addresses rather than whatever this machine's NICs happen to be. Set it before Start.
-	localAddrsFn func(*LocalAllowList) []netip.Addr
-
 	// Local cache of answers from light houses
 	// map of vpn addr to answers
 	addrMap map[netip.Addr]*RemoteList
@@ -69,6 +68,18 @@ type LightHouse struct {
 
 	advertiseAddrs atomic.Pointer[[]netip.AddrPort]
 
+	// tunMTU mirrors tun.mtu so locally discovered addrs can be checked for
+	// links too small to carry a full-size nebula packet without fragmenting.
+	tunMTU atomic.Int64
+	// omitLowMTUAddrs drops such addrs from lighthouse updates (and demotes
+	// the associated warnings to debug logs) instead of advertising them.
+	omitLowMTUAddrs atomic.Bool
+	// mtuWarned tracks the last classification logged per local addr so a
+	// warning is only emitted when the classification changes, not on every
+	// periodic update.
+	mtuWarnLock sync.Mutex
+	mtuWarned   map[mtuWarnKey]linkMTUTier
+
 	// Addr's of relays that can be used by peers to access me
 	relaysForMe atomic.Pointer[[]netip.Addr]
 
@@ -109,12 +120,9 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
 		punchy:             p,
 		updateTrigger:      make(chan struct{}, 1),
 		queryChan:          make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)),
+		mtuWarned:          make(map[mtuWarnKey]linkMTUTier),
 		l:                  l,
 	}
-	h.localAddrsFn = func(al *LocalAllowList) []netip.Addr {
-		return localAddrs(h.l, al)
-	}
-
 	lighthouses := make([]netip.Addr, 0)
 	h.lighthouses.Store(&lighthouses)
 	staticList := make(map[netip.Addr]struct{})
@@ -224,6 +232,23 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
 		}
 	}
 
+	if initial || c.HasChanged("tun.mtu") || c.HasChanged("lighthouse.omit_low_mtu_addrs") {
+		lh.tunMTU.Store(int64(c.GetInt("tun.mtu", overlay.DefaultMTU)))
+		lh.omitLowMTUAddrs.Store(c.GetBool("lighthouse.omit_low_mtu_addrs", false))
+
+		// Re-log any addrs whose links are still too small under the new values
+		lh.mtuWarnLock.Lock()
+		clear(lh.mtuWarned)
+		lh.mtuWarnLock.Unlock()
+
+		if !initial {
+			lh.l.Info("tun.mtu and/or lighthouse.omit_low_mtu_addrs has changed",
+				"tunMTU", lh.tunMTU.Load(),
+				"omitLowMTUAddrs", lh.omitLowMTUAddrs.Load(),
+			)
+		}
+	}
+
 	if initial || c.HasChanged("lighthouse.interval") {
 		lh.interval.Store(int64(c.GetInt("lighthouse.interval", 10)))
 
@@ -913,6 +938,108 @@ func (lh *LightHouse) TriggerUpdate() {
 	}
 }
 
+// linkMTUTier classifies how well a local addr's link MTU can carry
+// full-size nebula packets built from a tun packet of tun.mtu bytes.
+type linkMTUTier uint8
+
+// mtuWarnKey identifies a local addr for MTU warning dedup purposes. The
+// interface name is included because the same addr can exist on multiple
+// links with different MTUs.
+type mtuWarnKey struct {
+	ifName string
+	addr   netip.Addr
+}
+
+const (
+	// The link can carry both normal and relayed nebula traffic
+	linkMTUOk linkMTUTier = iota
+	// The link can carry normal nebula traffic, but relayed traffic (which
+	// adds a second layer of encapsulation) will not fit
+	linkMTUTooSmallForRelay
+	// Even normal nebula traffic will not fit
+	linkMTUTooSmall
+)
+
+const (
+	// Both AES-256-GCM and ChaCha20-Poly1305 append a 16 byte AEAD tag
+	cipherTagLen = 16
+	udpHeaderLen = 8
+)
+
+// requiredLinkMTU returns the minimum underlay link MTU that can carry a
+// full-size tun packet to an addr of the given family without fragmentation,
+// both directly and via a relay (which wraps the packet in a second nebula
+// header and AEAD tag).
+func requiredLinkMTU(tunMTU int, is4 bool) (direct, relayed int) {
+	ipHeaderLen := ipv6.HeaderLen
+	if is4 {
+		ipHeaderLen = ipv4.HeaderLen
+	}
+
+	direct = tunMTU + header.Len + cipherTagLen + udpHeaderLen + ipHeaderLen
+	relayed = direct + header.Len + cipherTagLen
+	return direct, relayed
+}
+
+// checkLocalLinkMTU classifies e's link MTU, logs when the classification
+// changes, and reports whether e should be advertised to lighthouses.
+func (lh *LightHouse) checkLocalLinkMTU(e localAddr) bool {
+	tunMTU := int(lh.tunMTU.Load())
+	omit := lh.omitLowMTUAddrs.Load()
+
+	tier := linkMTUOk
+	direct, relayed := requiredLinkMTU(tunMTU, e.addr.Is4())
+	if e.linkMTU > 0 { // links with an unknown MTU are advertised as-is
+		if e.linkMTU < direct {
+			tier = linkMTUTooSmall
+		} else if e.linkMTU < relayed {
+			tier = linkMTUTooSmallForRelay
+		}
+	}
+	advertise := tier != linkMTUTooSmall || !omit
+
+	key := mtuWarnKey{ifName: e.ifName, addr: e.addr}
+	lh.mtuWarnLock.Lock()
+	changed := lh.mtuWarned[key] != tier
+	if changed {
+		if tier == linkMTUOk {
+			delete(lh.mtuWarned, key)
+		} else {
+			lh.mtuWarned[key] = tier
+		}
+	}
+	lh.mtuWarnLock.Unlock()
+
+	if !changed || tier == linkMTUOk {
+		return advertise
+	}
+
+	level := slog.LevelWarn
+	if omit {
+		level = slog.LevelDebug
+	}
+
+	if lh.l.Enabled(context.Background(), level) {
+		msg := "Link MTU too small for nebula traffic, expect fragmentation or drops"
+		if !advertise {
+			msg = "Omitting addr with too-small link MTU from lighthouse report"
+		} else if tier == linkMTUTooSmallForRelay {
+			msg = "Link MTU too small for relayed nebula traffic"
+		}
+
+		lh.l.Log(context.Background(), level, msg,
+			"localAddr", e.addr,
+			"interface", e.ifName,
+			"linkMTU", e.linkMTU,
+			"requiredMTU", direct,
+			"requiredRelayMTU", relayed,
+			"tunMTU", tunMTU,
+		)
+	}
+
+	return advertise
+}
+
 func (lh *LightHouse) SendUpdate() {
 	var v4 []*V4AddrPort
 	var v6 []*V6AddrPort
@@ -926,16 +1053,20 @@ func (lh *LightHouse) SendUpdate() {
 	}
 
 	lal := lh.GetLocalAllowList()
-	for _, e := range lh.localAddrsFn(lal) {
+	for _, e := range localAddrs(lh.l, lal) {
-		if lh.myVpnNetworksTable.Contains(e) {
+		if lh.myVpnNetworksTable.Contains(e.addr) {
+			continue
+		}
+
+		if !lh.checkLocalLinkMTU(e) {
 			continue
 		}
 
 		// Only add addrs that aren't my VPN/tun networks
-		if e.Is4() {
+		if e.addr.Is4() {
-			v4 = append(v4, netAddrToProtoV4AddrPort(e, uint16(lh.nebulaPort)))
+			v4 = append(v4, netAddrToProtoV4AddrPort(e.addr, uint16(lh.nebulaPort)))
 		} else {
-			v6 = append(v6, netAddrToProtoV6AddrPort(e, uint16(lh.nebulaPort)))
+			v6 = append(v6, netAddrToProtoV6AddrPort(e.addr, uint16(lh.nebulaPort)))
 		}
 	}
 

lighthouse_test.go

@@ -738,3 +738,86 @@ func TestLighthouse_DeletesWork(t *testing.T) {
 	out = lh.Query(testHost)
 	assert.Nil(t, out)
 }
+
+func Test_requiredLinkMTU(t *testing.T) {
+	// tun packet + nebula header (16) + AEAD tag (16) + udp (8) + ip header
+	direct, relayed := requiredLinkMTU(1300, true)
+	assert.Equal(t, 1360, direct)
+	assert.Equal(t, 1392, relayed)
+
+	direct, relayed = requiredLinkMTU(1300, false)
+	assert.Equal(t, 1380, direct)
+	assert.Equal(t, 1412, relayed)
+}
+
+func Test_checkLocalLinkMTU(t *testing.T) {
+	lh := &LightHouse{l: test.NewLogger(), mtuWarned: make(map[mtuWarnKey]linkMTUTier)}
+	lh.tunMTU.Store(1300)
+
+	v4 := netip.MustParseAddr("192.168.1.2")
+	v6 := netip.MustParseAddr("fd00::2")
+	mkAddr := func(a netip.Addr, mtu int) localAddr {
+		return localAddr{addr: a, ifName: "test0", linkMTU: mtu}
+	}
+
+	// Plenty of room, no state recorded
+	assert.True(t, lh.checkLocalLinkMTU(mkAddr(v4, 1500)))
+	assert.Empty(t, lh.mtuWarned)
+
+	// Unknown link MTU is not classified
+	assert.True(t, lh.checkLocalLinkMTU(mkAddr(v4, 0)))
+	assert.Empty(t, lh.mtuWarned)
+
+	// Too small for even normal traffic, still advertised by default
+	assert.True(t, lh.checkLocalLinkMTU(mkAddr(v4, 1359)))
+	assert.Equal(t, linkMTUTooSmall, lh.mtuWarned[mtuWarnKey{ifName: "test0", addr: v4}])
+
+	// Fits normal traffic but not relayed traffic
+	assert.True(t, lh.checkLocalLinkMTU(mkAddr(v4, 1360)))
+	assert.Equal(t, linkMTUTooSmallForRelay, lh.mtuWarned[mtuWarnKey{ifName: "test0", addr: v4}])
+
+	// Exactly enough for relayed traffic clears the state
+	assert.True(t, lh.checkLocalLinkMTU(mkAddr(v4, 1392)))
+	assert.Empty(t, lh.mtuWarned)
+
+	// v6 addrs need 20 more bytes of headroom
+	assert.True(t, lh.checkLocalLinkMTU(mkAddr(v6, 1380)))
+	assert.Equal(t, linkMTUTooSmallForRelay, lh.mtuWarned[mtuWarnKey{ifName: "test0", addr: v6}])
+	assert.True(t, lh.checkLocalLinkMTU(mkAddr(v6, 1379)))
+	assert.Equal(t, linkMTUTooSmall, lh.mtuWarned[mtuWarnKey{ifName: "test0", addr: v6}])
+	assert.True(t, lh.checkLocalLinkMTU(mkAddr(v6, 1412)))
+	assert.Empty(t, lh.mtuWarned)
+
+	// With omit enabled, only addrs that can't fit normal traffic are dropped
+	lh.omitLowMTUAddrs.Store(true)
+	assert.False(t, lh.checkLocalLinkMTU(mkAddr(v4, 1359)))
+	assert.Equal(t, linkMTUTooSmall, lh.mtuWarned[mtuWarnKey{ifName: "test0", addr: v4}])
+	assert.True(t, lh.checkLocalLinkMTU(mkAddr(v4, 1360)))
+	assert.Equal(t, linkMTUTooSmallForRelay, lh.mtuWarned[mtuWarnKey{ifName: "test0", addr: v4}])
+	assert.False(t, lh.checkLocalLinkMTU(mkAddr(v4, 1359)))
+}
+
+func Test_lighthouseMTUConfig(t *testing.T) {
+	l := test.NewLogger()
+	myVpnNet := netip.MustParsePrefix("10.128.0.1/16")
+	nt := new(bart.Lite)
+	nt.Insert(myVpnNet)
+	cs := &CertState{
+		myVpnNetworks:      []netip.Prefix{myVpnNet},
+		myVpnNetworksTable: nt,
+	}
+
+	c := config.NewC(l)
+	lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
+	require.NoError(t, err)
+	assert.Equal(t, int64(1300), lh.tunMTU.Load())
+	assert.False(t, lh.omitLowMTUAddrs.Load())
+
+	c = config.NewC(l)
+	c.Settings["tun"] = map[string]any{"mtu": 8000}
+	c.Settings["lighthouse"] = map[string]any{"omit_low_mtu_addrs": true}
+	lh, err = NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
+	require.NoError(t, err)
+	assert.Equal(t, int64(8000), lh.tunMTU.Load())
+	assert.True(t, lh.omitLowMTUAddrs.Load())
+}

main.go

@@ -260,8 +260,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		return nil, util.ContextualizeIfNeeded("Failed to start stats emitter", err)
 	}
 
-	infoAPI := newInfoAPIServerFromConfig(ctx, l, pki, hostMap, c)
-
 	if configTest {
 		return nil, nil
 	}
@@ -270,8 +268,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 
 	attachCommands(l, c, ssh, ifce)
 
-	networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c)
-
 	return &Control{
 		state:                  StateReady,
 		f:                      ifce,
@@ -281,9 +277,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		sshStart:               sshStart,
 		statsStart:             stats.Start,
 		dnsStart:               ds.Start,
-		infoAPIStart:           infoAPI.Start,
 		lighthouseStart:        lightHouse.StartUpdateWorker,
-		networkChangeStart:     networkChanges.Start,
 		connectionManagerStart: connManager.Start,
 	}, nil
 }

outside.go

@@ -102,31 +102,27 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 		return
 	}
 
-	if len(packet) < header.Len+hostinfo.ConnectionState.dKey.Overhead() {
-		f.messageMetrics.RxInvalid(1)
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			f.l.Debug("packet too small", "from", via, "length", len(packet))
-		}
-		return
-	}
-
 	// All remaining packets are encrypted
-	if isMessageRelay {
+	ci := hostinfo.ConnectionState
-		// Relay packets are special, this branch should always early-return
+	if !ci.window.Check(f.l, h.MessageCounter) {
-		if err = hostinfo.ConnectionState.VerifyRelay(f.l, h.MessageCounter, packet, nb); err != nil {
-			if f.l.Enabled(context.Background(), slog.LevelDebug) {
-				hostinfo.logger(f.l).Debug("Failed to verify relay packet", "error", err, "from", via, "header", h)
-			}
-			return
-		}
-		f.handleOutsideRelayPacket(hostinfo, via, out, packet, h, fwPacket, lhf, nb, q, localCache)
 		return
 	}
 
-	out, err = hostinfo.ConnectionState.Decrypt(f.l, h.MessageCounter, out, packet, nb)
+	// Relay packets are special
+	if isMessageRelay {
+		f.handleOutsideRelayPacket(hostinfo, via, out, packet, h, fwPacket, lhf, nb, q, localCache)
+
+		return
+	}
+
+	out, err = f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 	if err != nil {
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			hostinfo.logger(f.l).Debug("Failed to decrypt packet", "error", err, "from", via, "header", h)
+			hostinfo.logger(f.l).Debug("Failed to decrypt packet",
+				"error", err,
+				"from", via,
+				"header", h,
+			)
 		}
 		return
 	}
@@ -155,7 +151,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 			// No-op, useful for the Roaming and connectionManager side-effects above
 		case header.TestRequest:
 			//recycle the input packet ciphertext as our output buffer
-			f.send(header.Test, header.TestReply, hostinfo.ConnectionState, hostinfo, out, nb, packet)
+			f.send(header.Test, header.TestReply, ci, hostinfo, out, nb, packet)
 		default:
 			hostinfo.logger(f.l).Error("IsValidSubType was true, but unexpected test subtype seen", "from", via, "header", h)
 			return
@@ -174,8 +170,27 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 }
 
 func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
-	// Successfully validated the thing. Get rid of the Relay header and the AEAD tag
+	// The entire body is sent as AD, not encrypted.
-	signedPayload := packet[header.Len : len(packet)-hostinfo.ConnectionState.dKey.Overhead()]
+	// The packet consists of a 16-byte parsed Nebula header, Associated Data-protected payload, and a trailing 16-byte AEAD signature value.
+	// The packet is guaranteed to be at least 16 bytes at this point, b/c it got past the h.Parse() call above. If it's
+	// otherwise malformed (meaning, there is no trailing 16 byte AEAD value), then this will result in at worst a 0-length slice
+	// which will gracefully fail in the DecryptDanger call.
+	signedPayload := packet[:len(packet)-hostinfo.ConnectionState.dKey.Overhead()]
+	signatureValue := packet[len(packet)-hostinfo.ConnectionState.dKey.Overhead():]
+	var err error
+	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, signedPayload, signatureValue, h.MessageCounter, nb)
+	if err != nil {
+		return
+	}
+	// Advance the replay window now that the frame is authenticated
+	if !hostinfo.ConnectionState.window.Update(f.l, h.MessageCounter) {
+		if f.l.Enabled(context.Background(), slog.LevelDebug) {
+			hostinfo.logger(f.l).Debug("dropping out of window relay packet", "header", h)
+		}
+		return
+	}
+	// Successfully validated the thing. Get rid of the Relay header.
+	signedPayload = signedPayload[header.Len:]
 	// Pull the Roaming parts up here, and return in all call paths.
 	f.handleHostRoaming(hostinfo, via)
 	// Track usage of both the HostInfo and the Relay for the received & authenticated packet
@@ -199,6 +214,7 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
 		via = ViaSender{
 			UdpAddr:   via.UdpAddr,
 			relayHI:   hostinfo,
+			remoteIdx: relay.RemoteIndex,
 			relay:     relay,
 			IsRelayed: true,
 		}
@@ -219,10 +235,9 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
 		if targetRelay.State == Established {
 			switch targetRelay.Type {
 			case ForwardingType:
-				// Forward this packet through the relay tunnel, rebuilding it in place.
+				// Forward this packet through the relay tunnel
-				// Encode overwrites the old outer header, and the new AEAD tag lands where the old one was
+				// Find the target HostInfo
-				fwdBuf := packet[:0:len(packet)] // Cap to len(packet) to protect memory from a larger parent buffer
+				f.SendVia(targetHI, targetRelay, signedPayload, nb, out, false)
-				f.SendVia(targetHI, targetRelay, signedPayload, nb, fwdBuf, true)
 			case TerminalType:
 				hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
 				return
@@ -489,6 +504,20 @@ func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
 	return nil
 }
 
+func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, out []byte, packet []byte, h *header.H, nb []byte) ([]byte, error) {
+	var err error
+	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], mc, nb)
+	if err != nil {
+		return nil, err
+	}
+
+	if !hostinfo.ConnectionState.window.Update(f.l, mc) {
+		return nil, ErrOutOfWindow
+	}
+
+	return out, nil
+}
+
 func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache) {
 	err := newPacket(out, true, fwPacket)
 	if err != nil {

udp/netchange.go

@@ -1,61 +0,0 @@
-package udp
-
-import (
-	"context"
-	"log/slog"
-
-	"github.com/slackhq/nebula/config"
-)
-
-// NetworkChangeMonitor rebinds the udp listener when the local network moves out from under it.
-//
-// Detection lives here in the udp package, next to the socket it concerns and the platform matrix that already knows
-// which sockets go stale. What to do about a change — updating the lighthouse, requerying tunnels — is not the udp
-// package's business, so Start takes the reaction as a plain function. Passing it at Start rather than holding it
-// keeps this package from referencing whatever owns the rebind.
-//
-// On platforms whose sockets do not go stale, watchNetworkChanges hands back a nil channel and Start returns.
-type NetworkChangeMonitor struct {
-	l       *slog.Logger
-	ctx     context.Context
-	enabled bool
-}
-
-// NewNetworkChangeMonitor builds a monitor for local network changes. The returned monitor is always usable: Start
-// is safe to call unconditionally, it no-ops when disabled or on a platform that does not need it.
-func NewNetworkChangeMonitor(ctx context.Context, l *slog.Logger, c *config.C) *NetworkChangeMonitor {
-	return &NetworkChangeMonitor{
-		l:       l,
-		ctx:     ctx,
-		enabled: c.GetBool("listen.rebind_on_network_change", true),
-	}
-}
-
-// Start watches for network changes until the context is cancelled, calling rebind once per settled change. It
-// blocks, so callers run it in a goroutine, and it no-ops when disabled, unsupported, or with nothing to rebind.
-func (m *NetworkChangeMonitor) Start(rebind func()) {
-	if !m.enabled || rebind == nil || m.ctx.Err() != nil {
-		return
-	}
-
-	changes, err := watchNetworkChanges(m.ctx, m.l)
-	if err != nil {
-		// Not fatal. Everything else still works, we just won't notice a network change on our own.
-		m.l.Error("Failed to watch for network changes, will not rebind the udp listener when the network moves",
-			"error", err,
-		)
-		return
-	}
-
-	if changes == nil {
-		// This platform's sockets don't go stale, so there is nothing to watch for.
-		return
-	}
-
-	m.l.Info("Watching for network changes to rebind the udp listener")
-
-	for range changes {
-		m.l.Info("Local network changed, rebinding the udp listener")
-		rebind()
-	}
-}

udp/netchange_darwin.go

@@ -1,164 +0,0 @@
-//go:build darwin && !ios && !e2e_testing
-// +build darwin,!ios,!e2e_testing
-
-package udp
-
-import (
-	"context"
-	"encoding/binary"
-	"errors"
-	"log/slog"
-	"os"
-	"time"
-
-	"golang.org/x/sys/unix"
-)
-
-const (
-	// netChangeSettleWindow is how long we keep swallowing routing messages after the first interesting one. A
-	// single network change is never a single message, it is a burst: the link drops, addresses go away, new ones
-	// arrive, routes get rewritten. Reporting part way through that just means reporting again.
-	netChangeSettleWindow = time.Second
-
-	// netChangeReadBuffer is sized well past any rt_msghdr plus its addresses. A short read would be discarded by
-	// the kernel, so being generous here is how we avoid missing a message.
-	netChangeReadBuffer = 4096
-)
-
-// watchNetworkChanges reports when the local network moves out from under us, so the listener can be rebound.
-//
-// Darwin scopes a udp socket to whatever interface it came up on. Move between networks and we keep sending out an
-// interface that no longer has a route, which surfaces as an instant "no route to host" with no packet ever leaving
-// the box. Rebind clears that, but only if something notices the change and calls it. iOS has always been told by
-// the host app off NWPathMonitor. This is the equivalent for everything else that runs on darwin.
-//
-// The returned channel is buffered and coalescing: a send is dropped if one is already pending, since both mean the
-// same thing to a reader. It is closed when ctx is cancelled or the routing socket fails, so a caller can simply
-// range over it. Platforms whose sockets do not need rebinding return a nil channel and no error.
-func watchNetworkChanges(ctx context.Context, l *slog.Logger) (<-chan struct{}, error) {
-	sock, err := openRouteSocket()
-	if err != nil {
-		return nil, err
-	}
-
-	changes := make(chan struct{}, 1)
-
-	go func() {
-		defer close(changes)
-		defer func() { _ = sock.Close() }()
-
-		// Closing the socket is what unblocks the read in watchRouteSocket, so this turns cancellation into a
-		// close. It is scoped to this call so it cannot outlive the watch it belongs to.
-		done := make(chan struct{})
-		defer close(done)
-		go func() {
-			select {
-			case <-ctx.Done():
-				_ = sock.Close()
-			case <-done:
-			}
-		}()
-
-		watchRouteSocket(l, sock, changes)
-	}()
-
-	return changes, nil
-}
-
-// watchRouteSocket blocks reading the routing socket, reporting once per settled burst of changes. It returns when
-// the socket is closed, which is how cancellation gets us out of here.
-func watchRouteSocket(l *slog.Logger, sock *os.File, changes chan<- struct{}) {
-	buf := make([]byte, netChangeReadBuffer)
-
-	for {
-		n, err := sock.Read(buf)
-		if err != nil {
-			logRouteSocketError(l, err)
-			return
-		}
-
-		if !isNetworkChange(buf[:n]) {
-			continue
-		}
-
-		// Swallow the rest of the burst. The deadline is absolute and not extended by what arrives, so this always
-		// ends after the settle window no matter how chatty the socket is. Changes that land after the window
-		// simply produce another report, which is the correct outcome anyway.
-		deadline := time.Now().Add(netChangeSettleWindow)
-		for {
-			if err = sock.SetReadDeadline(deadline); err != nil {
-				logRouteSocketError(l, err)
-				return
-			}
-
-			if _, err = sock.Read(buf); err != nil {
-				if os.IsTimeout(err) {
-					break
-				}
-				logRouteSocketError(l, err)
-				return
-			}
-		}
-
-		if err = sock.SetReadDeadline(time.Time{}); err != nil {
-			logRouteSocketError(l, err)
-			return
-		}
-
-		select {
-		case changes <- struct{}{}:
-		default:
-			// One already pending, and a second "the network moved" tells the reader nothing new.
-		}
-	}
-}
-
-// logRouteSocketError reports a routing socket failure unless it is just us shutting the socket down.
-func logRouteSocketError(l *slog.Logger, err error) {
-	if errors.Is(err, os.ErrClosed) {
-		return
-	}
-
-	l.Error("Error reading the routing socket, will no longer notice local network changes", "error", err)
-}
-
-// openRouteSocket returns the routing socket as a non blocking os.File. Going through os.File puts reads on the go
-// poller, which buys us both a working read deadline and a Close that unblocks a read in progress.
-func openRouteSocket() (*os.File, error) {
-	fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
-	if err != nil {
-		return nil, err
-	}
-
-	if err = unix.SetNonblock(fd, true); err != nil {
-		_ = unix.Close(fd)
-		return nil, err
-	}
-
-	return os.NewFile(uintptr(fd), "route"), nil
-}
-
-// isNetworkChange reports whether a routing message means our local addressing may have moved out from under us.
-//
-// We read the header instead of parsing the message because the type is the only part we need, and a full parse can
-// fail on shapes we don't care about, which would turn "a message I can't parse" into "a change I missed".
-// rt_msghdr, if_msghdr and ifa_msghdr all begin with the same three fields, so this is the same for every type.
-func isNetworkChange(msg []byte) bool {
-	if len(msg) < 4 {
-		return false
-	}
-
-	// u_short msglen, u_char version, u_char type
-	if int(binary.NativeEndian.Uint16(msg[0:2])) > len(msg) || msg[2] != unix.RTM_VERSION {
-		return false
-	}
-
-	switch msg[3] {
-	case unix.RTM_NEWADDR, unix.RTM_DELADDR, unix.RTM_IFINFO:
-		// An address arrived or left, or a link changed state. Anything else on this socket is either a route
-		// churning underneath us, which a rebind doesn't help with, or unrelated traffic.
-		return true
-	default:
-		return false
-	}
-}

udp/netchange_darwin_test.go

@@ -1,244 +0,0 @@
-//go:build darwin && !ios && !e2e_testing
-// +build darwin,!ios,!e2e_testing
-
-package udp
-
-import (
-	"context"
-	"encoding/binary"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/test"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"go.uber.org/goleak"
-	"golang.org/x/sys/unix"
-)
-
-// routeMsg builds the first four bytes of a routing message, which is all isNetworkChange reads.
-func routeMsg(msgType uint8, extra int) []byte {
-	msg := make([]byte, 4+extra)
-	binary.NativeEndian.PutUint16(msg[0:2], uint16(len(msg)))
-	msg[2] = unix.RTM_VERSION
-	msg[3] = msgType
-	return msg
-}
-
-func TestIsNetworkChange(t *testing.T) {
-	// The three that mean our addressing may have moved
-	assert.True(t, isNetworkChange(routeMsg(unix.RTM_NEWADDR, 0)))
-	assert.True(t, isNetworkChange(routeMsg(unix.RTM_DELADDR, 0)))
-	assert.True(t, isNetworkChange(routeMsg(unix.RTM_IFINFO, 0)))
-
-	// Route churn is not something a rebind helps with
-	assert.False(t, isNetworkChange(routeMsg(unix.RTM_ADD, 0)))
-	assert.False(t, isNetworkChange(routeMsg(unix.RTM_DELETE, 0)))
-	assert.False(t, isNetworkChange(routeMsg(unix.RTM_GET, 0)))
-
-	// Garbage must not be mistaken for a change
-	assert.False(t, isNetworkChange(nil), "empty")
-	assert.False(t, isNetworkChange([]byte{0, 0, 0}), "short header")
-
-	wrongVersion := routeMsg(unix.RTM_NEWADDR, 0)
-	wrongVersion[2] = unix.RTM_VERSION + 1
-	assert.False(t, isNetworkChange(wrongVersion), "wrong rtm_version")
-
-	lying := routeMsg(unix.RTM_NEWADDR, 0)
-	binary.NativeEndian.PutUint16(lying[0:2], 512)
-	assert.False(t, isNetworkChange(lying), "msglen longer than what we read")
-}
-
-// socketPair returns a connected pair of datagram sockets, the first wrapped the same way the routing socket is. It
-// stands in for the kernel so the watch loop can be driven with synthetic messages.
-func socketPair(t *testing.T) (*os.File, int) {
-	t.Helper()
-
-	fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_DGRAM, 0)
-	require.NoError(t, err)
-	require.NoError(t, unix.SetNonblock(fds[0], true))
-
-	f := os.NewFile(uintptr(fds[0]), "route")
-	t.Cleanup(func() {
-		_ = f.Close()
-		_ = unix.Close(fds[1])
-	})
-
-	return f, fds[1]
-}
-
-func TestWatchRouteSocketCoalescesABurst(t *testing.T) {
-	sock, kernel := socketPair(t)
-	changes := make(chan struct{}, 1)
-
-	done := make(chan struct{})
-	go func() {
-		watchRouteSocket(test.NewLogger(), sock, changes)
-		close(done)
-	}()
-
-	// One network change is a burst of messages. All of these land inside the settle window, so they must produce
-	// exactly one report rather than one apiece.
-	for range 5 {
-		_, err := unix.Write(kernel, routeMsg(unix.RTM_NEWADDR, 8))
-		require.NoError(t, err)
-	}
-	// Uninteresting messages in the middle of a burst must not add a report of their own either.
-	_, err := unix.Write(kernel, routeMsg(unix.RTM_ADD, 8))
-	require.NoError(t, err)
-
-	select {
-	case <-changes:
-	case <-time.After(netChangeSettleWindow * 4):
-		t.Fatal("a burst should have reported a change")
-	}
-
-	// Nothing more from that burst
-	select {
-	case <-changes:
-		t.Fatal("a burst should report exactly once")
-	case <-time.After(netChangeSettleWindow):
-	}
-
-	// A change after the window has closed is a separate event and gets its own report.
-	_, err = unix.Write(kernel, routeMsg(unix.RTM_IFINFO, 8))
-	require.NoError(t, err)
-	select {
-	case <-changes:
-	case <-time.After(netChangeSettleWindow * 4):
-		t.Fatal("a later change should report again")
-	}
-
-	// Closing the socket is how the real thing shuts down
-	require.NoError(t, sock.Close())
-	select {
-	case <-done:
-	case <-time.After(time.Second * 5):
-		t.Fatal("watchRouteSocket did not return after the socket was closed")
-	}
-}
-
-func TestWatchRouteSocketIgnoresUninterestingMessages(t *testing.T) {
-	sock, kernel := socketPair(t)
-	changes := make(chan struct{}, 1)
-
-	done := make(chan struct{})
-	go func() {
-		watchRouteSocket(test.NewLogger(), sock, changes)
-		close(done)
-	}()
-
-	for _, msgType := range []uint8{unix.RTM_ADD, unix.RTM_DELETE, unix.RTM_GET, unix.RTM_MISS} {
-		_, err := unix.Write(kernel, routeMsg(msgType, 8))
-		require.NoError(t, err)
-	}
-
-	select {
-	case <-changes:
-		t.Fatal("route churn alone must not report a change")
-	case <-time.After(netChangeSettleWindow * 2):
-	}
-
-	require.NoError(t, sock.Close())
-	select {
-	case <-done:
-	case <-time.After(time.Second * 5):
-		t.Fatal("watchRouteSocket did not return after the socket was closed")
-	}
-}
-
-// TestWatchRouteSocketDropsRatherThanBlocks covers the coalescing send. A reader that is busy rebinding must not
-// wedge the watcher, and a second pending "the network moved" tells it nothing new anyway.
-func TestWatchRouteSocketDropsRatherThanBlocks(t *testing.T) {
-	sock, kernel := socketPair(t)
-	changes := make(chan struct{}, 1)
-
-	done := make(chan struct{})
-	go func() {
-		watchRouteSocket(test.NewLogger(), sock, changes)
-		close(done)
-	}()
-
-	// Nobody is reading changes, so after the first report the buffer is full for the rest of this test
-	for range 3 {
-		_, err := unix.Write(kernel, routeMsg(unix.RTM_NEWADDR, 8))
-		require.NoError(t, err)
-		time.Sleep(netChangeSettleWindow + time.Millisecond*250)
-	}
-
-	// The watcher must still be alive and responsive to a close
-	require.NoError(t, sock.Close())
-	select {
-	case <-done:
-	case <-time.After(time.Second * 5):
-		t.Fatal("watchRouteSocket wedged on a full channel")
-	}
-
-	assert.Len(t, changes, 1, "the pending report should have coalesced, not queued")
-}
-
-// TestWatchNetworkChangesStopsWithContext covers the detection path against a real routing socket, including that
-// cancelling the context closes the channel so a ranging caller falls out of its loop.
-func TestWatchNetworkChangesStopsWithContext(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-
-	changes, err := watchNetworkChanges(ctx, test.NewLogger())
-	require.NoError(t, err)
-	require.NotNil(t, changes, "darwin should support watching")
-
-	drained := make(chan struct{})
-	go func() {
-		for range changes {
-		}
-		close(drained)
-	}()
-
-	cancel()
-	select {
-	case <-drained:
-	case <-time.After(time.Second * 5):
-		t.Fatal("cancelling the context should close the changes channel")
-	}
-}
-
-// TestNetworkChangeMonitorStopsWithContext drives the whole monitor against a real routing socket: Start must block
-// watching, and cancelling the context (which is all Control does on shutdown, it never stops the monitor directly)
-// must return it and clean up the watch goroutines.
-func TestNetworkChangeMonitorStopsWithContext(t *testing.T) {
-	// IgnoreCurrent because other tests in this package leave readers running; we only care about what this test
-	// leaks itself.
-	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
-
-	ctx, cancel := context.WithCancel(context.Background())
-
-	l := test.NewLogger()
-	c := config.NewC(l)
-	require.NoError(t, c.LoadString("listen:\n  rebind_on_network_change: true\n"))
-	m := NewNetworkChangeMonitor(ctx, l, c)
-
-	done := make(chan struct{})
-	go func() {
-		m.Start(func() {})
-		close(done)
-	}()
-
-	// Start should be sitting on the routing socket, not have fallen out. If it returned early it either failed to
-	// watch or no-op'd, both of which we want to catch.
-	select {
-	case <-done:
-		t.Fatal("Start returned instead of watching")
-	case <-time.After(time.Millisecond * 250):
-	}
-
-	cancel()
-	select {
-	case <-done:
-	case <-time.After(time.Second * 5):
-		t.Fatal("Start did not return after the context was cancelled")
-	}
-
-	// Starting again after the context is dead must not open anything.
-	m.Start(func() {})
-}

udp/netchange_generic.go

@@ -1,22 +0,0 @@
-//go:build !darwin || ios || e2e_testing
-// +build !darwin ios e2e_testing
-
-package udp
-
-import (
-	"context"
-	"log/slog"
-)
-
-// watchNetworkChanges is a no-op outside of darwin.
-//
-// Darwin is the platform that scopes a udp socket to the interface it came up on, so it is the platform whose socket
-// goes stale when the local network changes. Everywhere else Rebind has nothing to do, so there is nothing to watch
-// for. iOS is excluded on purpose even though it is darwin: the host app already drives the rebind off NWPathMonitor,
-// and two things racing to rebind the same socket is worse than one.
-//
-// A nil channel means "not supported here", which callers must treat as "do not start a watcher" rather than
-// selecting on it, since a receive from a nil channel blocks forever.
-func watchNetworkChanges(_ context.Context, _ *slog.Logger) (<-chan struct{}, error) {
-	return nil, nil
-}

udp/netchange_test.go

@@ -1,39 +0,0 @@
-package udp
-
-import (
-	"context"
-	"testing"
-
-	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/test"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func newMonitor(t *testing.T, ctx context.Context, cfg string) *NetworkChangeMonitor {
-	t.Helper()
-	l := test.NewLogger()
-	c := config.NewC(l)
-	require.NoError(t, c.LoadString(cfg))
-	return NewNetworkChangeMonitor(ctx, l, c)
-}
-
-func TestNetworkChangeMonitorDefaultsOn(t *testing.T) {
-	// Says nothing about rebinding, so this covers the default.
-	m := newMonitor(t, context.Background(), "listen:\n  host: 0.0.0.0\n")
-	assert.True(t, m.enabled, "should default to on")
-}
-
-func TestNetworkChangeMonitorDisabledIsANoOp(t *testing.T) {
-	m := newMonitor(t, context.Background(), "listen:\n  rebind_on_network_change: false\n")
-	require.False(t, m.enabled)
-
-	// Must return without opening a socket. If it watched anything this would block.
-	m.Start(func() {})
-}
-
-func TestNetworkChangeMonitorNilRebindIsANoOp(t *testing.T) {
-	// Nothing to rebind, so there is no point watching, on any platform.
-	m := newMonitor(t, context.Background(), "listen:\n  rebind_on_network_change: true\n")
-	m.Start(nil)
-}

udp/udp_darwin.go

@@ -187,9 +187,6 @@ func (u *StdConn) SupportsMultipleReaders() bool {
 	return false
 }
 
-// Rebind clears the interface the kernel scoped this socket to, so that sends are routed against the current
-// routing table instead of the interface we happened to be on when the socket was created. Darwin pins sockets
-// this way on its own, which is what strands us after the underlying network changes.
 func (u *StdConn) Rebind() error {
 	var err error
 	if u.isV4 {
@@ -198,5 +195,9 @@ func (u *StdConn) Rebind() error {
 		err = syscall.SetsockoptInt(int(u.sysFd), syscall.IPPROTO_IPV6, syscall.IPV6_BOUND_IF, 0)
 	}
 
-	return err
+	if err != nil {
+		u.l.Error("Failed to rebind udp socket", "error", err)
+	}
+
+	return nil
 }

udp/udp_linux.go

@@ -4,13 +4,12 @@
 package udp
 
 import (
+	"context"
 	"encoding/binary"
-	"errors"
 	"fmt"
 	"log/slog"
 	"net"
 	"net/netip"
-	"sync/atomic"
 	"syscall"
 	"unsafe"
 
@@ -20,51 +19,58 @@ import (
 )
 
 type StdConn struct {
-	sysFd  int
+	udpConn *net.UDPConn
-	closed atomic.Bool
+	rawConn syscall.RawConn
-	isV4   bool
+	isV4    bool
-	l      *slog.Logger
+	l       *slog.Logger
-	batch  int
+	batch   int
+}
+
+func setReusePort(network, address string, c syscall.RawConn) error {
+	var opErr error
+	err := c.Control(func(fd uintptr) {
+		opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
+		//CloseOnExec already set by the runtime
+	})
+	if err != nil {
+		return err
+	}
+	return opErr
 }
 
 func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
-	af := unix.AF_INET6
+	listen := netip.AddrPortFrom(ip, uint16(port))
-	if ip.Is4() {
+	lc := net.ListenConfig{}
-		af = unix.AF_INET
-	}
-	syscall.ForkLock.RLock()
-	fd, err := unix.Socket(af, unix.SOCK_DGRAM, unix.IPPROTO_UDP)
-	if err == nil {
-		unix.CloseOnExec(fd)
-	}
-	syscall.ForkLock.RUnlock()
-	if err != nil {
-		return nil, fmt.Errorf("unable to open socket: %w", err)
-	}
-
 	if multi {
-		if err = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
+		lc.Control = setReusePort
-			_ = unix.Close(fd)
+	}
-			return nil, fmt.Errorf("unable to set SO_REUSEPORT: %w", err)
+	//this context is only used during the bind operation, you can't cancel it to kill the socket
-		}
+	pc, err := lc.ListenPacket(context.Background(), "udp", listen.String())
+	if err != nil {
+		return nil, fmt.Errorf("unable to open socket: %s", err)
+	}
+	udpConn := pc.(*net.UDPConn)
+	rawConn, err := udpConn.SyscallConn()
+	if err != nil {
+		_ = udpConn.Close()
+		return nil, err
+	}
+	//gotta find out if we got an AF_INET6 socket or not:
+	out := &StdConn{
+		udpConn: udpConn,
+		rawConn: rawConn,
+		l:       l,
+		batch:   batch,
 	}
 
-	var sa unix.Sockaddr
+	af, err := out.getSockOptInt(unix.SO_DOMAIN)
-	if ip.Is4() {
+	if err != nil {
-		sa4 := &unix.SockaddrInet4{Port: port}
+		_ = out.Close()
-		sa4.Addr = ip.As4()
+		return nil, err
-		sa = sa4
-	} else {
-		sa6 := &unix.SockaddrInet6{Port: port}
-		sa6.Addr = ip.As16()
-		sa = sa6
-	}
-	if err = unix.Bind(fd, sa); err != nil {
-		_ = unix.Close(fd)
-		return nil, fmt.Errorf("unable to bind to socket: %w", err)
 	}
+	out.isV4 = af == unix.AF_INET
 
-	return &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch}, nil
+	return out, nil
 }
 
 func (u *StdConn) SupportsMultipleReaders() bool {
@@ -75,111 +81,134 @@ func (u *StdConn) Rebind() error {
 	return nil
 }
 
+func (u *StdConn) getSockOptInt(opt int) (int, error) {
+	if u.rawConn == nil {
+		return 0, fmt.Errorf("no UDP connection")
+	}
+	var out int
+	var opErr error
+	err := u.rawConn.Control(func(fd uintptr) {
+		out, opErr = unix.GetsockoptInt(int(fd), unix.SOL_SOCKET, opt)
+	})
+	if err != nil {
+		return 0, err
+	}
+	return out, opErr
+}
+
+func (u *StdConn) setSockOptInt(opt int, n int) error {
+	if u.rawConn == nil {
+		return fmt.Errorf("no UDP connection")
+	}
+	var opErr error
+	err := u.rawConn.Control(func(fd uintptr) {
+		opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, opt, n)
+	})
+	if err != nil {
+		return err
+	}
+	return opErr
+}
+
 func (u *StdConn) SetRecvBuffer(n int) error {
-	return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUFFORCE, n)
+	return u.setSockOptInt(unix.SO_RCVBUFFORCE, n)
 }
 
 func (u *StdConn) SetSendBuffer(n int) error {
-	return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUFFORCE, n)
+	return u.setSockOptInt(unix.SO_SNDBUFFORCE, n)
 }
 
 func (u *StdConn) SetSoMark(mark int) error {
-	return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK, mark)
+	return u.setSockOptInt(unix.SO_MARK, mark)
 }
 
 func (u *StdConn) GetRecvBuffer() (int, error) {
-	return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUF)
+	return u.getSockOptInt(unix.SO_RCVBUF)
 }
 
 func (u *StdConn) GetSendBuffer() (int, error) {
-	return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUF)
+	return u.getSockOptInt(unix.SO_SNDBUF)
 }
 
 func (u *StdConn) GetSoMark() (int, error) {
-	return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK)
+	return u.getSockOptInt(unix.SO_MARK)
 }
 
 func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
-	sa, err := unix.Getsockname(u.sysFd)
+	a := u.udpConn.LocalAddr()
-	if err != nil {
+
-		return netip.AddrPort{}, err
+	switch v := a.(type) {
-	}
+	case *net.UDPAddr:
-	switch sa := sa.(type) {
+		addr, ok := netip.AddrFromSlice(v.IP)
-	case *unix.SockaddrInet4:
+		if !ok {
-		return netip.AddrPortFrom(netip.AddrFrom4(sa.Addr), uint16(sa.Port)), nil
+			return netip.AddrPort{}, fmt.Errorf("LocalAddr returned invalid IP address: %s", v.IP)
-	case *unix.SockaddrInet6:
+		}
-		return netip.AddrPortFrom(netip.AddrFrom16(sa.Addr), uint16(sa.Port)), nil
+		return netip.AddrPortFrom(addr, uint16(v.Port)), nil
+
 	default:
-		return netip.AddrPort{}, fmt.Errorf("unsupported sock type: %T", sa)
+		return netip.AddrPort{}, fmt.Errorf("LocalAddr returned: %#v", a)
 	}
 }
 
-// recvmmsg does one blocking recvmmsg (MSG_WAITFORONE), reading up to len(msgs) datagrams
+func recvmmsg(fd uintptr, msgs []rawMessage) (int, bool, error) {
-func (u *StdConn) recvmmsg(msgs []rawMessage) (int, error) {
+	var errno syscall.Errno
-	r, _, errno := unix.Syscall6(
+	n, _, errno := unix.Syscall6(
 		unix.SYS_RECVMMSG,
-		uintptr(u.sysFd),
+		fd,
 		uintptr(unsafe.Pointer(&msgs[0])),
 		uintptr(len(msgs)),
 		unix.MSG_WAITFORONE,
 		0,
 		0,
 	)
+	if errno == syscall.EAGAIN || errno == syscall.EWOULDBLOCK {
+		// No data available, block for I/O and try again.
+		return int(n), false, nil
+	}
 	if errno != 0 {
-		if u.closed.Load() {
+		return int(n), true, &net.OpError{Op: "recvmmsg", Err: errno}
-			return 0, net.ErrClosed
-		}
-		return 0, &net.OpError{Op: "recvmmsg", Err: errno}
 	}
-	n := int(r)
+	return int(n), true, nil
-	if (n == 0 || msgs[0].Len == 0) && u.closed.Load() {
-		return 0, net.ErrClosed
-	}
-	return n, nil
 }
 
-// recvmsg does one blocking recvmsg into msgs[0]
+func (u *StdConn) listenOutSingle(r EncReader) error {
-func (u *StdConn) recvmsg(msgs []rawMessage) (int, error) {
+	var err error
-	r, _, errno := unix.Syscall6(
+	var n int
-		unix.SYS_RECVMSG,
+	var from netip.AddrPort
-		uintptr(u.sysFd),
+	buffer := make([]byte, MTU)
-		uintptr(unsafe.Pointer(&msgs[0].Hdr)),
+
-		0,
+	for {
-		0,
+		n, from, err = u.udpConn.ReadFromUDPAddrPort(buffer)
-		0,
+		if err != nil {
-		0,
+			return err
-	)
-	if errno != 0 {
-		if u.closed.Load() {
-			return 0, net.ErrClosed
 		}
-		return 0, &net.OpError{Op: "recvmsg", Err: errno}
+		from = netip.AddrPortFrom(from.Addr().Unmap(), from.Port())
+		r(from, buffer[:n])
 	}
-	if r == 0 && u.closed.Load() {
-		return 0, net.ErrClosed
-	}
-	msgs[0].Len = uint32(r)
-	return 1, nil
 }
 
-func (u *StdConn) ListenOut(r EncReader) error {
+func (u *StdConn) listenOutBatch(r EncReader) error {
 	var ip netip.Addr
+	var n int
+	var operr error
+
 	msgs, buffers, names := u.PrepareRawMessages(u.batch)
-	read := u.recvmmsg
+
-	if u.batch == 1 {
+	//reader needs to capture variables from this function, since it's used as a lambda with rawConn.Read
-		read = u.recvmsg
+	//defining it outside the loop so it gets re-used
+	reader := func(fd uintptr) (done bool) {
+		n, done, operr = recvmmsg(fd, msgs)
+		return done
 	}
 
 	for {
-		n, err := read(msgs)
+		err := u.rawConn.Read(reader)
 		if err != nil {
-			if errors.Is(err, unix.EINTR) {
-				continue // interrupted by a signal, retry the read
-			}
-			// net.ErrClosed after Close() is teardown, absorbed by the caller's
-			// closed flag like the other platforms; anything else is a real error.
 			return err
 		}
+		if operr != nil {
+			return operr
+		}
 
 		for i := 0; i < n; i++ {
 			// Its ok to skip the ok check here, the slicing is the only error that can occur and it will panic
@@ -193,68 +222,26 @@ func (u *StdConn) ListenOut(r EncReader) error {
 	}
 }
 
+func (u *StdConn) ListenOut(r EncReader) error {
+	if u.batch == 1 {
+		return u.listenOutSingle(r)
+	} else {
+		return u.listenOutBatch(r)
+	}
+}
+
 func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
-	if u.isV4 {
+	_, err := u.udpConn.WriteToUDPAddrPort(b, ip)
-		return u.writeTo4(b, ip)
+	return err
-	}
-	return u.writeTo6(b, ip)
-}
-
-func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
-	var rsa unix.RawSockaddrInet6
-	rsa.Family = unix.AF_INET6
-	rsa.Addr = ip.Addr().As16()
-	binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
-
-	for {
-		_, _, err := unix.Syscall6(
-			unix.SYS_SENDTO,
-			uintptr(u.sysFd),
-			uintptr(unsafe.Pointer(&b[0])),
-			uintptr(len(b)),
-			uintptr(0),
-			uintptr(unsafe.Pointer(&rsa)),
-			uintptr(unix.SizeofSockaddrInet6),
-		)
-		if err != 0 {
-			return &net.OpError{Op: "sendto", Err: err}
-		}
-		return nil
-	}
-}
-
-func (u *StdConn) writeTo4(b []byte, ip netip.AddrPort) error {
-	if !ip.Addr().Is4() {
-		return ErrInvalidIPv6RemoteForSocket
-	}
-
-	var rsa unix.RawSockaddrInet4
-	rsa.Family = unix.AF_INET
-	rsa.Addr = ip.Addr().As4()
-	binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
-
-	for {
-		_, _, err := unix.Syscall6(
-			unix.SYS_SENDTO,
-			uintptr(u.sysFd),
-			uintptr(unsafe.Pointer(&b[0])),
-			uintptr(len(b)),
-			uintptr(0),
-			uintptr(unsafe.Pointer(&rsa)),
-			uintptr(unix.SizeofSockaddrInet4),
-		)
-		if err != 0 {
-			return &net.OpError{Op: "sendto", Err: err}
-		}
-		return nil
-	}
 }
 
 func (u *StdConn) ReloadConfig(c *config.C) {
 	b := c.GetInt("listen.read_buffer", 0)
 	if b > 0 {
-		if err := u.SetRecvBuffer(b); err == nil {
+		err := u.SetRecvBuffer(b)
-			if s, err := u.GetRecvBuffer(); err == nil {
+		if err == nil {
+			s, err := u.GetRecvBuffer()
+			if err == nil {
 				u.l.Info("listen.read_buffer was set", "size", s)
 			} else {
 				u.l.Warn("Failed to get listen.read_buffer", "error", err)
@@ -266,8 +253,10 @@ func (u *StdConn) ReloadConfig(c *config.C) {
 
 	b = c.GetInt("listen.write_buffer", 0)
 	if b > 0 {
-		if err := u.SetSendBuffer(b); err == nil {
+		err := u.SetSendBuffer(b)
-			if s, err := u.GetSendBuffer(); err == nil {
+		if err == nil {
+			s, err := u.GetSendBuffer()
+			if err == nil {
 				u.l.Info("listen.write_buffer was set", "size", s)
 			} else {
 				u.l.Warn("Failed to get listen.write_buffer", "error", err)
@@ -280,8 +269,10 @@ func (u *StdConn) ReloadConfig(c *config.C) {
 	b = c.GetInt("listen.so_mark", 0)
 	s, err := u.GetSoMark()
 	if b > 0 || (err == nil && s != 0) {
-		if err := u.SetSoMark(b); err == nil {
+		err := u.SetSoMark(b)
-			if s, err := u.GetSoMark(); err == nil {
+		if err == nil {
+			s, err := u.GetSoMark()
+			if err == nil {
 				u.l.Info("listen.so_mark was set", "mark", s)
 			} else {
 				u.l.Warn("Failed to get listen.so_mark", "error", err)
@@ -294,20 +285,28 @@ func (u *StdConn) ReloadConfig(c *config.C) {
 
 func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
 	var vallen uint32 = 4 * unix.SK_MEMINFO_VARS
-	_, _, err := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(u.sysFd), uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
+
-	if err != 0 {
+	if u.rawConn == nil {
+		return fmt.Errorf("no UDP connection")
+	}
+	var opErr error
+	err := u.rawConn.Control(func(fd uintptr) {
+		_, _, syserr := unix.Syscall6(unix.SYS_GETSOCKOPT, fd, uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
+		if syserr != 0 {
+			opErr = syserr
+		}
+	})
+	if err != nil {
 		return err
 	}
-	return nil
+	return opErr
 }
 
 func (u *StdConn) Close() error {
-	u.closed.Store(true)
+	if u.udpConn != nil {
-	// Wake the reader parked in recvmmsg/recvmsg. shutdown(2) on an unconnected socket
+		return u.udpConn.Close()
-	// returns ENOTCONN but still wakes it, so ignore the error.
+	}
-	// The reader then sees closed and stops touching the fd, making the Close below safe.
+	return nil
-	_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
-	return unix.Close(u.sysFd)
 }
 
 func NewUDPStatsEmitter(udpConns []Conn) func() {

udp/udp_linux_test.go

@@ -1,179 +0,0 @@
-//go:build linux && !android && !e2e_testing
-
-package udp
-
-import (
-	"errors"
-	"fmt"
-	"log/slog"
-	"net"
-	"net/netip"
-	"os"
-	"runtime"
-	"sync/atomic"
-	"testing"
-	"time"
-
-	"golang.org/x/sys/unix"
-)
-
-func testLogger() *slog.Logger {
-	return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))
-}
-
-// TestShutdownWakesAfterRx_Mechanism exercises the kernel quirk our teardown
-// relies on: once a socket has received a packet, shutdown(2) wakes a blocked
-// recvmmsg with n>=1/Len==0 (not n==0). recvmmsg must turn that into net.ErrClosed
-// once Close set closed, so a parked reader exits instead of spinning.
-func TestShutdownWakesAfterRx_Mechanism(t *testing.T) {
-	c, err := NewListener(testLogger(), netip.MustParseAddr("127.0.0.1"), 0, true, 64)
-	if err != nil {
-		t.Fatalf("NewListener: %v", err)
-	}
-	sc := c.(*StdConn)
-	addr, err := sc.LocalAddr()
-	if err != nil {
-		t.Fatalf("LocalAddr: %v", err)
-	}
-	msgs, _, _ := sc.PrepareRawMessages(sc.batch)
-
-	// Receive a real packet so the socket has carried data.
-	send, err := net.Dial("udp", addr.String())
-	if err != nil {
-		t.Fatalf("dial: %v", err)
-	}
-	if _, err := send.Write([]byte("hello")); err != nil {
-		t.Fatalf("write: %v", err)
-	}
-	time.Sleep(50 * time.Millisecond)
-	n, err := sc.recvmmsg(msgs)
-	t.Logf("drain of real packet: n=%d err=%v msgs[0].Len=%d", n, err, msgs[0].Len)
-	_ = send.Close()
-
-	// Block a reader on the now-empty queue, then tear down as Close() does.
-	// recvmmsg must return net.ErrClosed (not hang, not spin) even post-rx.
-	done := make(chan error, 1)
-	go func() {
-		_, err := sc.recvmmsg(msgs)
-		done <- err
-	}()
-	time.Sleep(150 * time.Millisecond) // let it park in recvmmsg
-
-	sc.closed.Store(true)
-	if serr := unix.Shutdown(sc.sysFd, unix.SHUT_RDWR); serr != nil {
-		t.Logf("shutdown returned %v (expected ENOTCONN on unconnected UDP)", serr)
-	}
-
-	select {
-	case err := <-done:
-		if !errors.Is(err, net.ErrClosed) {
-			t.Errorf("recvmmsg after post-rx shutdown returned %v, want net.ErrClosed", err)
-		}
-	case <-time.After(2 * time.Second):
-		t.Fatalf("HANG: recvmmsg did not return after shutdown following a received packet")
-	}
-	_ = unix.Close(sc.sysFd)
-}
-
-// TestListenOutTeardown_TrafficPatterns reproduces the field report: a blocking
-// reader must tear down cleanly on Close() regardless of what the socket has
-// carried. The three cases the report called out:
-//
-//	no traffic ever      -> works (shutdown wakes recvmmsg with n==0)
-//	ping once, then idle -> historically HUNG: once the socket has received a
-//	                        packet, shutdown(2) wakes recvmmsg with n>=1/Len==0,
-//	                        which an n==0-only teardown check misses
-//	continuous traffic   -> works (a real packet is always arriving)
-//
-// All three must return within the deadline; a hang dumps goroutines so the
-// stuck reader is visible.
-func TestListenOutTeardown_TrafficPatterns(t *testing.T) {
-	cases := []struct {
-		name    string
-		traffic func(send net.Conn, stop <-chan struct{})
-	}{
-		{"no_traffic_ever", func(net.Conn, <-chan struct{}) {}},
-		{"ping_once_then_idle", func(send net.Conn, _ <-chan struct{}) {
-			_, _ = send.Write([]byte("hello"))
-		}},
-		{"continuous", func(send net.Conn, stop <-chan struct{}) {
-			for {
-				select {
-				case <-stop:
-					return
-				default:
-					_, _ = send.Write([]byte("hello"))
-					time.Sleep(2 * time.Millisecond)
-				}
-			}
-		}},
-	}
-
-	// batch 1 exercises the recvmsg path, batch 64 the recvmmsg path; both must
-	// tear down cleanly.
-	for _, batch := range []int{1, 64} {
-		for _, tc := range cases {
-			t.Run(fmt.Sprintf("batch%d/%s", batch, tc.name), func(t *testing.T) {
-				runTeardownCase(t, batch, tc.name, tc.traffic)
-			})
-		}
-	}
-}
-
-func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net.Conn, stop <-chan struct{})) {
-	c, err := NewListener(testLogger(), netip.MustParseAddr("127.0.0.1"), 0, true, batch)
-	if err != nil {
-		t.Fatalf("NewListener: %v", err)
-	}
-	sc := c.(*StdConn)
-	addr, err := sc.LocalAddr()
-	if err != nil {
-		t.Fatalf("LocalAddr: %v", err)
-	}
-
-	var received atomic.Int64
-	loopDone := make(chan error, 1)
-	go func() {
-		loopDone <- sc.ListenOut(func(netip.AddrPort, []byte) {
-			received.Add(1)
-		})
-	}()
-
-	send, err := net.Dial("udp", addr.String())
-	if err != nil {
-		t.Fatalf("dial: %v", err)
-	}
-	defer send.Close()
-
-	stop := make(chan struct{})
-	trafficDone := make(chan struct{})
-	go func() {
-		traffic(send, stop)
-		close(trafficDone)
-	}()
-
-	// Let the pattern run and, for the idle case, the reader park again on an
-	// empty queue with the socket already having received a packet.
-	time.Sleep(500 * time.Millisecond)
-
-	start := time.Now()
-	if err := sc.Close(); err != nil {
-		t.Fatalf("Close: %v", err)
-	}
-	close(stop)
-
-	select {
-	case err := <-loopDone:
-		// Clean teardown surfaces as net.ErrClosed (propagated like the other
-		// platforms); the caller absorbs it via its closed flag.
-		if err != nil && !errors.Is(err, net.ErrClosed) {
-			t.Fatalf("%s: ListenOut returned unexpected error on teardown: %v", name, err)
-		}
-		t.Logf("%s: closed in %v (received %d packets)", name, time.Since(start), received.Load())
-	case <-time.After(3 * time.Second):
-		buf := make([]byte, 1<<20)
-		n := runtime.Stack(buf, true)
-		t.Fatalf("%s: HANG, ListenOut did not return within 3s of Close\n%s", name, buf[:n])
-	}
-	<-trafficDone
-}

udp/udp_tester.go

@@ -10,7 +10,6 @@ import (
 	"net/netip"
 	"os"
 	"sync"
-	"sync/atomic"
 
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
@@ -65,9 +64,7 @@ func acquirePacket() *Packet {
 }
 
 type TesterConn struct {
-	// addr is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
+	Addr netip.AddrPort
-	// move it mid-run to simulate roaming, so it is atomic rather than a plain field.
-	addr atomic.Pointer[netip.AddrPort]
 
 	RxPackets chan *Packet // Packets to receive into nebula
 	TxPackets chan *Packet // Packets transmitted outside by nebula
@@ -85,24 +82,13 @@ type TesterConn struct {
 }
 
 func NewListener(l *slog.Logger, ip netip.Addr, port int, _ bool, _ int) (Conn, error) {
-	c := &TesterConn{
+	return &TesterConn{
+		Addr:      netip.AddrPortFrom(ip, uint16(port)),
 		RxPackets: make(chan *Packet, 10),
 		TxPackets: make(chan *Packet, 10),
 		done:      make(chan struct{}),
 		l:         l,
-	}
+	}, nil
-	c.SetAddr(netip.AddrPortFrom(ip, uint16(port)))
-	return c, nil
-}
-
-// GetAddr returns the underlay address this conn currently sends from.
-func (u *TesterConn) GetAddr() netip.AddrPort {
-	return *u.addr.Load()
-}
-
-// SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network.
-func (u *TesterConn) SetAddr(addr netip.AddrPort) {
-	u.addr.Store(&addr)
 }
 
 // Send will place a UdpPacket onto the receive queue for nebula to consume
@@ -161,7 +147,7 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
 		p.Data = p.Data[:len(b)]
 	}
 	copy(p.Data, b)
-	p.From = u.GetAddr()
+	p.From = u.Addr
 	p.To = addr
 	select {
 	case <-u.done:
@@ -192,7 +178,7 @@ func NewUDPStatsEmitter(_ []Conn) func() {
 }
 
 func (u *TesterConn) LocalAddr() (netip.AddrPort, error) {
-	return u.GetAddr(), nil
+	return u.Addr, nil
 }
 
 func (u *TesterConn) SupportsMultipleReaders() bool {