use P256 as the default curve for fips140 mode

Co-authored-by: Jack Doan <me@jackdoan.com>

.github/workflows/release.yml

@@ -12,9 +12,9 @@ jobs:
     steps:
       - uses: actions/checkout@v7
 
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v7
         with:
-          go-version: '1.25'
+          go-version: '1.26'
           check-latest: true
 
       - name: Build
@@ -38,9 +38,9 @@ jobs:
     steps:
       - uses: actions/checkout@v7
 
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v7
         with:
-          go-version: '1.25'
+          go-version: '1.26'
           check-latest: true
 
       - name: Build
@@ -78,9 +78,9 @@ jobs:
     steps:
       - uses: actions/checkout@v7
 
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v7
         with:
-          go-version: '1.25'
+          go-version: '1.26'
           check-latest: true
 
       - name: Import certificates

.github/workflows/smoke-extra.yml

@@ -32,9 +32,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     - name: add hashicorp source
@@ -64,9 +64,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     - name: add hashicorp source
@@ -90,9 +90,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     # WSL2 + Ubuntu so the smoke can run a real linux peer with its own

.github/workflows/smoke.yml

@@ -20,44 +20,24 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
-    - name: build
+    - name: Smoke Docker
-      run: make bin-docker CGO_ENABLED=1 BUILD_ARGS=-race
+      run: make smoke-docker
 
-    - name: setup docker image
+    - name: Smoke Docker IPv6 overlay
-      working-directory: ./.github/workflows/smoke
+      run: make smoke-docker-ipv6
-      run: ./build.sh
 
-    - name: run smoke
+    - name: Smoke Relay Docker
-      working-directory: ./.github/workflows/smoke
+      run: make smoke-relay-docker
-      run: ./smoke.sh
 
-    - name: setup docker image ipv6
+    - name: Smoke Docker boringcrypto
-      working-directory: ./.github/workflows/smoke
+      run: make boringcrypto smoke-docker
-      run: SMOKE_OVERLAY_IPV6=1 ./build.sh
 
-    - name: run smoke ipv6
+    - name: Smoke Docker fips140
-      working-directory: ./.github/workflows/smoke
+      run: make fips140-all GOALS=smoke-docker
-      run: SMOKE_OVERLAY_IPV6=1 ./smoke.sh
-
-    - name: setup relay docker image
-      working-directory: ./.github/workflows/smoke
-      run: ./build-relay.sh
-
-    - name: run smoke relay
-      working-directory: ./.github/workflows/smoke
-      run: ./smoke-relay.sh
-
-    - name: setup docker image for P256
-      working-directory: ./.github/workflows/smoke
-      run: NAME="smoke-p256" CURVE=P256 ./build.sh
-
-    - name: run smoke-p256
-      working-directory: ./.github/workflows/smoke
-      run: NAME="smoke-p256" ./smoke.sh
 
     timeout-minutes: 10

.github/workflows/test.yml

@@ -20,9 +20,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     - name: Install goimports
@@ -42,7 +42,7 @@ jobs:
     - name: golangci-lint
       uses: golangci/golangci-lint-action@v9
       with:
-        version: v2.5
+        version: v2.12
 
   test:
     name: Test ${{ matrix.name }}
@@ -58,9 +58,14 @@ jobs:
           e2e-cmd: make e2evv
         - name: linux-boringcrypto
           os: ubuntu-latest
-          build-cmd: make bin-boringcrypto
+          build-cmd: make boringcrypto
-          test-cmd: make test-boringcrypto
+          test-cmd: make boringcrypto test
-          e2e-cmd: make e2e GOEXPERIMENT=boringcrypto CGO_ENABLED=1 TEST_ENV="TEST_LOGS=1" TEST_FLAGS="-v -ldflags -checklinkname=0"
+          e2e-cmd: make boringcrypto e2evv
+        - name: linux-fips140
+          os: ubuntu-latest
+          build-cmd: make fips140-all
+          test-cmd: make fips140-all GOALS=test
+          e2e-cmd: make fips140-all GOALS=e2evv
         - name: linux-pkcs11
           os: ubuntu-latest
           build-cmd: make bin-pkcs11
@@ -80,9 +85,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     - name: Build
@@ -125,9 +130,9 @@ jobs:
 
     - uses: actions/checkout@v7
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     - name: Build ${{ matrix.name }}

Makefile

@@ -72,6 +72,17 @@ ALL_CROSS_LINUX_ARM   = linux-arm-5 linux-arm-6 linux-arm-7 linux-arm64
 ALL_CROSS_LINUX_MIPS  = linux-mips linux-mipsle linux-mips64 linux-mips64le linux-mips-softfloat
 ALL_CROSS_LINUX_OTHER = linux-386 linux-ppc64le linux-riscv64 linux-loong64
 
+# Based on section 2.2 of the Go Cryptographic Module CVMP Security Policy #5247
+ALL_FIPS140 = linux-amd64-fips140 \
+	linux-arm64-fips140 \
+	windows-amd64-fips140 \
+	windows-arm64-fips140 \
+	darwin-arm64-fips140 \
+	freebsd-amd64-fips140 \
+	linux-arm-7-fips140 \
+	linux-mips64-fips140 \
+	linux-ppc64le-fips140
+
 e2e:
 	$(TEST_ENV) go test -tags=e2e_testing -count=1 $(TEST_FLAGS) ./e2e
 
@@ -137,6 +148,8 @@ release-netbsd: $(ALL_NETBSD:%=build/nebula-%.tar.gz)
 
 release-boringcrypto: build/nebula-linux-$(shell go env GOARCH)-boringcrypto.tar.gz
 
+release-fips140: $(ALL_FIPS140:%=build/nebula-%.tar.gz)
+
 BUILD_ARGS += -trimpath
 
 bin-windows: build/windows-amd64/nebula.exe build/windows-amd64/nebula-cert.exe
@@ -157,17 +170,20 @@ bin-freebsd-arm64: build/freebsd-arm64/nebula build/freebsd-arm64/nebula-cert
 bin-boringcrypto: build/linux-$(shell go env GOARCH)-boringcrypto/nebula build/linux-$(shell go env GOARCH)-boringcrypto/nebula-cert
 	mv $? .
 
+bin-fips140: build/linux-$(shell go env GOARCH)-fips140/nebula build/linux-$(shell go env GOARCH)-fips140/nebula-cert
+	mv $? .
+
 bin-pkcs11: BUILD_ARGS += -tags pkcs11
 bin-pkcs11: CGO_ENABLED = 1
 bin-pkcs11: bin
 
 bin:
-	go build $(BUILD_ARGS) -ldflags "$(LDFLAGS)" -o ./nebula${NEBULA_CMD_SUFFIX} ${NEBULA_CMD_PATH}
+	$(GOENV) go build $(BUILD_ARGS) -ldflags "$(LDFLAGS)" -o ./nebula${NEBULA_CMD_SUFFIX} ${NEBULA_CMD_PATH}
-	go build $(BUILD_ARGS) -ldflags "$(LDFLAGS)" -o ./nebula-cert${NEBULA_CMD_SUFFIX} ./cmd/nebula-cert
+	$(GOENV) go build $(BUILD_ARGS) -ldflags "$(LDFLAGS)" -o ./nebula-cert${NEBULA_CMD_SUFFIX} ./cmd/nebula-cert
 
 install:
-	go install $(BUILD_ARGS) -ldflags "$(LDFLAGS)" ${NEBULA_CMD_PATH}
+	$(GOENV) go install $(BUILD_ARGS) -ldflags "$(LDFLAGS)" ${NEBULA_CMD_PATH}
-	go install $(BUILD_ARGS) -ldflags "$(LDFLAGS)" ./cmd/nebula-cert
+	$(GOENV) go install $(BUILD_ARGS) -ldflags "$(LDFLAGS)" ./cmd/nebula-cert
 
 build/linux-arm-%: GOENV += GOARM=$(word 3, $(subst -, ,$*))
 build/linux-mips-%: GOENV += GOMIPS=$(word 3, $(subst -, ,$*))
@@ -178,8 +194,11 @@ build/linux-mips-softfloat/%: LDFLAGS += -s -w
 # boringcrypto
 build/linux-amd64-boringcrypto/%: GOENV += GOEXPERIMENT=boringcrypto CGO_ENABLED=1
 build/linux-arm64-boringcrypto/%: GOENV += GOEXPERIMENT=boringcrypto CGO_ENABLED=1
-build/linux-amd64-boringcrypto/%: LDFLAGS += -checklinkname=0
+
-build/linux-arm64-boringcrypto/%: LDFLAGS += -checklinkname=0
+# fips140
+FIPSVERSION = v1.0.0
+$(foreach _rule, $(ALL_FIPS140), build/$(_rule)/%): GOENV += GOFIPS140=$(FIPSVERSION)
+$(foreach _rule, $(ALL_FIPS140), build/$(_rule)/%): BUILD_ARGS += -tags fips140enforce
 
 build/%/nebula: .FORCE
 	GOOS=$(firstword $(subst -, , $*)) \
@@ -210,10 +229,7 @@ vet:
 	go vet $(VET_FLAGS) -v ./...
 
 test:
-	go test -v ./...
+	$(TEST_ENV) go test $(TEST_FLAGS) -v ./...
-
-test-boringcrypto:
-	GOEXPERIMENT=boringcrypto CGO_ENABLED=1 go test -ldflags "-checklinkname=0" -v ./...
 
 test-pkcs11:
 	CGO_ENABLED=1 go test -v -tags pkcs11 ./...
@@ -256,29 +272,72 @@ ifeq ($(words $(MAKECMDGOALS)),1)
 	@$(MAKE) service ${.DEFAULT_GOAL} --no-print-directory
 endif
 
+# Useful to chain together, like:
+# - make fips140 e2evv
+# - make fips140 smoke-docker
+# Use `release-fips140` to build release binaries
+fips140:
+	@echo > $(NULL_FILE)
+ifeq ($(strip $(GOFIPS140)),)
+	$(eval GOFIPS140 = $(FIPSVERSION))
+endif
+	$(eval GOENV += GOFIPS140=$(GOFIPS140))
+	$(eval BUILD_ARGS += -tags fips140enforce)
+	$(eval TEST_ENV += $(GOENV))
+	$(eval CURVE = P256)
+ifeq ($(words $(MAKECMDGOALS)),1)
+	@$(MAKE) fips140 GOFIPS140=$(GOFIPS140) ${.DEFAULT_GOAL} --no-print-directory
+endif
+
+# To test the future pending module, use like `make fips140-latest test`
+ALL_GOFIPS140 = v1.0.0 v1.26.0 latest
+define FIPS140_rule
+fips140-$(1): GOFIPS140 = $(1)
+fips140-$(1): fips140
+endef
+$(foreach _rule, $(ALL_GOFIPS140), $(eval $(call FIPS140_rule,$(_rule))))
+
+# Iterate and run the goals for all fips versions, like `make fips140-all GOALS=test`
+fips140-all:
+	@$(foreach _v,$(ALL_GOFIPS140),$(MAKE) fips140-$(_v) $(GOALS) &&) true
+
+# Useful to chain together, like:
+# - make boringcrypto e2evv
+# - make boringcrypto smoke-docker
+# Use `release-boringcrypto` or `bin-boringcrypto` to build release binaries
+boringcrypto:
+	@echo > $(NULL_FILE)
+	$(eval GOENV += GOEXPERIMENT=boringcrypto CGO_ENABLED=1)
+	$(eval TEST_ENV += $(GOENV))
+	$(eval CURVE = P256)
+ifeq ($(words $(MAKECMDGOALS)),1)
+	@$(MAKE) boringcrypto ${.DEFAULT_GOAL} --no-print-directory
+endif
+
 bin-docker: bin build/linux-amd64/nebula build/linux-amd64/nebula-cert
 
+smoke-docker: BUILD_ARGS += -race
+smoke-docker: GOENV += CGO_ENABLED=1
 smoke-docker: bin-docker
-	cd .github/workflows/smoke/ && ./build.sh
+	# This is so we can limit `fips140` smoke test to just P256 curve.
-	cd .github/workflows/smoke/ && ./smoke.sh
+	if [ "$(CURVE)" != "P256" ]; then cd .github/workflows/smoke/ && $(GOENV) ./build.sh; fi
-	cd .github/workflows/smoke/ && NAME="smoke-p256" CURVE="P256" ./build.sh
+	if [ "$(CURVE)" != "P256" ]; then cd .github/workflows/smoke/ && $(GOENV) ./smoke.sh; fi
-	cd .github/workflows/smoke/ && NAME="smoke-p256" ./smoke.sh
+	cd .github/workflows/smoke/ && $(GOENV) NAME="smoke-p256" CURVE="P256" ./build.sh
+	cd .github/workflows/smoke/ && $(GOENV) NAME="smoke-p256" ./smoke.sh
 
+smoke-relay-docker: BUILD_ARGS += -race
+smoke-relay-docker: GOENV += CGO_ENABLED=1
 smoke-relay-docker: bin-docker
-	cd .github/workflows/smoke/ && ./build-relay.sh
+	cd .github/workflows/smoke/ && $(GOENV) ./build-relay.sh
-	cd .github/workflows/smoke/ && ./smoke-relay.sh
+	cd .github/workflows/smoke/ && $(GOENV) ./smoke-relay.sh
 
 smoke-docker-ipv6: export SMOKE_OVERLAY_IPV6 = 1
 smoke-docker-ipv6: smoke-docker
 
-smoke-docker-race: BUILD_ARGS = -race
-smoke-docker-race: CGO_ENABLED = 1
-smoke-docker-race: smoke-docker
-
 smoke-vagrant/%: bin-docker build/%/nebula
 	cd .github/workflows/smoke/ && ./build.sh $*
 	cd .github/workflows/smoke/ && ./smoke-vagrant.sh $*
 
 .FORCE:
-.PHONY: all all-linux all-freebsd all-openbsd all-netbsd all-darwin all-windows all-cross-linux all-cross-linux-arm all-cross-linux-mips all-cross-linux-other all-cross-darwin all-cross-windows bench bench-cpu bench-cpu-long bin build-test-mobile e2e e2ev e2evv e2evvv e2evvvv proto release service smoke-docker smoke-docker-race test test-cov-html smoke-vagrant/%
+.PHONY: all all-linux all-freebsd all-openbsd all-netbsd all-darwin all-windows all-cross-linux all-cross-linux-arm all-cross-linux-mips all-cross-linux-other all-cross-darwin all-cross-windows bench bench-cpu bench-cpu-long bin bin-windows bin-windows-arm64 bin-darwin bin-freebsd bin-freebsd-arm64 bin-boringcrypto bin-fips140 bin-pkcs11 bin-docker boringcrypto build-test-mobile docker e2e e2ev e2evv e2evvv e2evvvv e2e-bench fips140 fips140-all $(ALL_GOFIPS140:%=fips140-%) install proto release release-linux release-freebsd release-openbsd release-netbsd release-boringcrypto release-fips140 service smoke-docker smoke-relay-docker smoke-docker-ipv6 test test-pkcs11 test-cov-html vet smoke-vagrant/%
 .DEFAULT_GOAL := bin

README.md

@@ -145,17 +145,27 @@ To build nebula for a specific platform (ex, Windows):
 
 See the [Makefile](Makefile) for more details on build targets
 
-## Curve P256 and BoringCrypto
+## Curve P256 and FIPS 140-3 mode
 
 The default curve used for cryptographic handshakes and signatures is Curve25519. This is the recommended setting for most users. If your deployment has certain compliance requirements, you have the option of creating your CA using `nebula-cert ca -curve P256` to use NIST Curve P256. The CA will then sign certificates using ECDSA P256, and any hosts using these certificates will use P256 for ECDH handshakes.
 
-In addition, Nebula can be built using the [BoringCrypto GOEXPERIMENT](https://github.com/golang/go/blob/go1.20/src/crypto/internal/boring/README.md) by running either of the following make targets:
+Nebula can be built to support the [FIPS 140-3](https://go.dev/doc/security/fips140) mode of Go by running either of the following make targets. (This sets GOFIPS140=v1.0.0, which must be done at compile time so that the correct AES-GCM can be used for FIPS 140-3 enforcement mode).
+
+```sh
+make fips140
+make fips140 test
+make release-fips140
+```
+
+Nebula can also be built using the [BoringCrypto GOEXPERIMENT](https://github.com/golang/go/blob/go1.20/src/crypto/internal/boring/README.md) by running either of the following make targets.
 
 ```sh
 make bin-boringcrypto
 make release-boringcrypto
 ```
 
+NOTE: boringcrypto support is deprecated and will be removed in the next release. Users should migrate to the native FIPS 140-3 mode described above.
+
 This is not the recommended default deployment, but may be useful based on your compliance requirements.
 
 ## Credits

cmd/nebula-cert/ca.go

@@ -3,6 +3,7 @@ package main
 import (
 	"crypto/ecdsa"
 	"crypto/elliptic"
+	"crypto/fips140"
 	"crypto/rand"
 	"flag"
 	"fmt"
@@ -43,6 +44,13 @@ type caFlags struct {
 	subnets *string
 }
 
+func defaultCurve() string {
+	if fips140.Enforced() {
+		return "P256"
+	}
+	return "25519"
+}
+
 func newCaFlags() *caFlags {
 	cf := caFlags{set: flag.NewFlagSet("ca", flag.ContinueOnError)}
 	cf.set.Usage = func() {}
@@ -59,7 +67,7 @@ func newCaFlags() *caFlags {
 	cf.argonParallelism = cf.set.Uint("argon-parallelism", 4, "Optional: Argon2 parallelism parameter used for encrypted private key passphrase")
 	cf.argonIterations = cf.set.Uint("argon-iterations", 1, "Optional: Argon2 iterations parameter used for encrypted private key passphrase")
 	cf.encryption = cf.set.Bool("encrypt", false, "Optional: prompt for passphrase and write out-key in an encrypted format")
-	cf.curve = cf.set.String("curve", "25519", "EdDSA/ECDSA Curve (25519, P256)")
+	cf.curve = cf.set.String("curve", defaultCurve(), "EdDSA/ECDSA Curve (25519, P256)")
 	cf.p11url = p11Flag(cf.set)
 
 	cf.ips = cf.set.String("ips", "", "Deprecated, see -networks")

cmd/nebula-cert/fips140enforce.go

@@ -0,0 +1,5 @@
+//go:build fips140enforce
+
+//go:debug fips140=only
+
+package main

cmd/nebula-cert/keygen.go

@@ -24,7 +24,7 @@ func newKeygenFlags() *keygenFlags {
 	cf.set.Usage = func() {}
 	cf.outPubPath = cf.set.String("out-pub", "", "Required: path to write the public key to")
 	cf.outKeyPath = cf.set.String("out-key", "", "Required: path to write the private key to")
-	cf.curve = cf.set.String("curve", "25519", "ECDH Curve (25519, P256)")
+	cf.curve = cf.set.String("curve", defaultCurve(), "ECDH Curve (25519, P256)")
 	cf.p11url = p11Flag(cf.set)
 	return &cf
 }

cmd/nebula-service/fips140enforce.go

@@ -0,0 +1,5 @@
+//go:build fips140enforce
+
+//go:debug fips140=only
+
+package main

cmd/nebula/close_on_timer_test.go

@@ -0,0 +1,96 @@
+//go:build linux && !android && !e2e_testing
+
+package main
+
+import (
+	"fmt"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/cert"
+	cert_test "github.com/slackhq/nebula/cert_test"
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/test"
+	"github.com/stretchr/testify/require"
+)
+
+// TestControlStopClosesOnTimer reproduces the dnclient lifecycle: nebula runs as
+// a library, and on a config update dnclient calls Stop() in-process to tear the
+// old instance down before starting a new one. This boots a real nebula (real
+// blocking UDP sockets, tun disabled), lets it run, then Stop()s it on a timer
+// and asserts it actually closes. If the reader goroutines parked in recvmmsg
+// don't wake on Close(), Wait() blocks forever and this fails with a goroutine
+// dump instead of relying on a process signal to unstick them.
+func TestControlStopClosesOnTimer(t *testing.T) {
+	l := test.NewLogger()
+	dir := t.TempDir()
+
+	before := time.Now().Add(-time.Hour)
+	after := time.Now().Add(time.Hour)
+	ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, before, after, nil, nil, nil)
+	networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
+	_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "close-on-timer", before, after, networks, nil, nil)
+
+	caPath := filepath.Join(dir, "ca.pem")
+	certPath := filepath.Join(dir, "cert.pem")
+	keyPath := filepath.Join(dir, "key.pem")
+	require.NoError(t, os.WriteFile(caPath, caPEM, 0o600))
+	require.NoError(t, os.WriteFile(certPath, certPEM, 0o600))
+	require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600))
+
+	// tun disabled so no device/root is needed; routines: 2 so we exercise the
+	// multi-socket (SO_REUSEPORT) teardown, which is where dnclient runs.
+	configBody := fmt.Sprintf(`
+pki:
+  ca: %s
+  cert: %s
+  key: %s
+listen:
+  host: 127.0.0.1
+  port: 0
+tun:
+  disabled: true
+firewall:
+  outbound:
+    - port: any
+      proto: any
+      host: any
+  inbound:
+    - port: any
+      proto: any
+      host: any
+routines: 2
+`, caPath, certPath, keyPath)
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(configBody), 0o600))
+
+	c := config.NewC(l)
+	require.NoError(t, c.Load(dir))
+
+	ctrl, err := nebula.Main(c, false, "close-on-timer", l, nil)
+	require.NoError(t, err)
+	require.NoError(t, ctrl.Start())
+
+	// Run like a live nebula, then close on a timer, exactly as dnclient does.
+	<-time.NewTimer(5 * time.Second).C
+
+	stopped := make(chan struct{})
+	go func() {
+		ctrl.Stop() // closes the udp sockets (shutdown(2)) and the tun
+		ctrl.Wait() // blocks until every reader goroutine has returned
+		close(stopped)
+	}()
+
+	select {
+	case <-stopped:
+		t.Log("nebula closed cleanly on timer")
+	case <-time.After(10 * time.Second):
+		buf := make([]byte, 1<<20)
+		n := runtime.Stack(buf, true)
+		t.Fatalf("nebula did NOT close within 10s of Stop(): a blocking reader never woke\n%s", buf[:n])
+	}
+}

cmd/nebula/fips140enforce.go

@@ -0,0 +1,5 @@
+//go:build fips140enforce
+
+//go:debug fips140=only
+
+package main

connection_manager_test.go

@@ -25,6 +25,7 @@ func newTestLighthouse() *LightHouse {
 	lighthouses := []netip.Addr{}
 	staticList := map[netip.Addr]struct{}{}
 
+	lh.localAddrsFn = func(*LocalAllowList) []netip.Addr { return nil }
 	lh.lighthouses.Store(&lighthouses)
 	lh.staticList.Store(&staticList)
 

connection_state.go

@@ -2,11 +2,13 @@ package nebula
 
 import (
 	"encoding/json"
+	"log/slog"
 	"sync"
 	"sync/atomic"
 
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/handshake"
+	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/noiseutil"
 )
 
@@ -20,6 +22,7 @@ type ConnectionState struct {
 	initiator      bool
 	messageCounter atomic.Uint64
 	window         *Bits
+	decryptLock    sync.Mutex
 	writeLock      sync.Mutex
 }
 
@@ -54,3 +57,52 @@ func (cs *ConnectionState) MarshalJSON() ([]byte, error) {
 func (cs *ConnectionState) Curve() cert.Curve {
 	return cs.myCert.Curve()
 }
+
+func (cs *ConnectionState) Decrypt(l *slog.Logger, messageCounter uint64, out []byte, packet []byte, nb []byte) ([]byte, error) {
+	var err error
+	cs.decryptLock.Lock()
+	result := cs.window.Check(l, messageCounter)
+	cs.decryptLock.Unlock()
+	if !result {
+		return nil, ErrAlreadySeen
+	}
+
+	out, err = cs.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
+	if err != nil {
+		return nil, err
+	}
+
+	cs.decryptLock.Lock()
+	result = cs.window.Update(l, messageCounter)
+	cs.decryptLock.Unlock()
+	if !result {
+		return nil, ErrAlreadySeen
+	}
+	return out, nil
+}
+
+// VerifyRelay verifies AEAD protected (but not encrypted) relay frames. packet must be length-checked by the caller.
+func (cs *ConnectionState) VerifyRelay(l *slog.Logger, messageCounter uint64, packet []byte, nb []byte) error {
+	cs.decryptLock.Lock()
+	result := cs.window.Check(l, messageCounter)
+	cs.decryptLock.Unlock()
+	if !result {
+		return ErrAlreadySeen
+	}
+
+	signedPayload := packet[:len(packet)-cs.dKey.Overhead()]
+	signatureValue := packet[len(packet)-cs.dKey.Overhead():]
+	_, err := cs.dKey.DecryptDanger(nil, signedPayload, signatureValue, messageCounter, nb)
+	if err != nil {
+		return err
+	}
+
+	cs.decryptLock.Lock()
+	result = cs.window.Update(l, messageCounter)
+	cs.decryptLock.Unlock()
+	if !result {
+		return ErrAlreadySeen
+	}
+
+	return nil
+}

control.go

@@ -53,6 +53,7 @@ type Control struct {
 	statsStart             func()
 	dnsStart               func()
 	lighthouseStart        func()
+	networkChangeStart     func(rebind func())
 	connectionManagerStart func(context.Context)
 }
 
@@ -104,6 +105,9 @@ func (c *Control) Start() error {
 	if c.dnsStart != nil {
 		go c.dnsStart()
 	}
+	if c.networkChangeStart != nil {
+		go c.networkChangeStart(c.RebindUDPServer)
+	}
 	if c.connectionManagerStart != nil {
 		go c.connectionManagerStart(c.ctx)
 	}
@@ -198,7 +202,11 @@ func (c *Control) RebindUDPServer() {
 		return
 	}
 
-	_ = c.f.outside.Rebind()
+	// A failure here means we are likely still pinned to the interface we came up on, so the rest of this is
+	// unlikely to help. Say so instead of silently carrying on as if we rebound.
+	if err := c.f.outside.Rebind(); err != nil {
+		c.l.Error("Failed to rebind udp socket", "error", err)
+	}
 
 	// Trigger a lighthouse update, useful for mobile clients that should have an update interval of 0
 	c.f.lightHouse.SendUpdate()

control_lifecycle_test.go

@@ -11,7 +11,6 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/test"
 	"github.com/slackhq/nebula/udp"
@@ -31,9 +30,9 @@ func newFakeDevice() *fakeDevice {
 
 // Read blocks until Close like a real tun with no traffic, then reports EOF
 // the same way a closed device does
-func (d *fakeDevice) Read() ([]tio.Packet, error) {
+func (d *fakeDevice) Read(p []byte) (int, error) {
 	<-d.closedCh
-	return nil, io.EOF
+	return 0, io.EOF
 }
 
 func (d *fakeDevice) Write(p []byte) (int, error) { return len(p), nil }
@@ -50,8 +49,10 @@ func (d *fakeDevice) Activate() error                       { return nil }
 func (d *fakeDevice) Networks() []netip.Prefix              { return nil }
 func (d *fakeDevice) Name() string                          { return "fake" }
 func (d *fakeDevice) RoutesFor(netip.Addr) routing.Gateways { return nil }
-
+func (d *fakeDevice) SupportsMultiqueue() bool              { return false }
-func (d *fakeDevice) Queues(int) ([]tio.Queue, error) { return []tio.Queue{d}, nil }
+func (d *fakeDevice) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, errors.New("unsupported")
+}
 
 // newReadyControl hand-builds the minimum Control that Main would have
 // produced right before Start, including the construction token NewInterface
@@ -77,6 +78,7 @@ func newReadyControl(t *testing.T) (*Control, *fakeDevice, *fakeConn) {
 		inside:     dev,
 		outside:    conn,
 		writers:    []udp.Conn{conn},
+		readers:    make([]io.ReadWriteCloser, 1),
 		routines:   1,
 		hostMap:    newHostMap(l),
 		lightHouse: lh,
@@ -153,14 +155,7 @@ type multiqueueDevice struct {
 	*fakeDevice
 }
 
-// Queues claims multiqueue support but fails to open the second queue,
+func (d *multiqueueDevice) SupportsMultiqueue() bool { return true }
-// exercising the activation error path.
-func (d *multiqueueDevice) Queues(n int) ([]tio.Queue, error) {
-	if n > 1 {
-		return nil, errors.New("second queue failed to open")
-	}
-	return d.fakeDevice.Queues(n)
-}
 
 func TestControl_StartMultiqueueFailureReleases(t *testing.T) {
 	dev := &multiqueueDevice{fakeDevice: newFakeDevice()}
@@ -171,6 +166,7 @@ func TestControl_StartMultiqueueFailureReleases(t *testing.T) {
 		inside:   dev,
 		outside:  conn,
 		writers:  []udp.Conn{conn},
+		readers:  make([]io.ReadWriteCloser, 2),
 		routines: 2,
 		l:        test.NewLogger(),
 	}

control_tester.go

@@ -108,7 +108,19 @@ func (c *Control) GetVpnAddrs() []netip.Addr {
 }
 
 func (c *Control) GetUDPAddr() netip.AddrPort {
-	return c.f.outside.(*udp.TesterConn).Addr
+	return c.f.outside.(*udp.TesterConn).GetAddr()
+}
+
+// SetUDPAddr moves this node to a new underlay address, standing in for a laptop waking up on a different
+// network. Register the new address with the router as well or nothing will route back.
+func (c *Control) SetUDPAddr(addr netip.AddrPort) {
+	c.f.outside.(*udp.TesterConn).SetAddr(addr)
+}
+
+// SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of
+// whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker.
+func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {
+	c.f.lightHouse.localAddrsFn = fn
 }
 
 func (c *Control) KillPendingTunnel(vpnIp netip.Addr) bool {

dns_server.go

@@ -97,8 +97,7 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
 	newAddr := getDnsServerAddr(c)
 
 	d.serverMu.Lock()
-	running := d.server
+	running := d.server != nil
-	runningStarted := d.started
 	sameAddr := d.addr == newAddr
 	d.addr = newAddr
 	d.enabled.Store(enabled)
@@ -112,7 +111,7 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
 	}
 
 	if !enabled {
-		if running != nil {
+		if running {
 			d.Stop()
 		}
 		// Drop any records that accumulated while enabled; a later re-enable
@@ -121,12 +120,12 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
 		return nil
 	}
 
-	if running == nil {
+	if !running {
 		// Was disabled (or never started); bring it up now.
 		go d.Start()
 	} else if !sameAddr {
-		d.shutdownServer(running, runningStarted, "reload")
+		// Stop clears the slot before shutting down, otherwise the Start below can find the dying server and refuse
-		// Old Start goroutine has now exited; bring up a fresh listener on the new address.
+		d.Stop()
 		go d.Start()
 	}
 
@@ -162,7 +161,9 @@ func (d *dnsServer) Start() {
 
 	started := make(chan struct{})
 	d.serverMu.Lock()
-	if d.ctx.Err() != nil {
+	// Re-check enabled under the lock, a disable that raced our check above snapshots the slot under it too.
+	// Two reloads in quick succession can both spawn a Start, the loser would orphan the live listener past Stop
+	if d.ctx.Err() != nil || d.server != nil || !d.enabled.Load() {
 		d.serverMu.Unlock()
 		return
 	}
@@ -200,6 +201,14 @@ func (d *dnsServer) Start() {
 		close(started)
 	}
 
+	// Release our slot, unless a reload already replaced us, so a dead listener can't block a future Start
+	d.serverMu.Lock()
+	if d.server == server {
+		d.server = nil
+		d.started = nil
+	}
+	d.serverMu.Unlock()
+
 	if err != nil {
 		d.l.Warn("Failed to run the DNS responder", "error", err)
 	}

dns_server_test.go

@@ -194,14 +194,51 @@ func TestDnsServer_reload_initial_serveDnsWithoutLighthouse(t *testing.T) {
 }
 
 func TestDnsServer_reload_sameAddr_noOp(t *testing.T) {
+	port := freeUDPPort(t)
 	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", "0", true, true)
+	setDnsConfig(c, "127.0.0.1", port, true, true)
-
 	require.NoError(t, ds.reload(c, true))
-	// No server running yet, no addr change. Reload should not spawn anything.
+
+	go ds.Start()
+	waitForBind(t, ds)
+
+	ds.serverMu.Lock()
+	before := ds.server
+	ds.serverMu.Unlock()
+	require.NotNil(t, before)
+
+	// Same address, so the running listener must be left alone rather than rebuilt under live queries
 	require.NoError(t, ds.reload(c, false))
 	assert.True(t, ds.enabled.Load())
-	assert.Nil(t, ds.server)
+
+	ds.serverMu.Lock()
+	after := ds.server
+	ds.serverMu.Unlock()
+	assert.Same(t, before, after, "a same-address reload must not restart the listener")
+
+	ds.Stop()
+}
+
+// The branch the old sameAddr test was accidentally hitting: enabled with nothing running means reload starts it.
+func TestDnsServer_reload_whenNotRunning_starts(t *testing.T) {
+	port := freeUDPPort(t)
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", port, true, true)
+
+	// initial only records config, it never starts anything
+	require.NoError(t, ds.reload(c, true))
+	ds.serverMu.Lock()
+	assert.Nil(t, ds.server, "the initial reload must not start a listener")
+	ds.serverMu.Unlock()
+
+	require.NoError(t, ds.reload(c, false))
+	waitForBind(t, ds)
+
+	ds.serverMu.Lock()
+	assert.NotNil(t, ds.server, "a reload with nothing running should bring DNS up")
+	ds.serverMu.Unlock()
+
+	ds.Stop()
 }
 
 func TestDnsServer_StartStop_lifecycle(t *testing.T) {
@@ -427,3 +464,168 @@ func waitFor(t *testing.T, cond func() bool) {
 	}
 	t.Fatal("timed out waiting for condition")
 }
+
+// Two reloads in quick succession, or a HUP before Control.Start, can race two Starts at the same listener.
+func TestDnsServer_Start_isIdempotent(t *testing.T) {
+	port := freeUDPPort(t)
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", port, true, true)
+	require.NoError(t, ds.reload(c, true))
+
+	go ds.Start()
+	waitForBind(t, ds)
+
+	ds.serverMu.Lock()
+	first := ds.server
+	ds.serverMu.Unlock()
+	require.NotNil(t, first)
+
+	// If the second Start replaces the tracked server, Stop kills the wrong one and the port leaks
+	done := make(chan struct{})
+	go func() {
+		ds.Start()
+		close(done)
+	}()
+	select {
+	case <-done:
+	case <-time.After(time.Second * 5):
+		t.Fatal("second Start never returned")
+	}
+
+	ds.serverMu.Lock()
+	second := ds.server
+	ds.serverMu.Unlock()
+	assert.Same(t, first, second, "a second Start must not replace the running server")
+
+	// The real proof, after Stop the port must actually be free
+	ds.Stop()
+	waitFor(t, func() bool {
+		pc, err := net.ListenPacket("udp", "127.0.0.1:"+port)
+		if err != nil {
+			return false
+		}
+		_ = pc.Close()
+		return true
+	})
+}
+
+// An address change must actually end up listening on the new port. Start's guard refuses when a server is already
+// installed, so reload has to clear the slot before shutting the old one down.
+func TestDnsServer_reload_addrChange_restarts(t *testing.T) {
+	first := freeUDPPort(t)
+	second := freeUDPPort(t)
+
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", first, true, true)
+	require.NoError(t, ds.reload(c, true))
+
+	go ds.Start()
+	waitForBind(t, ds)
+
+	// Cycle a few times, the failure this guards against depends on which goroutine wins serverMu
+	for i := range 8 {
+		want := second
+		if i%2 == 1 {
+			want = first
+		}
+		setDnsConfig(c, "127.0.0.1", want, true, true)
+		require.NoError(t, ds.reload(c, false))
+		waitForBind(t, ds)
+
+		ds.serverMu.Lock()
+		srv := ds.server
+		ds.serverMu.Unlock()
+		require.NotNil(t, srv, "reload left DNS down instead of restarting it")
+		require.Equal(t, "127.0.0.1:"+want, srv.Addr, "reload should be serving the new address")
+	}
+
+	// Land back on second so the port assertions below are meaningful
+	setDnsConfig(c, "127.0.0.1", second, true, true)
+	require.NoError(t, ds.reload(c, false))
+	waitForBind(t, ds)
+
+	// The old port must be released and the new one actually held
+	waitFor(t, func() bool {
+		pc, err := net.ListenPacket("udp", "127.0.0.1:"+first)
+		if err != nil {
+			return false
+		}
+		_ = pc.Close()
+		return true
+	})
+	_, err := net.ListenPacket("udp", "127.0.0.1:"+second)
+	require.Error(t, err, "the new address should be bound by the DNS responder")
+
+	ds.Stop()
+}
+
+// A listener that dies on its own must release the slot, or a later same-addr reload sees it as running and no-ops.
+func TestDnsServer_Start_bindFailure_releasesSlot(t *testing.T) {
+	port := freeUDPPort(t)
+	blocker, err := net.ListenPacket("udp", "127.0.0.1:"+port)
+	require.NoError(t, err)
+
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", port, true, true)
+	require.NoError(t, ds.reload(c, true))
+
+	ds.Start() // returns once the bind fails
+
+	ds.serverMu.Lock()
+	assert.Nil(t, ds.server, "a listener that failed to bind must not stay parked in the slot")
+	ds.serverMu.Unlock()
+
+	// With the slot released, a reload can retry once the port frees up
+	require.NoError(t, blocker.Close())
+	require.NoError(t, ds.reload(c, false))
+	waitForBind(t, ds)
+
+	ds.serverMu.Lock()
+	assert.NotNil(t, ds.server, "a same-addr reload should retry after a failed bind")
+	ds.serverMu.Unlock()
+
+	ds.Stop()
+}
+
+// A disable that lands while Start is between its unlocked check and the guard must not leave a listener behind.
+func TestDnsServer_Start_refusesWhenDisabledUnderLock(t *testing.T) {
+	port := freeUDPPort(t)
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", port, true, true)
+	require.NoError(t, ds.reload(c, true))
+	require.True(t, ds.enabled.Load())
+
+	// Holding serverMu parks Start on the lock, the only way to land the disable in that window on purpose
+	ds.serverMu.Lock()
+
+	done := make(chan struct{})
+	go func() {
+		ds.Start()
+		close(done)
+	}()
+
+	select {
+	case <-done:
+		ds.serverMu.Unlock()
+		t.Fatal("Start returned early, the test never exercised the window")
+	case <-time.After(time.Millisecond * 100):
+	}
+
+	// The disable reload's critical section. It sees nothing running, so it never calls Stop.
+	ds.enabled.Store(false)
+	ds.serverMu.Unlock()
+
+	select {
+	case <-done:
+	case <-time.After(time.Second * 5):
+		t.Fatal("Start never returned")
+	}
+
+	ds.serverMu.Lock()
+	assert.Nil(t, ds.server, "Start must not install a listener a disable already cancelled")
+	ds.serverMu.Unlock()
+
+	pc, err := net.ListenPacket("udp", "127.0.0.1:"+port)
+	require.NoError(t, err, "an orphaned listener is still holding the port")
+	_ = pc.Close()
+}

e2e/handshakes_test.go

@@ -725,6 +725,70 @@ func TestReestablishRelays(t *testing.T) {
 
 }
 
+func TestRelayHandshakeOverDisestablishedEntry(t *testing.T) {
+	t.Parallel()
+	// If them tears down the tunnel while me keeps Established relay state, me's next
+	// handshake flows through the relay with no fresh CreateRelayRequest and lands on
+	// them's Disestablished terminal relay entry. them must re-establish that entry, or
+	// its first transmit deletes its only relay and the tunnel is born transmit-dead:
+	// them can receive but every send is silently dropped.
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
+	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay  ", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them   ", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
+
+	// Teach my how to get to the relay and that their can be reached via the relay
+	myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
+	myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
+	relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+
+	// Build a router so we don't have to reason who gets which packet
+	r := router.NewR(t, myControl, relayControl, theirControl)
+	defer r.RenderFlow()
+
+	// Start the servers
+	myControl.Start()
+	relayControl.Start()
+	theirControl.Start()
+
+	t.Log("Trigger a handshake from me to them via the relay")
+	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
+
+	p := r.RouteForAllUntilTxTun(theirControl)
+	assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
+	oldIdx := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false).LocalIndex
+
+	t.Log("Close the tunnel on them only, marking their relay entry Disestablished")
+	theirControl.CloseTunnel(myVpnIpNet[0].Addr(), true)
+
+	t.Log("Re-handshake from me, riding the still-Established relay state")
+	myControl.ReHandshake(theirVpnIpNet[0].Addr())
+	for {
+		h := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
+		if h != nil && h.LocalIndex != oldIdx && h.RemoteIndex != 0 {
+			break
+		}
+		r.RouteForAllExitFunc(func(*udp.Packet, *nebula.Control) router.ExitType {
+			return router.RouteAndExit
+		})
+	}
+
+	hAtThem := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
+	require.NotNil(t, hAtThem, "them should have completed the relayed handshake")
+	require.Equal(t, []netip.Addr{relayVpnIpNet[0].Addr()}, hAtThem.CurrentRelaysToMe, "them should know a relay for the new tunnel")
+
+	t.Log("Send from them to me; their only relay entry must survive the transmit")
+	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them")))
+	require.Never(t, func() bool {
+		h := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
+		return h == nil || len(h.CurrentRelaysToMe) == 0
+	}, time.Second, 10*time.Millisecond, "them deleted its only relay entry; the tunnel is permanently transmit-dead")
+
+	p = r.RouteForAllUntilTxTun(myControl)
+	assertUdpPacket(t, []byte("Hi from them"), p, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), 80, 80)
+	r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
+}
+
 func TestStage1RaceRelays(t *testing.T) {
 	t.Parallel()
 	//NOTE: this is a race between me and relay resulting in a full tunnel from me to them via relay

e2e/rebind_test.go

@@ -0,0 +1,225 @@
+//go:build e2e_testing
+// +build e2e_testing
+
+package e2e
+
+import (
+	"net/netip"
+	"testing"
+	"time"
+
+	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/cert_test"
+	"github.com/slackhq/nebula/e2e/router"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/udp"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// reportedAddrs is what the lighthouse would hand a peer asking where vpnAddr is.
+func reportedAddrs(t *testing.T, lh *nebula.Control, vpnAddr netip.Addr) []netip.AddrPort {
+	t.Helper()
+	cm := lh.QueryLighthouse(vpnAddr)
+	if cm == nil {
+		return nil
+	}
+	var out []netip.AddrPort
+	for _, c := range *cm {
+		out = append(out, c.Reported...)
+		out = append(out, c.Learned...)
+	}
+	return out
+}
+
+// waitForLighthouseMsg routes until a lighthouse message lands on lh, or gives up. Reports whether one arrived.
+func waitForLighthouseMsg(t *testing.T, r *router.R, lh *nebula.Control, wait time.Duration) bool {
+	t.Helper()
+	h := &header.H{}
+	return r.RouteForAllExitFuncOrTimeout(wait, func(p *udp.Packet, c *nebula.Control) router.ExitType {
+		if c != lh {
+			return router.KeepRouting
+		}
+		// Punches are a single byte and never parse, they are just not what we are after
+		if err := h.Parse(p.Data); err != nil {
+			return router.KeepRouting
+		}
+		if h.Type == header.LightHouse {
+			return router.RouteAndExit
+		}
+		return router.KeepRouting
+	})
+}
+
+// A laptop that changes networks has to tell the lighthouse promptly, otherwise the lighthouse keeps handing peers
+// the old address and their punches land nowhere. On a long lighthouse interval the only thing that closes that
+// window is the rebind, which on darwin the network change monitor drives. The e2e build compiles the monitor out,
+// so we call RebindUDPServer directly, which is the same thing the monitor does.
+func TestRebindSendsLighthouseUpdate(t *testing.T) {
+	t.Parallel()
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
+	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
+		"lighthouse": m{"am_lighthouse": true},
+	})
+
+	// 600s interval, so nothing scheduled can send an update during this test. A rebind is the only thing that can.
+	myControl, _, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{
+		"lighthouse": m{
+			"hosts":    []any{lhVpnIpNet[0].Addr().String()},
+			"interval": 600,
+		},
+		"static_host_map": m{
+			lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
+		},
+	})
+
+	r := router.NewR(t, lhControl, myControl)
+	defer r.RenderFlow()
+
+	lhControl.Start()
+	myControl.Start()
+
+	// Let the startup registration finish, then clear everything it left behind
+	require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an initial registration")
+	r.RouteFor(time.Millisecond * 400)
+
+	// Nothing should be talking to the lighthouse on its own now
+	require.False(t, waitForLighthouseMsg(t, r, lhControl, time.Millisecond*200),
+		"nothing should reach the lighthouse before the rebind")
+
+	myControl.RebindUDPServer()
+
+	assert.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5),
+		"a rebind should push an update to the lighthouse rather than waiting out the interval")
+
+	lhControl.Stop()
+	myControl.Stop()
+}
+
+// The other half of a rebind: every live tunnel requeries the lighthouse on its next send. That query is what makes
+// the lighthouse tell the peer to punch toward our new address, which is the part that actually revives a tunnel
+// whose remote NAT state died while we were on a different network.
+func TestRebindRequeriesPeersOnNextSend(t *testing.T) {
+	t.Parallel()
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
+	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
+		"lighthouse": m{"am_lighthouse": true},
+	})
+
+	lhCfg := m{
+		"lighthouse": m{
+			"hosts":    []any{lhVpnIpNet[0].Addr().String()},
+			"interval": 600,
+			// Without this the peers advertise this machine's real addresses and then try to punch at them,
+			// which the router has no route for.
+			"local_allow_list": m{
+				"10.0.0.0/24": true,
+				"::/0":        false,
+			},
+		},
+		"static_host_map": m{
+			lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
+		},
+	}
+
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", lhCfg)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.3/24", lhCfg)
+
+	r := router.NewR(t, lhControl, myControl, theirControl)
+	defer r.RenderFlow()
+
+	lhControl.Start()
+	myControl.Start()
+	theirControl.Start()
+	r.RouteFor(time.Millisecond * 500)
+
+	// Point the peers at each other directly, this test is about the rebind and not about lighthouse discovery
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("initial")))
+	r.RouteFor(time.Second)
+	require.NotNil(t, myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false), "expected a tunnel to them")
+	r.RouteFor(time.Millisecond * 300)
+
+	// Assert on what the peer sees rather than on lighthouse traffic. A query for them makes the lighthouse send
+	// them a punch notification, which is the whole point. Our own update to the lighthouse sends them nothing,
+	// so this cannot be satisfied by the update the rebind itself pushes.
+	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("quiet")))
+	require.False(t, waitForLighthouseMsg(t, r, theirControl, time.Millisecond*300),
+		"an ordinary send should not requery the lighthouse")
+
+	myControl.RebindUDPServer()
+	r.RouteFor(time.Millisecond * 300) // let the update the rebind itself sends pass by
+
+	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("after rebind")))
+	assert.True(t, waitForLighthouseMsg(t, r, theirControl, time.Second*5),
+		"the first send after a rebind should requery the lighthouse, which then tells the peer to punch at us")
+
+	lhControl.Stop()
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+// The scenario this whole thing exists for: a laptop sleeps at the office and wakes up at home on a new address.
+// Until it tells the lighthouse, the lighthouse keeps handing peers the office address, so their punches land
+// nowhere and the tunnel stays dead. On a long interval the rebind is the only thing that closes that window.
+func TestRebindAdvertisesNewAddressAfterMove(t *testing.T) {
+	t.Parallel()
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
+	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
+		"lighthouse": m{"am_lighthouse": true},
+	})
+
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{
+		"lighthouse": m{
+			"hosts":    []any{lhVpnIpNet[0].Addr().String()},
+			"interval": 600,
+		},
+		"static_host_map": m{
+			lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
+		},
+	})
+
+	// Advertise wherever we currently are rather than this machine's real NICs, read fresh each time so a move
+	// is picked up.
+	myControl.SetLocalAddrsFn(func(*nebula.LocalAllowList) []netip.Addr {
+		return []netip.Addr{myControl.GetUDPAddr().Addr()}
+	})
+
+	r := router.NewR(t, lhControl, myControl)
+	defer r.RenderFlow()
+
+	lhControl.Start()
+	myControl.Start()
+
+	require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an initial registration")
+	r.RouteFor(time.Millisecond * 400)
+
+	require.Contains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), myUdpAddr,
+		"the lighthouse should know the address we started on")
+
+	// Wake up somewhere else
+	newAddr := netip.MustParseAddrPort("10.0.0.99:4242")
+	myControl.SetUDPAddr(newAddr)
+	r.AddRoute(newAddr.Addr(), newAddr.Port(), myControl)
+
+	// Nothing has told the lighthouse, and with interval 600 nothing scheduled will
+	r.RouteFor(time.Millisecond * 400)
+	require.NotContains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), newAddr,
+		"the lighthouse should still be handing out the old address before the rebind")
+
+	myControl.RebindUDPServer()
+	require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an update after the rebind")
+	r.RouteFor(time.Millisecond * 400)
+
+	assert.Contains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), newAddr,
+		"after the rebind the lighthouse should hand peers our new address")
+
+	lhControl.Stop()
+	myControl.Stop()
+}

e2e/router/router.go

@@ -114,6 +114,28 @@ type packet struct {
 	packet *udp.Packet
 	tun    bool // a packet pulled off a tun device
 	rx     bool // the packet was received by a udp device
+
+	// h is the nebula header, parsed once when the packet is recorded. parseErr says why there isn't one, which
+	// the flow log reports rather than hiding. Punchy sends a single byte, so an unparseable packet is normal.
+	h        header.H
+	parseErr error
+}
+
+// fromAddr and toAddr are the addresses this packet actually travelled between. Reading them off the control
+// instead would misreport the whole history once a test moves a node. Tun packets are synthesized without
+// addresses, so they fall back to the control.
+func (p *packet) fromAddr() netip.AddrPort {
+	if p.tun || !p.packet.From.IsValid() {
+		return p.from.GetUDPAddr()
+	}
+	return p.packet.From
+}
+
+func (p *packet) toAddr() netip.AddrPort {
+	if p.tun || !p.packet.To.IsValid() {
+		return p.to.GetUDPAddr()
+	}
+	return p.packet.To
 }
 
 func (p *packet) WasReceived() {
@@ -249,7 +271,7 @@ func (r *R) renderFlow() {
 			continue
 		}
 
-		addr := e.packet.from.GetUDPAddr()
+		addr := e.packet.fromAddr()
 		if _, ok := participants[addr]; ok {
 			continue
 		}
@@ -268,7 +290,6 @@ func (r *R) renderFlow() {
 	}
 
 	// Print packets
-	h := &header.H{}
 	for _, e := range r.flow {
 		if e.packet == nil {
 			//fmt.Fprintf(f, "    note over %s: %s\n", strings.Join(participantsVals, ", "), e.note)
@@ -280,21 +301,22 @@ func (r *R) renderFlow() {
 			fmt.Fprintln(f, r.formatUdpPacket(p))
 
 		} else {
-			if err := h.Parse(p.packet.Data); err != nil {
-				panic(err)
-			}
-
 			line := "--x"
 			if p.rx {
 				line = "->>"
 			}
 
-			fmt.Fprintf(f,
+			detail := fmt.Sprintf("%s(%s), index %v, counter: %v",
-				"    %s%s%s: %s(%s), index %v, counter: %v\n",
+				p.h.TypeName(), p.h.SubTypeName(), p.h.RemoteIndex, p.h.MessageCounter)
-				normalizeName(p.from.GetUDPAddr().String()),
+			if p.parseErr != nil {
+				detail = fmt.Sprintf("unparsed, %v (%d bytes)", p.parseErr, len(p.packet.Data))
+			}
+
+			fmt.Fprintf(f, "    %s%s%s: %s\n",
+				normalizeName(p.fromAddr().String()),
 				line,
-				normalizeName(p.to.GetUDPAddr().String()),
+				normalizeName(p.toAddr().String()),
-				h.TypeName(), h.SubTypeName(), h.RemoteIndex, h.MessageCounter,
+				detail,
 			)
 		}
 	}
@@ -408,21 +430,24 @@ func (r *R) unlockedInjectFlow(from, to *nebula.Control, p *udp.Packet, tun bool
 
 	r.renderHostmaps(fmt.Sprintf("Packet %v", len(r.flow)))
 
-	if len(r.ignoreFlows) > 0 {
 	var h header.H
-		err := h.Parse(p.Data)
+	var parseErr error
-		if err != nil {
+	if !tun {
-			panic(err)
+		parseErr = h.Parse(p.Data)
 	}
 
+	// Decide before copying, the copy comes from a freelist and an ignored packet would never be released
 	for _, i := range r.ignoreFlows {
-			if !tun {
+		if tun {
-				if i.messageType == h.Type && i.subType == h.Subtype {
+			if i.tun.HasValue && i.tun.IsTrue {
 				return nil
 			}
-			} else if i.tun.HasValue && i.tun.IsTrue {
+			continue
-				return nil
 		}
+
+		// A packet we could not parse has no type to match against, so no rule can ignore it
+		if parseErr == nil && i.messageType == h.Type && i.subType == h.Subtype {
+			return nil
 		}
 	}
 
@@ -431,6 +456,8 @@ func (r *R) unlockedInjectFlow(from, to *nebula.Control, p *udp.Packet, tun bool
 		to:       to,
 		packet:   p.Copy(),
 		tun:      tun,
+		h:        h,
+		parseErr: parseErr,
 	}
 
 	r.flow = append(r.flow, flowEntry{packet: fp})
@@ -690,6 +717,81 @@ func (r *R) RouteUntilAfterMsgType(sender *nebula.Control, msgType header.Messag
 	})
 }
 
+// RouteFor routes everything that shows up for the given duration and then returns. Use it to let a test settle
+// deterministically rather than sleeping and hoping: a single FlushAll races a completing handshake, which queues
+// more packets right behind it.
+func (r *R) RouteFor(d time.Duration) {
+	r.RouteForAllExitFuncOrTimeout(d, func(*udp.Packet, *nebula.Control) ExitType {
+		return KeepRouting
+	})
+}
+
+// RouteForAllExitFuncOrTimeout is RouteForAllExitFunc with a deadline, reporting whether whatDo asked to exit
+// before time ran out. The unbounded version blocks forever on a quiet network, so this is what a test needs to
+// assert that something does NOT happen, or to route for a fixed settling period.
+func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc) bool {
+	sc := make([]reflect.SelectCase, 0, len(r.controls)+1)
+	cm := make([]*nebula.Control, 0, len(r.controls))
+
+	for _, c := range r.controls {
+		sc = append(sc, reflect.SelectCase{
+			Dir:  reflect.SelectRecv,
+			Chan: reflect.ValueOf(c.GetUDPTxChan()),
+			Send: reflect.Value{},
+		})
+		cm = append(cm, c)
+	}
+
+	timer := time.NewTimer(timeout)
+	defer timer.Stop()
+	sc = append(sc, reflect.SelectCase{
+		Dir:  reflect.SelectRecv,
+		Chan: reflect.ValueOf(timer.C),
+		Send: reflect.Value{},
+	})
+
+	for {
+		x, rx, _ := reflect.Select(sc)
+		if x == len(cm) {
+			return false
+		}
+
+		r.Lock()
+		p := rx.Interface().(*udp.Packet)
+		receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
+		if receiver == nil {
+			r.Unlock()
+			panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String())
+		}
+
+		e := whatDo(p, receiver)
+		switch e {
+		case ExitNow:
+			r.Unlock()
+			p.Release()
+			return true
+
+		case RouteAndExit:
+			fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
+			receiver.InjectUDPPacket(p)
+			fp.WasReceived()
+			r.Unlock()
+			p.Release()
+			return true
+
+		case KeepRouting:
+			fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
+			receiver.InjectUDPPacket(p)
+			fp.WasReceived()
+
+		default:
+			panic(fmt.Sprintf("Unknown exitFunc return: %v", e))
+		}
+		r.Unlock()
+		p.Release()
+	}
+}
+
 func (r *R) RouteForAllUntilAfterMsgTypeTo(receiver *nebula.Control, msgType header.MessageType, subType header.MessageSubType) {
 	h := &header.H{}
 	r.RouteForAllExitFunc(func(p *udp.Packet, r *nebula.Control) ExitType {

examples/config.yml

@@ -146,6 +146,14 @@ listen:
   # Default true; set to false to leave WDF in charge of inbound decisions on the listener port. Not reloadable.
   #windows_bypass_wdf: true
 
+  # On macOS only
+  # macOS scopes the udp socket to the interface it was created on, so moving between networks (wifi to wired,
+  # office to home) leaves Nebula sending out an interface that no longer has a route. When true, Nebula watches
+  # the routing socket and rebinds the listener once the change settles.
+  # iOS does not use this, the host app drives the same rebind itself.
+  # Default true. Not reloadable.
+  #rebind_on_network_change: true
+
   # By default, Nebula replies to packets it has no tunnel for with a "recv_error" packet. This packet helps speed up reconnection
   # in the case that Nebula on either side did not shut down cleanly. This response can be abused as a way to discover if Nebula is running
   # on a host though. This option lets you configure if you want to send "recv_error" packets always, never, or only to private network remotes.
@@ -254,20 +262,6 @@ tun:
   # Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic
   mtu: 1300
 
-  # Linux only. pin_threads pins each tun reader/encrypt OS thread to a single CPU. This keeps every goroutine's
-  # sends flowing through one XPS-selected NIC TX ring, so packets within a flow stay ordered on the wire
-  # instead of being sprayed across multiple TX rings and reordered. Not reloadable.
-  #pin_threads: true
-
-  # Linux only. cpu_affinity overrides which CPUs the tun reader threads pin to: a list of CPU IDs, one per routine
-  # (see the top-level `routines` setting). Lists shorter than `routines` are modulo-cycled across the queues; extra
-  # entries are ignored. IDs must be within the process's allowed CPU set, so this respects taskset / cgroup cpusets;
-  # a non-integer or not-allowed entry disables the override and falls back to spreading queues across the allowed
-  # CPUs. Only meaningful while pin_threads is true. Not reloadable.
-  #cpu_affinity:
-  #  - 2
-  #  - 4
-
   # Route based MTU overrides, you have known vpn ip paths that can support larger MTUs you can increase/decrease them here
   routes:
     #- mtu: 8800

examples/service_scripts/nebula.service

@@ -8,6 +8,15 @@ Before=sshd.service
 Type=notify
 NotifyAccess=main
 SyslogIdentifier=nebula
+
+# Uncomment to run as an unprivileged user with only CAP_NET_ADMIN. Requires a
+# nebula user that owns the config directory. Add CAP_NET_BIND_SERVICE to both
+# lines if any listener (lighthouse DNS, listen.port, stats, sshd) binds <1024.
+#User=nebula
+#Group=nebula
+#CapabilityBoundingSet=CAP_NET_ADMIN
+#AmbientCapabilities=CAP_NET_ADMIN
+
 ExecReload=/bin/kill -HUP $MAINPID
 ExecStart=/usr/local/bin/nebula -config /etc/nebula/config.yml
 Restart=always

go.mod

@@ -1,6 +1,6 @@
 module github.com/slackhq/nebula
 
-go 1.25.0
+go 1.26.0
 
 require (
 	dario.cat/mergo v1.0.2
@@ -24,12 +24,12 @@ require (
 	github.com/vishvananda/netlink v1.3.1
 	go.uber.org/goleak v1.3.0
 	go.yaml.in/yaml/v3 v3.0.4
-	golang.org/x/crypto v0.53.0
+	golang.org/x/crypto v0.54.0
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
-	golang.org/x/net v0.56.0
+	golang.org/x/net v0.57.0
-	golang.org/x/sync v0.21.0
+	golang.org/x/sync v0.22.0
-	golang.org/x/sys v0.46.0
+	golang.org/x/sys v0.47.0
-	golang.org/x/term v0.44.0
+	golang.org/x/term v0.45.0
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
 	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
 	golang.zx2c4.com/wireguard/windows v1.0.1

go.sum

@@ -162,8 +162,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
+golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
-golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
+golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
@@ -182,8 +182,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
+golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
-golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
+golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -191,8 +191,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
+golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
-golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -208,11 +208,11 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
+golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
-golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
+golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
-golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
+golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

handshake_manager.go

@@ -295,7 +295,13 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 		hm.messageMetrics.Tx(header.Handshake, hh.machine.Subtype(), 1)
 		err := hm.outside.WriteTo(stage0, addr)
 		if err != nil {
-			hostinfo.logger(hm.l).Error("Failed to send handshake message",
+			// These repeat every attempt, so match the success log below and only shout when the remotes changed
+			level := slog.LevelDebug
+			if remotesHaveChanged {
+				level = slog.LevelError
+			}
+
+			hostinfo.logger(hm.l).Log(context.Background(), level, "Failed to send handshake message",
 				"udpAddr", addr,
 				"initiatorIndex", hostinfo.localIndexId,
 				"handshake", hsFields,
@@ -529,8 +535,10 @@ func (hm *HandshakeManager) DeleteHostInfo(hostinfo *HostInfo) {
 
 func (hm *HandshakeManager) unlockedDeleteHostInfo(hostinfo *HostInfo) {
 	for _, addr := range hostinfo.vpnAddrs {
+		if cur, ok := hm.vpnIps[addr]; ok && cur.hostinfo == hostinfo {
 			delete(hm.vpnIps, addr)
 		}
+	}
 
 	if len(hm.vpnIps) == 0 {
 		hm.vpnIps = map[netip.Addr]*HandshakeHostInfo{}
@@ -1077,7 +1085,7 @@ func (hm *HandshakeManager) sendHandshakeResponse(via ViaSender, msg []byte, hos
 		hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
 		// We received a valid handshake on this relay, so make sure the relay
 		// state reflects that, in case it had been marked Disestablished.
-		via.relayHI.relayState.UpdateRelayForByIdxState(via.remoteIdx, Established)
+		via.relayHI.relayState.UpdateRelayForByIdxState(via.relay.LocalIndex, Established)
 		f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false)
 		f.l.Info("Handshake message sent", append(logFields, "relay", via.relayHI.vpnAddrs[0])...)
 	}

hostmap.go

@@ -287,7 +287,6 @@ type HostInfo struct {
 type ViaSender struct {
 	UdpAddr   netip.AddrPort
 	relayHI   *HostInfo // relayHI is the host info object of the relay
-	remoteIdx uint32    // remoteIdx is the index included in the header of the received packet
 	relay     *Relay    // relay contains the rest of the relay information, including the PeerIP of the host trying to communicate with us.
 	IsRelayed bool      // IsRelayed is true if the packet was sent through a relay
 }

inside.go

@@ -37,7 +37,7 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 		// routes packets from the Nebula addr to the Nebula addr through the Nebula
 		// TUN device.
 		if immediatelyForwardToSelf {
-			_, err := f.queues[q].Write(packet)
+			_, err := f.readers[q].Write(packet)
 			if err != nil {
 				f.l.Error("Failed to forward to tun", "error", err)
 			}
@@ -96,7 +96,7 @@ func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
 		return
 	}
 
-	_, err := f.queues[q].Write(out)
+	_, err := f.readers[q].Write(out)
 	if err != nil {
 		f.l.Error("Failed to write to tun", "error", err)
 	}
@@ -408,7 +408,7 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 		if err != nil {
 			hostinfo.logger(f.l).Error("Failed to write outgoing packet",
 				"error", err,
-				"udpAddr", remote,
+				"udpAddr", hr,
 			)
 		}
 	} else {

interface.go

@@ -2,11 +2,12 @@ package nebula
 
 import (
 	"context"
+	"crypto/fips140"
 	"errors"
 	"fmt"
+	"io"
 	"log/slog"
 	"net/netip"
-	"runtime"
 	"slices"
 	"sync"
 	"sync/atomic"
@@ -20,9 +21,7 @@ import (
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/overlay"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/udp"
-	"github.com/slackhq/nebula/util"
 )
 
 const mtu = 9001
@@ -51,18 +50,6 @@ type InterfaceConfig struct {
 	reQueryWait     time.Duration
 
 	ConntrackCacheTimeout time.Duration
-
-	// CpuAffinity, when non-empty, names the CPUs each TUN reader goroutine
-	// should pin to. Queue i pins to CpuAffinity[i % len(CpuAffinity)] —
-	// shorter lists than `routines` cycle. Empty list keeps the default
-	// pin-to-(i % NumCPU) behavior. Only consulted when PinThreads is true.
-	CpuAffinity []int
-	// PinThreads controls whether each TUN reader OS thread is pinned to a
-	// single CPU (via tun.pin_threads, default true). Pinning keeps each
-	// goroutine's UDP sends on one XPS-selected NIC TX ring so per-flow
-	// packets stay ordered on the wire.
-	PinThreads bool
-
 	l                     *slog.Logger
 }
 
@@ -87,15 +74,6 @@ type Interface struct {
 	routines              int
 	disconnectInvalid     atomic.Bool
 	closed                atomic.Bool
-	// cpuAffinity, when non-empty, names the CPUs each TUN reader goroutine
-	// should pin to. Queue i pins to cpuAffinity[i % len(cpuAffinity)].
-	// Empty falls back to the default pin-to-(allowed CPU) behavior.
-	// Only consulted when pinThreads is true.
-	cpuAffinity []int
-	// pinThreads controls whether listenIn pins each TUN reader OS thread to
-	// a CPU at all (tun.pin_threads, default true). When false, threads are
-	// left free to migrate as on stock nebula.
-	pinThreads   bool
 	relayManager          *relayManager
 
 	tryPromoteEvery atomic.Uint32
@@ -113,7 +91,7 @@ type Interface struct {
 
 	ctx     context.Context
 	writers []udp.Conn
-	queues  []tio.Queue
+	readers []io.ReadWriteCloser
 	wg      sync.WaitGroup
 
 	// fatalErr holds the first unexpected reader error that caused shutdown.
@@ -212,6 +190,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 		routines:              c.routines,
 		version:               c.version,
 		writers:               make([]udp.Conn, c.routines),
+		readers:               make([]io.ReadWriteCloser, c.routines),
 		myVpnNetworks:         cs.myVpnNetworks,
 		myVpnNetworksTable:    cs.myVpnNetworksTable,
 		myVpnAddrs:            cs.myVpnAddrs,
@@ -220,8 +199,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 		relayManager:          c.relayManager,
 		connectionManager:     c.connectionManager,
 		conntrackCacheTimeout: c.ConntrackCacheTimeout,
-		cpuAffinity:           c.CpuAffinity,
-		pinThreads:            c.PinThreads,
 
 		metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)),
 		messageMetrics:   c.MessageMetrics,
@@ -262,28 +239,31 @@ func (f *Interface) activate() error {
 		"build", f.version,
 		"udpAddr", addr,
 		"boringcrypto", boringEnabled(),
+		"fips140Version", fips140.Version(),
+		"fips140Enabled", fips140.Enabled(),
+		"fips140Enforced", fips140.Enforced(),
 	)
 
-	if f.routines > 1 && !f.outside.SupportsMultipleReaders() {
+	if f.routines > 1 {
+		if !f.inside.SupportsMultiqueue() || !f.outside.SupportsMultipleReaders() {
 			f.routines = 1
-		f.l.Warn("multiple udp readers are not supported on this platform, falling back to a single routine")
+			f.l.Warn("routines is not supported on this platform, falling back to a single routine")
+		}
 	}
 
-	// Prepare the tun queues. A device that can't open that many hands back
+	metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
-	// fewer (a single queue on platforms without multiqueue support) and we
+
-	// size the reader routines to what we actually got.
+	// Prepare n tun queues
-	queues, err := f.inside.Queues(f.routines)
+	var reader io.ReadWriteCloser = f.inside
+	for i := 0; i < f.routines; i++ {
+		if i > 0 {
+			reader, err = f.inside.NewMultiQueueReader()
 			if err != nil {
 				return err
 			}
-	if len(queues) < f.routines {
-		f.l.Warn("tun multiqueue is not supported on this platform, falling back to fewer routines",
-			"requested", f.routines, "opened", len(queues))
-		f.routines = len(queues)
 		}
-	f.queues = queues
+		f.readers[i] = reader
-
+	}
-	metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
 
 	// On error the caller owns the cleanup, Control.Start cancels the service context
 	// before releasing our resources so a waiter never observes a live context
@@ -305,7 +285,7 @@ func (f *Interface) run() {
 	// Launch n queues to read packets from tun dev
 	for i := 0; i < f.routines; i++ {
 		f.wg.Go(func() {
-			f.listenIn(f.queues[i], i)
+			f.listenIn(f.readers[i], i)
 		})
 	}
 
@@ -360,29 +340,8 @@ func (f *Interface) listenOut(i int) {
 	f.l.Debug("underlay reader is done", "reader", i)
 }
 
-func (f *Interface) listenIn(queue tio.Queue, i int) {
+func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
-	// Pinning this thread (and goroutine) to a single CPU keeps every UDP send from this goroutine going through
+	packet := make([]byte, mtu)
-	// the same TX ring on the nic (XPS selects the ring by CPU), so the wire sees per-flow order. Skip entirely
-	// when tun.pin_threads is false.
-	if f.pinThreads {
-		var cpu int
-		if n := len(f.cpuAffinity); n > 0 {
-			// Explicit tun.cpu_affinity list wins; parseCpuAffinity already
-			// validated the entries against the allowed CPU set.
-			cpu = f.cpuAffinity[i%n]
-		} else if allowed, err := util.AllowedCPUs(); err == nil && len(allowed) > 0 {
-			// Default: spread queues across the CPUs we're actually allowed to
-			// run on. Under a cpuset/taskset mask these aren't 0..NumCPU-1, so
-			// i % NumCPU would pick unrunnable IDs and every pin would fail.
-			cpu = allowed[i%len(allowed)]
-		} else {
-			cpu = i % runtime.NumCPU()
-		}
-		if err := util.PinThreadToCPU(cpu); err != nil {
-			f.l.Warn("failed to pin tun reader to CPU", "queue", i, "cpu", cpu, "err", err)
-		}
-	}
-
 	out := make([]byte, mtu)
 	fwPacket := &firewall.Packet{}
 	nb := make([]byte, 12, 12)
@@ -390,7 +349,7 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
 	conntrackCache := firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout)
 
 	for {
-		pkts, err := queue.Read()
+		n, err := reader.Read(packet)
 		if err != nil {
 			// Same shutdown noise handling as listenOut
 			if !f.closed.Load() && f.ctx.Err() == nil {
@@ -400,11 +359,7 @@ func (f *Interface) listenIn(queue tio.Queue, i int) {
 			break
 		}
 
-		for _, pkt := range pkts {
+		f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get())
-			// borrowed: pkt.Bytes is owned by the queue and only valid until
-			// the next Read; consumeInsidePacket reads it synchronously.
-			f.consumeInsidePacket(pkt.Bytes, fwPacket, nb, out, i, conntrackCache.Get())
-		}
 	}
 
 	f.l.Debug("overlay reader is done", "reader", i)

lighthouse.go

@@ -36,6 +36,10 @@ type LightHouse struct {
 	myVpnNetworksTable *bart.Lite
 	punchy             *Punchy
 
+	// localAddrsFn enumerates the underlay addresses we advertise. It is a field so tests can supply simulated
+	// addresses rather than whatever this machine's NICs happen to be. Set it before Start.
+	localAddrsFn func(*LocalAllowList) []netip.Addr
+
 	// Local cache of answers from light houses
 	// map of vpn addr to answers
 	addrMap map[netip.Addr]*RemoteList
@@ -107,6 +111,10 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
 		queryChan:          make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)),
 		l:                  l,
 	}
+	h.localAddrsFn = func(al *LocalAllowList) []netip.Addr {
+		return localAddrs(h.l, al)
+	}
+
 	lighthouses := make([]netip.Addr, 0)
 	h.lighthouses.Store(&lighthouses)
 	staticList := make(map[netip.Addr]struct{})
@@ -918,7 +926,7 @@ func (lh *LightHouse) SendUpdate() {
 	}
 
 	lal := lh.GetLocalAllowList()
-	for _, e := range localAddrs(lh.l, lal) {
+	for _, e := range lh.localAddrsFn(lal) {
 		if lh.myVpnNetworksTable.Contains(e) {
 			continue
 		}

main.go

@@ -7,7 +7,6 @@ import (
 	"net"
 	"net/netip"
 	"runtime/debug"
-	"slices"
 	"strings"
 	"time"
 
@@ -232,8 +231,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		relayManager:          NewRelayManager(ctx, l, hostMap, c),
 		punchy:                punchy,
 		ConntrackCacheTimeout: conntrackCacheTimeout,
-		CpuAffinity:           parseCpuAffinity(c, l, routines),
-		PinThreads:            c.GetBool("tun.pin_threads", true),
 		l:                     l,
 	}
 
@@ -271,6 +268,8 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 
 	attachCommands(l, c, ssh, ifce)
 
+	networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c)
+
 	return &Control{
 		state:                  StateReady,
 		f:                      ifce,
@@ -281,74 +280,11 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		statsStart:             stats.Start,
 		dnsStart:               ds.Start,
 		lighthouseStart:        lightHouse.StartUpdateWorker,
+		networkChangeStart:     networkChanges.Start,
 		connectionManagerStart: connManager.Start,
 	}, nil
 }
 
-// parseCpuAffinity reads `tun.cpu_affinity` from the config — a list of
-// integer CPU IDs, one per TUN reader goroutine. Empty / unset returns nil
-// (listenIn falls back to spreading queues across the allowed CPU set).
-// Length mismatch with `routines` is a warning, not an error: shorter lists
-// are modulo-cycled across queues, longer lists' tail is ignored. Invalid
-// entries (non-integer, or a CPU ID we're not allowed to run on) are also a
-// warning and disable the override entirely so we don't silently pin to the
-// wrong CPU. Entries are validated against the process's current affinity
-// mask (util.AllowedCPUs) rather than 0..NumCPU-1: under a cgroup cpuset or
-// taskset the runnable IDs are frequently not that contiguous range, and
-// pinning to an unrunnable ID always fails. If the allowed set can't be
-// determined we fall back to a plain non-negative check.
-func parseCpuAffinity(c *config.C, l *slog.Logger, routines int) []int {
-	raw := c.Get("tun.cpu_affinity")
-	if raw == nil {
-		return nil
-	}
-	rv, ok := raw.([]any)
-	if !ok {
-		l.Warn("tun.cpu_affinity must be a list of integers; ignoring", "value", raw)
-		return nil
-	}
-	// allowed is the set of CPU IDs we're actually permitted to run on. A nil
-	// slice (unsupported platform or lookup error) means "can't tell", so we
-	// only apply the weaker non-negative check in that case.
-	allowed, err := util.AllowedCPUs()
-	if err != nil {
-		l.Warn("could not determine allowed CPUs; validating tun.cpu_affinity against non-negative only", "error", err)
-		allowed = nil
-	}
-	cpus := make([]int, 0, len(rv))
-	for i, e := range rv {
-		var cpu int
-		switch v := e.(type) {
-		case int:
-			cpu = v
-		case int64:
-			cpu = int(v)
-		case float64:
-			cpu = int(v)
-		default:
-			l.Warn("tun.cpu_affinity entry not an integer; ignoring affinity",
-				"index", i, "value", e)
-			return nil
-		}
-		if cpu < 0 {
-			l.Warn("tun.cpu_affinity entry out of range; ignoring affinity",
-				"index", i, "cpu", cpu)
-			return nil
-		}
-		if len(allowed) > 0 && !slices.Contains(allowed, cpu) {
-			l.Warn("tun.cpu_affinity entry not in allowed CPU set; ignoring affinity",
-				"index", i, "cpu", cpu, "allowed", allowed)
-			return nil
-		}
-		cpus = append(cpus, cpu)
-	}
-	if len(cpus) != routines {
-		l.Warn("tun.cpu_affinity length doesn't match routines; queues will modulo-cycle through the list",
-			"affinity_len", len(cpus), "routines", routines)
-	}
-	return cpus
-}
-
 func moduleVersion() string {
 	info, ok := debug.ReadBuildInfo()
 	if !ok {

main_test.go

@@ -1,51 +0,0 @@
-package nebula
-
-import (
-	"testing"
-
-	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/test"
-	"github.com/slackhq/nebula/util"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestParseCpuAffinity(t *testing.T) {
-	l := test.NewLogger()
-
-	// newConfig returns a config.C with tun.cpu_affinity set to v. A nil v
-	// leaves the key unset.
-	newConfig := func(v any) *config.C {
-		c := config.NewC(l)
-		if v != nil {
-			c.Settings["tun"] = map[string]any{"cpu_affinity": v}
-		}
-		return c
-	}
-
-	// unset -> nil (listenIn falls back to spreading across the allowed set)
-	assert.Nil(t, parseCpuAffinity(newConfig(nil), l, 1))
-
-	// Pick a CPU we're actually allowed to run on so a valid list survives
-	// validation regardless of the host's affinity mask.
-	allowed, _ := util.AllowedCPUs()
-	validCPU := 0
-	if len(allowed) > 0 {
-		validCPU = allowed[0]
-	}
-
-	// valid list -> parsed through unchanged
-	assert.Equal(t, []int{validCPU, validCPU}, parseCpuAffinity(newConfig([]any{validCPU, validCPU}), l, 2))
-
-	// a negative entry is out of range on every platform -> disables the override
-	assert.Nil(t, parseCpuAffinity(newConfig([]any{validCPU, -1}), l, 2))
-
-	// a non-integer entry -> disables the override
-	assert.Nil(t, parseCpuAffinity(newConfig([]any{validCPU, "not-a-cpu"}), l, 2))
-
-	// a CPU id outside the allowed set -> disables the override. Only assertable
-	// where we can enumerate the allowed set (e.g. linux); 1<<20 is far beyond
-	// any representable CPU id so it can never be in the mask.
-	if len(allowed) > 0 {
-		assert.Nil(t, parseCpuAffinity(newConfig([]any{1 << 20}), l, 1))
-	}
-}

noiseutil/boring.go

@@ -4,77 +4,16 @@
 package noiseutil
 
 import (
-	"crypto/aes"
+	"crypto/boring"
-	"crypto/cipher"
-	"encoding/binary"
-
-	// unsafe needed for go:linkname
-	_ "unsafe"
 
 	"github.com/flynn/noise"
 )
 
+var CipherAESGCM noise.CipherFunc = CipherAESGCMFIPS140
+
 // EncryptLockNeeded indicates if calls to Encrypt need a lock
 // This is true for boringcrypto because the Seal function verifies that the
 // nonce is strictly increasing.
 const EncryptLockNeeded = true
 
-// NewGCMTLS is no longer exposed in go1.19+, so we need to link it in
+var boringEnabled = boring.Enabled()
-// See: https://github.com/golang/go/issues/56326
-//
-// NewGCMTLS is the internal method used with boringcrypto that provides a
-// validated mode of AES-GCM which enforces the nonce is strictly
-// monotonically increasing.  This is the TLS 1.2 specification for nonce
-// generation (which also matches the method used by the Noise Protocol)
-//
-// - https://github.com/golang/go/blob/go1.19/src/crypto/tls/cipher_suites.go#L520-L522
-// - https://github.com/golang/go/blob/go1.19/src/crypto/internal/boring/aes.go#L235-L237
-// - https://github.com/golang/go/blob/go1.19/src/crypto/internal/boring/aes.go#L250
-// - https://github.com/google/boringssl/blob/ae223d6138807a13006342edfeef32e813246b39/include/openssl/aead.h#L379-L381
-// - https://github.com/google/boringssl/blob/ae223d6138807a13006342edfeef32e813246b39/crypto/fipsmodule/cipher/e_aes.c#L1082-L1093
-//
-//go:linkname newGCMTLS crypto/internal/boring.NewGCMTLS
-func newGCMTLS(c cipher.Block) (cipher.AEAD, error)
-
-type cipherFn struct {
-	fn   func([32]byte) noise.Cipher
-	name string
-}
-
-func (c cipherFn) Cipher(k [32]byte) noise.Cipher { return c.fn(k) }
-func (c cipherFn) CipherName() string             { return c.name }
-
-// CipherAESGCM is the AES256-GCM AEAD cipher (using NewGCMTLS when GoBoring is present)
-var CipherAESGCM noise.CipherFunc = cipherFn{cipherAESGCMBoring, "AESGCM"}
-
-func cipherAESGCMBoring(k [32]byte) noise.Cipher {
-	c, err := aes.NewCipher(k[:])
-	if err != nil {
-		panic(err)
-	}
-	gcm, err := newGCMTLS(c)
-	if err != nil {
-		panic(err)
-	}
-	return aeadCipher{
-		gcm,
-		func(n uint64) []byte {
-			var nonce [12]byte
-			binary.BigEndian.PutUint64(nonce[4:], n)
-			return nonce[:]
-		},
-	}
-}
-
-type aeadCipher struct {
-	cipher.AEAD
-	nonce func(uint64) []byte
-}
-
-func (c aeadCipher) Encrypt(out []byte, n uint64, ad, plaintext []byte) []byte {
-	return c.Seal(out, c.nonce(n), plaintext, ad)
-}
-
-func (c aeadCipher) Decrypt(out []byte, n uint64, ad, ciphertext []byte) ([]byte, error) {
-	return c.Open(out, c.nonce(n), ciphertext, ad)
-}

noiseutil/boring_test.go

@@ -4,8 +4,6 @@
 package noiseutil
 
 import (
-	"crypto/boring"
-	"encoding/hex"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -14,33 +12,3 @@ import (
 func TestEncryptLockNeeded(t *testing.T) {
 	assert.True(t, EncryptLockNeeded)
 }
-
-// Ensure NewGCMTLS validates the nonce is non-repeating
-func TestNewGCMTLS(t *testing.T) {
-	assert.True(t, boring.Enabled())
-
-	// Test Case 16 from GCM Spec:
-	//  - (now dead link): http://csrc.nist.gov/groups/ST/toolkit/BCM/documents/proposedmodes/gcm/gcm-spec.pdf
-	//  - as listed in boringssl tests: https://github.com/google/boringssl/blob/fips-20220613/crypto/cipher_extra/test/cipher_tests.txt#L412-L418
-	key, _ := hex.DecodeString("feffe9928665731c6d6a8f9467308308feffe9928665731c6d6a8f9467308308")
-	iv, _ := hex.DecodeString("cafebabefacedbaddecaf888")
-	plaintext, _ := hex.DecodeString("d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39")
-	aad, _ := hex.DecodeString("feedfacedeadbeeffeedfacedeadbeefabaddad2")
-	expected, _ := hex.DecodeString("522dc1f099567d07f47f37a32a84427d643a8cdcbfe5c0c97598a2bd2555d1aa8cb08e48590dbb3da7b08b1056828838c5f61e6393ba7a0abcc9f662")
-	expectedTag, _ := hex.DecodeString("76fc6ece0f4e1768cddf8853bb2d551b")
-
-	expected = append(expected, expectedTag...)
-
-	var keyArray [32]byte
-	copy(keyArray[:], key)
-	c := CipherAESGCM.Cipher(keyArray)
-	aead := c.(aeadCipher).AEAD
-
-	dst := aead.Seal([]byte{}, iv, plaintext, aad)
-	assert.Equal(t, expected, dst)
-
-	// We expect this to fail since we are re-encrypting with a repeat IV
-	assert.PanicsWithError(t, "boringcrypto: EVP_AEAD_CTX_seal failed", func() {
-		dst = aead.Seal([]byte{}, iv, plaintext, aad)
-	})
-}

noiseutil/cipher_state.go

@@ -29,8 +29,11 @@ type CipherState interface {
 // NewCipherState wraps the post-handshake noise.CipherState in the per-cipher type that matches cipherFunc.
 // cipherFunc must be the same cipher used to build the noise CipherSuite that produced s.
 func NewCipherState(s *noise.CipherState, cipherFunc noise.CipherFunc) CipherState {
+	if cs, ok := s.Cipher().(CipherState); ok {
+		return cs
+	}
 	switch cipherFunc.CipherName() {
-	case CipherAESGCM.CipherName():
+	case noise.CipherAESGCM.CipherName():
 		return NewCipherStateAESGCM(s)
 	case noise.CipherChaChaPoly.CipherName():
 		return NewCipherStateChaChaPoly(s)

noiseutil/cipher_state_test.go

@@ -1,6 +1,7 @@
 package noiseutil
 
 import (
+	"crypto/fips140"
 	"testing"
 
 	"github.com/flynn/noise"
@@ -10,24 +11,30 @@ import (
 
 func TestCipherStateAESGCMRoundtrip(t *testing.T) {
 	enc, dec := buildCipherStates(t, CipherAESGCM)
-	roundtrip(t, NewCipherStateAESGCM(enc), NewCipherStateAESGCM(dec))
+	roundtrip(t, NewCipherState(enc, CipherAESGCM), NewCipherState(dec, CipherAESGCM))
 }
 
 func TestCipherStateChaChaPolyRoundtrip(t *testing.T) {
 	enc, dec := buildCipherStates(t, noise.CipherChaChaPoly)
-	roundtrip(t, NewCipherStateChaChaPoly(enc), NewCipherStateChaChaPoly(dec))
+	roundtrip(t, NewCipherState(enc, noise.CipherChaChaPoly), NewCipherState(dec, noise.CipherChaChaPoly))
 }
 
 func TestNewCipherStateDispatch(t *testing.T) {
 	encA, _ := buildCipherStates(t, CipherAESGCM)
 	encC, _ := buildCipherStates(t, noise.CipherChaChaPoly)
 
+	if !boringEnabled && !fips140.Enabled() {
 		assert.IsType(t, &CipherStateAESGCM{}, NewCipherState(encA, CipherAESGCM))
+	} else {
+		// fips140
+		assert.IsType(t, encA.Cipher(), NewCipherState(encA, CipherAESGCM))
+	}
+
 	assert.IsType(t, &CipherStateChaChaPoly{}, NewCipherState(encC, noise.CipherChaChaPoly))
 }
 
 func TestNewCipherStateUnsupportedPanics(t *testing.T) {
-	enc, _ := buildCipherStates(t, CipherAESGCM)
+	enc, _ := buildCipherStates(t, noise.CipherChaChaPoly)
 	assert.Panics(t, func() {
 		NewCipherState(enc, fakeCipher{})
 	})

noiseutil/fips140.go

@@ -0,0 +1,179 @@
+package noiseutil
+
+import (
+	"bytes"
+	"crypto/cipher"
+	"crypto/fips140"
+	"encoding/binary"
+	"fmt"
+	"reflect"
+	"runtime"
+	"unsafe"
+
+	// unsafe needed for go:linkname
+	_ "crypto/tls"
+	_ "unsafe"
+
+	"github.com/flynn/noise"
+)
+
+// TODO: Use NewGCMWithCounterNonce or NewGCMForQUIC once available:
+// - https://github.com/golang/go/issues/73110
+// - https://github.com/golang/go/issues/79219
+// Using tls.aeadAESGCMTLS13 gives us the TLS 1.3 GCM, which also verifies
+// that the nonce is strictly increasing. This works for both boringcrypto
+// and fips140.
+//
+//go:linkname aeadAESGCMTLS13 crypto/tls.aeadAESGCMTLS13
+func aeadAESGCMTLS13(key, noncePrefix []byte) cipher.AEAD
+
+type cipherFn struct {
+	fn   func([32]byte) noise.Cipher
+	name string
+}
+
+func (c cipherFn) Cipher(k [32]byte) noise.Cipher { return c.fn(k) }
+func (c cipherFn) CipherName() string             { return c.name }
+
+// CipherAESGCMFIPS140 is the AES256-GCM AEAD cipher (using tls.aeadAESGCMTLS13, for both boringcrypto and fips140)
+var CipherAESGCMFIPS140 noise.CipherFunc = cipherFn{cipherAESGCMFIPS140, "AESGCM"}
+
+// tls.aeadAESGCMTLS13 uses a 4 byte static prefix and an 8 byte XOR mask
+var emptyNonce = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
+
+func cipherAESGCMFIPS140(k [32]byte) noise.Cipher {
+	gcm := aeadAESGCMTLS13(k[:], emptyNonce)
+	gcm = extractFIPSAEAD(gcm)
+	return &aeadGCMFIPS140Cipher{
+		AEAD: gcm,
+	}
+}
+
+type aeadGCMFIPS140Cipher struct {
+	cipher.AEAD
+	ready bool
+}
+
+// Extract the internal FIPS GCM implementation from the tls wrapper. The TLS
+// wrapper is not thread safe around Open, so instead of locking around it we
+// can grab the internal implementation that is thread safe. This is the FIPS
+// module implementation: `crypto/internal/fips140/aes/gcm.GCMWithXORCounterNonce`
+//
+// - https://github.com/golang/go/blob/go1.26.4/src/crypto/internal/fips140/aes/gcm/gcm_nonces.go#L212-L287
+//
+// The wrapper is struct `crypto/tls.xorNonceAEAD` , with field `aead`:
+//
+// - https://github.com/golang/go/blob/go1.26.4/src/crypto/tls/cipher_suites.go#L482-L487
+//
+// This can be cleaned up once these FIPS implementations are exposed directly:
+//
+// - https://github.com/golang/go/issues/73110
+func extractFIPSAEAD(xorNonceAEAD cipher.AEAD) cipher.AEAD {
+	r := reflect.ValueOf(xorNonceAEAD)
+	v := r.Elem().FieldByName("aead")
+	if !v.IsValid() {
+		// The internal crypto/tls.xorNonceAEAD struct no longer has an `aead`
+		// field. This can only happen on a Go version this code was not built
+		// against; the package init() self-test guards against ever reaching
+		// this at runtime, so this is a defensive fail-fast.
+		panic(fmt.Sprintf("noiseutil: could not extract FIPS AEAD from %T on %s: no `aead` field (incompatible Go version)", xorNonceAEAD, runtime.Version()))
+	}
+	v2 := reflect.NewAt(v.Type(), unsafe.Pointer(v.UnsafeAddr())).Elem()
+	aead, ok := v2.Interface().(cipher.AEAD)
+	if !ok {
+		panic(fmt.Sprintf("noiseutil: extracted FIPS `aead` field is %s, not a cipher.AEAD, on %s (incompatible Go version)", v2.Type(), runtime.Version()))
+	}
+	return aead
+}
+
+func (c *aeadGCMFIPS140Cipher) init(nonce []byte) {
+	// GCMWithXORCounterNonce expects that the first call to Seal
+	// is with a counter of `0`, this is how it extracts the nonce mask.
+	// We can clean this up in the future when NewGCMWithCounterNonce or
+	// NewGCMForQUIC are available:
+	if !bytes.Equal(emptyNonce, nonce) {
+		c.AEAD.Seal([]byte{}, emptyNonce, []byte{}, []byte{})
+	}
+	c.ready = true
+}
+
+func (c *aeadGCMFIPS140Cipher) Seal(dst, nonce, plaintext, additionalData []byte) []byte {
+	if !c.ready {
+		c.init(nonce)
+	}
+	return c.AEAD.Seal(dst, nonce, plaintext, additionalData)
+}
+
+func (c *aeadGCMFIPS140Cipher) Encrypt(out []byte, n uint64, ad, plaintext []byte) []byte {
+	return c.Seal(out, aeadGCMFIPS140CipherNonce(n), plaintext, ad)
+}
+
+func (c *aeadGCMFIPS140Cipher) Decrypt(out []byte, n uint64, ad, ciphertext []byte) ([]byte, error) {
+	return c.Open(out, aeadGCMFIPS140CipherNonce(n), ciphertext, ad)
+}
+
+func (c *aeadGCMFIPS140Cipher) EncryptDanger(out, ad, plaintext []byte, n uint64, nb []byte) ([]byte, error) {
+	binary.BigEndian.PutUint64(nb[4:], n)
+	out = c.Seal(out, nb, plaintext, ad)
+	return out, nil
+}
+
+func (c *aeadGCMFIPS140Cipher) DecryptDanger(out, ad, ciphertext []byte, n uint64, nb []byte) ([]byte, error) {
+	binary.BigEndian.PutUint64(nb[4:], n)
+	return c.Open(out, nb, ciphertext, ad)
+}
+
+func aeadGCMFIPS140CipherNonce(n uint64) []byte {
+	// GCMWithXORCounterNonce uses a 4 byte static prefix and an 8 byte nonce
+	var nonce [12]byte
+	binary.BigEndian.PutUint64(nonce[4:], n)
+	return nonce[:]
+}
+
+// init validates the go:linkname + reflection extraction and the nonce-reuse
+// protection at startup, in every build. cipherAESGCMFIPS140 relies on unexported
+// crypto/tls and crypto/internal/fips140 internals; if a future Go version changes
+// those, this fails fast with a clear message instead of panicking per-handshake
+// (or, worse, silently losing the strictly-increasing nonce check that is the whole
+// point of using this cipher). Because this file has no build tag, this self-test
+// runs even in non-FIPS builds, so the default CI lane catches an incompatible Go.
+func init() {
+	var key [32]byte
+	c := cipherAESGCMFIPS140(key)
+
+	// Verify the extracted AEAD produces a working encrypt/decrypt roundtrip.
+	plaintext := []byte("nebula fips140 self-test")
+	ad := []byte("ad")
+	ct := c.Encrypt(nil, 1, ad, plaintext)
+	pt, err := c.Decrypt(nil, 1, ad, ct)
+	if err != nil {
+		panic(fmt.Sprintf("noiseutil: FIPS AES-GCM self-test roundtrip failed on %s: %v", runtime.Version(), err))
+	}
+	if !bytes.Equal(pt, plaintext) {
+		panic(fmt.Sprintf("noiseutil: FIPS AES-GCM self-test roundtrip returned wrong plaintext on %s", runtime.Version()))
+	}
+
+	// Verify the nonce-reuse protection still fires: re-encrypting with the same
+	// counter must panic. This is the guarantee we depend on for nonce safety, so
+	// if the extraction ever silently yields an AEAD without it, refuse to start.
+	// The strictly-increasing nonce check only exists under boringcrypto/fips140;
+	// in a plain build aeadAESGCMTLS13 wraps a standard GCM that does not enforce
+	// it (and CipherAESGCMFIPS140 is unused there anyway), so only assert it when
+	// one of those modes is active.
+	if (boringEnabled || fips140.Enabled()) && !reusePanics(c) {
+		panic(fmt.Sprintf("noiseutil: FIPS AES-GCM self-test did not reject a reused nonce on %s; nonce-reuse protection is missing (incompatible Go version)", runtime.Version()))
+	}
+}
+
+// reusePanics reports whether re-encrypting with an already-used counter panics,
+// as GCMWithXORCounterNonce is expected to.
+func reusePanics(c noise.Cipher) (panicked bool) {
+	c.Encrypt(nil, 2, nil, nil)
+	defer func() {
+		if recover() != nil {
+			panicked = true
+		}
+	}()
+	c.Encrypt(nil, 2, nil, nil)
+	return false
+}

noiseutil/fips140_test.go

@@ -0,0 +1,48 @@
+package noiseutil
+
+import (
+	"crypto/cipher"
+	"crypto/fips140"
+	"encoding/hex"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// Ensure NewAESGCM validates the nonce is non-repeating
+func TestNewAESGCM(t *testing.T) {
+	if !boringEnabled && !fips140.Enabled() {
+		t.Skip("TestNewAESGCM is only for fips140/boringcrypto")
+	}
+
+	key, _ := hex.DecodeString("feffe9928665731c6d6a8f9467308308feffe9928665731c6d6a8f9467308308")
+	iv, _ := hex.DecodeString("00000000facedbaddecaf888")
+	plaintext, _ := hex.DecodeString("d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39")
+	aad, _ := hex.DecodeString("feedfacedeadbeeffeedfacedeadbeefabaddad2")
+	expected, _ := hex.DecodeString("6a65c2edd45bd63c7e29f40e3d2ed8ba2b99f4c83135383d5676652f255059ceb24863ff10afb1089db701245da87fb88d3acd5f9dd0770cac220c3c04145caf25e190aeb775e7080401c628")
+
+	var keyArray [32]byte
+	copy(keyArray[:], key)
+	c := CipherAESGCM.Cipher(keyArray)
+	aead := c.(cipher.AEAD)
+
+	dst := aead.Seal([]byte{}, iv, plaintext, aad)
+	t.Logf("%x", dst)
+	assert.Equal(t, expected, dst)
+
+	// We expect this to fail since we are re-encrypting with a repeat IV
+	switch {
+	case boringEnabled:
+		assert.PanicsWithError(t, "boringcrypto: EVP_AEAD_CTX_seal failed", func() {
+			dst = aead.Seal([]byte{}, iv, plaintext, aad)
+		})
+	case fips140.Version() == "v1.0.0":
+		assert.PanicsWithValue(t, "crypto/cipher: counter decreased", func() {
+			dst = aead.Seal([]byte{}, iv, plaintext, aad)
+		})
+	default:
+		assert.PanicsWithValue(t, "crypto/cipher: counter decreased or remained the same", func() {
+			dst = aead.Seal([]byte{}, iv, plaintext, aad)
+		})
+	}
+}

noiseutil/fips140enforce.go

@@ -0,0 +1,13 @@
+//go:build fips140enforce
+
+package noiseutil
+
+import (
+	"crypto/fips140"
+)
+
+func init() {
+	if !fips140.Enforced() {
+		panic("Nebula compiled with fips140 expects FIPS140 to be enforced. Do not set GODEBUG=fips140, or if you do it must be set as GODEBUG=fips140=only")
+	}
+}

noiseutil/notboring.go

@@ -1,14 +1,25 @@
 //go:build !boringcrypto
-// +build !boringcrypto
 
 package noiseutil
 
 import (
+	"crypto/fips140"
+
 	"github.com/flynn/noise"
 )
 
 // EncryptLockNeeded indicates if calls to Encrypt need a lock
-const EncryptLockNeeded = false
+var EncryptLockNeeded = fips140.Enabled()
 
-// CipherAESGCM is the standard noise.CipherAESGCM when boringcrypto is not enabled
+var CipherAESGCM noise.CipherFunc = initAESGCM()
-var CipherAESGCM noise.CipherFunc = noise.CipherAESGCM
+
+func initAESGCM() noise.CipherFunc {
+	if fips140.Enabled() {
+		return CipherAESGCMFIPS140
+	} else {
+		return noise.CipherAESGCM
+	}
+
+}
+
+var boringEnabled = false

noiseutil/notboring_test.go

@@ -1,14 +0,0 @@
-//go:build !boringcrypto
-// +build !boringcrypto
-
-package noiseutil
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestEncryptLockNeeded(t *testing.T) {
-	assert.False(t, EncryptLockNeeded)
-}

outside.go

@@ -102,27 +102,31 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 		return
 	}
 
+	if len(packet) < header.Len+hostinfo.ConnectionState.dKey.Overhead() {
+		f.messageMetrics.RxInvalid(1)
+		if f.l.Enabled(context.Background(), slog.LevelDebug) {
+			f.l.Debug("packet too small", "from", via, "length", len(packet))
+		}
+		return
+	}
+
 	// All remaining packets are encrypted
-	ci := hostinfo.ConnectionState
-	if !ci.window.Check(f.l, h.MessageCounter) {
-		return
-	}
-
-	// Relay packets are special
 	if isMessageRelay {
+		// Relay packets are special, this branch should always early-return
+		if err = hostinfo.ConnectionState.VerifyRelay(f.l, h.MessageCounter, packet, nb); err != nil {
+			if f.l.Enabled(context.Background(), slog.LevelDebug) {
+				hostinfo.logger(f.l).Debug("Failed to verify relay packet", "error", err, "from", via, "header", h)
+			}
+			return
+		}
 		f.handleOutsideRelayPacket(hostinfo, via, out, packet, h, fwPacket, lhf, nb, q, localCache)
-
 		return
 	}
 
-	out, err = f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
+	out, err = hostinfo.ConnectionState.Decrypt(f.l, h.MessageCounter, out, packet, nb)
 	if err != nil {
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			hostinfo.logger(f.l).Debug("Failed to decrypt packet",
+			hostinfo.logger(f.l).Debug("Failed to decrypt packet", "error", err, "from", via, "header", h)
-				"error", err,
-				"from", via,
-				"header", h,
-			)
 		}
 		return
 	}
@@ -151,7 +155,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 			// No-op, useful for the Roaming and connectionManager side-effects above
 		case header.TestRequest:
 			//recycle the input packet ciphertext as our output buffer
-			f.send(header.Test, header.TestReply, ci, hostinfo, out, nb, packet)
+			f.send(header.Test, header.TestReply, hostinfo.ConnectionState, hostinfo, out, nb, packet)
 		default:
 			hostinfo.logger(f.l).Error("IsValidSubType was true, but unexpected test subtype seen", "from", via, "header", h)
 			return
@@ -170,27 +174,8 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 }
 
 func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
-	// The entire body is sent as AD, not encrypted.
+	// Successfully validated the thing. Get rid of the Relay header and the AEAD tag
-	// The packet consists of a 16-byte parsed Nebula header, Associated Data-protected payload, and a trailing 16-byte AEAD signature value.
+	signedPayload := packet[header.Len : len(packet)-hostinfo.ConnectionState.dKey.Overhead()]
-	// The packet is guaranteed to be at least 16 bytes at this point, b/c it got past the h.Parse() call above. If it's
-	// otherwise malformed (meaning, there is no trailing 16 byte AEAD value), then this will result in at worst a 0-length slice
-	// which will gracefully fail in the DecryptDanger call.
-	signedPayload := packet[:len(packet)-hostinfo.ConnectionState.dKey.Overhead()]
-	signatureValue := packet[len(packet)-hostinfo.ConnectionState.dKey.Overhead():]
-	var err error
-	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, signedPayload, signatureValue, h.MessageCounter, nb)
-	if err != nil {
-		return
-	}
-	// Advance the replay window now that the frame is authenticated
-	if !hostinfo.ConnectionState.window.Update(f.l, h.MessageCounter) {
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			hostinfo.logger(f.l).Debug("dropping out of window relay packet", "header", h)
-		}
-		return
-	}
-	// Successfully validated the thing. Get rid of the Relay header.
-	signedPayload = signedPayload[header.Len:]
 	// Pull the Roaming parts up here, and return in all call paths.
 	f.handleHostRoaming(hostinfo, via)
 	// Track usage of both the HostInfo and the Relay for the received & authenticated packet
@@ -214,7 +199,6 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
 		via = ViaSender{
 			UdpAddr:   via.UdpAddr,
 			relayHI:   hostinfo,
-			remoteIdx: relay.RemoteIndex,
 			relay:     relay,
 			IsRelayed: true,
 		}
@@ -235,9 +219,10 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
 		if targetRelay.State == Established {
 			switch targetRelay.Type {
 			case ForwardingType:
-				// Forward this packet through the relay tunnel
+				// Forward this packet through the relay tunnel, rebuilding it in place.
-				// Find the target HostInfo
+				// Encode overwrites the old outer header, and the new AEAD tag lands where the old one was
-				f.SendVia(targetHI, targetRelay, signedPayload, nb, out, false)
+				fwdBuf := packet[:0:len(packet)] // Cap to len(packet) to protect memory from a larger parent buffer
+				f.SendVia(targetHI, targetRelay, signedPayload, nb, fwdBuf, true)
 			case TerminalType:
 				hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
 				return
@@ -504,20 +489,6 @@ func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
 	return nil
 }
 
-func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, out []byte, packet []byte, h *header.H, nb []byte) ([]byte, error) {
-	var err error
-	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], mc, nb)
-	if err != nil {
-		return nil, err
-	}
-
-	if !hostinfo.ConnectionState.window.Update(f.l, mc) {
-		return nil, ErrOutOfWindow
-	}
-
-	return out, nil
-}
-
 func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache) {
 	err := newPacket(out, true, fwPacket)
 	if err != nil {
@@ -542,7 +513,7 @@ func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, out []byte, p
 		return
 	}
 
-	_, err = f.queues[q].Write(out)
+	_, err = f.readers[q].Write(out)
 	if err != nil {
 		f.l.Error("Failed to write to tun", "error", err)
 	}

overlay/device.go

@@ -4,25 +4,15 @@ import (
 	"io"
 	"net/netip"
 
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 )
 
-// defaultBatchBufSize is the per-Queue scratch size for Read. 65535 covers
-// any single IP packet.
-const defaultBatchBufSize = 65535
-
 type Device interface {
-	io.Closer
+	io.ReadWriteCloser
 	Activate() error
 	Networks() []netip.Prefix
 	Name() string
 	RoutesFor(netip.Addr) routing.Gateways
-	// Queues returns the device's packet queues, opening additional ones as
+	SupportsMultiqueue() bool
-	// needed until there are n. Platforms without multiqueue support return
+	NewMultiQueueReader() (io.ReadWriteCloser, error)
-	// their single queue regardless of n, so callers must size reader loops
-	// to len(result), not n; implementations never return more than n. An
-	// error means a queue that should have opened could not; the caller owns
-	// cleanup via Close. Called once, during interface activation.
-	Queues(n int) ([]tio.Queue, error)
 }

overlay/overlaytest/noop.go

@@ -3,9 +3,10 @@
 package overlaytest
 
 import (
+	"errors"
+	"io"
 	"net/netip"
 
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 )
 
@@ -30,16 +31,20 @@ func (NoopTun) Name() string {
 	return "noop"
 }
 
-func (NoopTun) Read() ([]tio.Packet, error) {
+func (NoopTun) Read([]byte) (int, error) {
-	return nil, nil
+	return 0, nil
 }
 
 func (NoopTun) Write([]byte) (int, error) {
 	return 0, nil
 }
 
-func (NoopTun) Queues(int) ([]tio.Queue, error) {
+func (NoopTun) SupportsMultiqueue() bool {
-	return []tio.Queue{NoopTun{}}, nil
+	return false
+}
+
+func (NoopTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, errors.New("unsupported")
 }
 
 func (NoopTun) Close() error {

overlay/tio/blockon_linux.go

@@ -1,45 +0,0 @@
-//go:build linux && !android
-// +build linux,!android
-
-package tio
-
-import (
-	"os"
-
-	"golang.org/x/sys/unix"
-)
-
-// blockOn parks the calling goroutine until fd is ready (events is POLLIN for
-// reads, POLLOUT for writes) or shutdownFd signals teardown. It builds the
-// pollfd array on the stack every call, so concurrent callers on the same
-// Queue never share Revents storage.
-//
-// Returns os.ErrClosed when shutdown was signaled (POLLIN on shutdownFd)
-// or either fd reported a problem condition (POLLHUP|POLLNVAL|POLLERR).
-func blockOn(fd, shutdownFd int32, events int16) error {
-	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
-	pfds := [2]unix.PollFd{
-		{Fd: fd, Events: events},
-		{Fd: shutdownFd, Events: unix.POLLIN},
-	}
-	var err error
-	for {
-		_, err = unix.Poll(pfds[:], -1)
-		if err != unix.EINTR {
-			break
-		}
-	}
-	tunEvents := pfds[0].Revents
-	shutdownEvents := pfds[1].Revents
-	// Check err before trusting the potentially bogus bits we just got.
-	if err != nil {
-		return err
-	}
-	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
-		return os.ErrClosed
-	}
-	if tunEvents&problemFlags != 0 {
-		return os.ErrClosed
-	}
-	return nil
-}

overlay/tio/queueset_poll_linux.go

@@ -1,90 +0,0 @@
-//go:build linux && !android
-// +build linux,!android
-
-package tio
-
-import (
-	"encoding/binary"
-	"errors"
-	"fmt"
-	"sync/atomic"
-
-	"golang.org/x/sys/unix"
-)
-
-type pollQueueSet struct {
-	pq []*Poll
-	// pqi is exactly the same as pq, but stored as the interface type
-	pqi        []Queue
-	shutdownFd int
-	closed     atomic.Bool
-}
-
-func NewPollQueueSet() (QueueSet, error) {
-	shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create eventfd: %w", err)
-	}
-
-	out := &pollQueueSet{
-		pq:         []*Poll{},
-		pqi:        []Queue{},
-		shutdownFd: shutdownFd,
-	}
-
-	return out, nil
-}
-
-func (c *pollQueueSet) Queues() []Queue {
-	return c.pqi
-}
-
-func (c *pollQueueSet) Add(fd int) error {
-	x, err := newPoll(fd, c.shutdownFd)
-	if err != nil {
-		return err
-	}
-	c.pq = append(c.pq, x)
-	c.pqi = append(c.pqi, x)
-
-	return nil
-}
-
-func (c *pollQueueSet) wakeForShutdown() error {
-	var buf [8]byte
-	binary.NativeEndian.PutUint64(buf[:], 1)
-	_, err := unix.Write(int(c.shutdownFd), buf[:])
-	return err
-}
-
-func (c *pollQueueSet) Close() error {
-	if c.closed.Swap(true) {
-		return nil
-	}
-
-	errs := []error{}
-
-	// Wake any reader blocked in poll so it observes POLLIN on the shutdown
-	// eventfd and returns os.ErrClosed.
-	if err := c.wakeForShutdown(); err != nil {
-		errs = append(errs, err)
-	}
-
-	// Close the per-queue tun fds; this also unblocks any in-flight reads.
-	// The per-queue Close deliberately leaves shutdownFd alone - it belongs
-	// to this container.
-	for _, x := range c.pq {
-		if err := x.Close(); err != nil {
-			errs = append(errs, err)
-		}
-	}
-
-	// Close the shutdown eventfd last: every reader's pollfd set references
-	// it, so it must outlive the wake + per-queue teardown above.
-	if err := unix.Close(c.shutdownFd); err != nil {
-		errs = append(errs, err)
-	}
-	c.shutdownFd = -1
-
-	return errors.Join(errs...)
-}

overlay/tio/single.go

@@ -1,50 +0,0 @@
-package tio
-
-import "io"
-
-// singleQueue adapts a legacy one-datagram-per-Read source into a Queue.
-// Read fills a private scratch buffer and returns exactly one Packet whose
-// Bytes borrow from that buffer, valid only until the next Read, per the
-// Queue contract. Single-reader like every Queue; Write is exactly as safe
-// for concurrent use as the underlying source's Write.
-type singleQueue struct {
-	rw     io.ReadWriter
-	closer io.Closer // nil: Close is a no-op (the source is shared and owned elsewhere)
-	buf    []byte
-	ret    [1]Packet
-}
-
-// NewSingleQueue wraps a one-datagram-per-Read ReadWriteCloser (a legacy tun
-// device) into a Queue. bufSize is the per-queue read scratch size and must
-// be at least the largest datagram the source can return. Close closes rwc.
-func NewSingleQueue(rwc io.ReadWriteCloser, bufSize int) Queue {
-	return &singleQueue{rw: rwc, closer: rwc, buf: make([]byte, bufSize)}
-}
-
-// NewSingleQueueNoClose is NewSingleQueue for a source owned by someone else,
-// e.g. several queues sharing one device. Close on the returned Queue is a
-// no-op so one queue can't tear the shared source out from under its
-// siblings; the owner remains responsible for closing the source itself.
-func NewSingleQueueNoClose(rw io.ReadWriter, bufSize int) Queue {
-	return &singleQueue{rw: rw, buf: make([]byte, bufSize)}
-}
-
-func (q *singleQueue) Read() ([]Packet, error) {
-	n, err := q.rw.Read(q.buf)
-	if err != nil {
-		return nil, err
-	}
-	q.ret[0] = Packet{Bytes: q.buf[:n]}
-	return q.ret[:], nil
-}
-
-func (q *singleQueue) Write(p []byte) (int, error) {
-	return q.rw.Write(p)
-}
-
-func (q *singleQueue) Close() error {
-	if q.closer == nil {
-		return nil
-	}
-	return q.closer.Close()
-}

overlay/tio/tio.go

@@ -1,52 +0,0 @@
-package tio
-
-import (
-	"io"
-)
-
-// QueueSet holds one or many Queue objects and helps close them in an orderly way.
-type QueueSet interface {
-	io.Closer
-	Queues() []Queue
-
-	// Add takes a tun fd, adds it to the set, and prepares it for use as a Queue.
-	Add(fd int) error
-}
-
-// Queue is a readable/writable packet queue. Concurrency contract: a single
-// read goroutine drives Read; plain Write is safe for concurrent callers.
-type Queue interface {
-	io.Closer
-
-	// Read returns one or more packets. The returned Packet.Bytes slices
-	// are borrowed from the Queue's internal buffer and are only valid
-	// until the next Read or Close on this Queue - callers must encrypt
-	// or copy each slice before the next call. Single-reader only: not
-	// safe for concurrent Reads (it reuses per-queue rx scratch each call).
-	Read() ([]Packet, error)
-
-	// Write emits a single packet on the plaintext (outside→inside)
-	// delivery path. Safe for concurrent use.
-	Write(p []byte) (int, error)
-}
-
-// Packet is the unit Queue.Read returns. Bytes points into the queue's
-// internal buffer and is only valid until the next Read or Close on the
-// queue that produced it.
-type Packet struct {
-	Bytes []byte
-}
-
-// Clone returns a Packet whose Bytes is a freshly allocated copy of p.Bytes,
-// safe to retain past the next Read or Close on the originating Queue.
-// Use this only when a caller genuinely needs to outlive the borrowed-slice
-// contract — the hot path reads should continue to consume the borrow
-// synchronously to avoid the allocation.
-func (p Packet) Clone() Packet {
-	if p.Bytes == nil {
-		return p
-	}
-	cp := make([]byte, len(p.Bytes))
-	copy(cp, p.Bytes)
-	return Packet{Bytes: cp}
-}

overlay/tio/tio_poll_linux.go

@@ -1,116 +0,0 @@
-//go:build linux && !android
-// +build linux,!android
-
-package tio
-
-import (
-	"fmt"
-	"os"
-	"sync/atomic"
-
-	"golang.org/x/sys/unix"
-)
-
-// Maximum size we accept for a single read from a TUN. 65535 covers any
-// single IP packet.
-const tunReadBufSize = 65535
-
-type Poll struct {
-	fd         int
-	shutdownFd int
-	closed     atomic.Bool
-
-	readBuf  []byte
-	batchRet [1]Packet
-}
-
-// newPoll wraps an existing tun fd. On failure it does NOT close fd: the
-// caller owns fd and is the sole closer (see pollQueueSet.Add callers in
-// overlay/tun_linux.go, which unix.Close on Add error). This keeps closes
-// at exactly one on every path.
-func newPoll(fd int, shutdownFd int) (*Poll, error) {
-	if err := unix.SetNonblock(fd, true); err != nil {
-		return nil, fmt.Errorf("failed to set Poll device as nonblocking: %w", err)
-	}
-
-	out := &Poll{
-		fd:         fd,
-		shutdownFd: shutdownFd,
-		readBuf:    make([]byte, tunReadBufSize),
-	}
-	return out, nil
-}
-
-// blockOnRead waits until the Poll fd is readable or shutdown has been signaled.
-// Returns os.ErrClosed if Close was called.
-func (t *Poll) blockOnRead() error {
-	return blockOn(int32(t.fd), int32(t.shutdownFd), unix.POLLIN)
-}
-
-func (t *Poll) blockOnWrite() error {
-	return blockOn(int32(t.fd), int32(t.shutdownFd), unix.POLLOUT)
-}
-
-func (t *Poll) Read() ([]Packet, error) {
-	n, err := t.readOne(t.readBuf)
-	if err != nil {
-		return nil, err
-	}
-	t.batchRet[0] = Packet{Bytes: t.readBuf[:n]}
-	return t.batchRet[:], nil
-}
-
-func (t *Poll) readOne(to []byte) (int, error) {
-	for {
-		n, errno := unix.Read(t.fd, to)
-		if errno == nil {
-			return n, nil
-		}
-		switch errno {
-		case unix.EAGAIN:
-			if err := t.blockOnRead(); err != nil {
-				return 0, err
-			}
-		case unix.EINTR:
-			// retry
-		case unix.EBADF:
-			return 0, os.ErrClosed
-		default:
-			return 0, errno
-		}
-	}
-}
-
-// Write is safe for concurrent use
-func (t *Poll) Write(from []byte) (int, error) {
-	for {
-		n, errno := unix.Write(t.fd, from)
-		if errno == nil {
-			return n, nil
-		}
-		switch errno {
-		case unix.EAGAIN:
-			if err := t.blockOnWrite(); err != nil {
-				return 0, err
-			}
-		case unix.EINTR:
-			// retry
-		case unix.EBADF:
-			return 0, os.ErrClosed
-		default:
-			return 0, errno
-		}
-	}
-}
-
-func (t *Poll) Close() error {
-	if t.closed.Swap(true) {
-		return nil
-	}
-	//shutdownFd is owned by the container, so we should not close it
-	// Close the underlying fd but do NOT null t.fd: a reader may still be
-	// loading it in readOne, and mutating the field would race that load.
-	// It gets EBADF -> os.ErrClosed (or wakes via the shutdown eventfd's
-	// ppoll first). closed.Swap already guarantees we only close once.
-	return unix.Close(t.fd)
-}

overlay/tio/tun_file_linux_test.go

@@ -1,208 +0,0 @@
-//go:build linux && !android && !e2e_testing
-// +build linux,!android,!e2e_testing
-
-package tio
-
-import (
-	"errors"
-	"os"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/require"
-	"golang.org/x/sys/unix"
-)
-
-// newReadPipe returns a read fd. The matching write fd is registered for cleanup.
-// The caller takes ownership of the read fd (pass it into a QueueSet).
-func newReadPipe(t *testing.T) int {
-	t.Helper()
-	var fds [2]int
-	if err := unix.Pipe2(fds[:], unix.O_CLOEXEC); err != nil {
-		t.Fatalf("pipe2: %v", err)
-	}
-	t.Cleanup(func() { _ = unix.Close(fds[1]) })
-	return fds[0]
-}
-
-func TestPoll_WakeForShutdown_WakesFriends(t *testing.T) {
-	pipe1 := newReadPipe(t)
-	pipe2 := newReadPipe(t)
-	parent, err := NewPollQueueSet()
-	require.NoError(t, err)
-	require.NoError(t, parent.Add(pipe1))
-	require.NoError(t, parent.Add(pipe2))
-	t.Cleanup(func() {
-		_ = unix.Close(pipe1)
-		_ = unix.Close(pipe2)
-	})
-
-	readers := parent.Queues()
-	errs := make([]error, len(readers))
-	var wg sync.WaitGroup
-	for i, r := range readers {
-		wg.Add(1)
-		go func(i int, r Queue) {
-			defer wg.Done()
-			_, errs[i] = r.Read()
-		}(i, r)
-	}
-
-	time.Sleep(50 * time.Millisecond)
-
-	if err := parent.Close(); err != nil {
-		t.Fatalf("Close: %v", err)
-	}
-
-	done := make(chan struct{})
-	go func() { wg.Wait(); close(done) }()
-	select {
-	case <-done:
-	case <-time.After(2 * time.Second):
-		t.Fatal("readers did not wake")
-	}
-
-	for i, err := range errs {
-		if !errors.Is(err, os.ErrClosed) {
-			t.Errorf("reader %d: expected os.ErrClosed, got %v", i, err)
-		}
-	}
-}
-
-// TestPoll_ConcurrentWrite_NoRace hammers a single Poll queue from two writer
-// goroutines while a reader drains the other end of the pipe. The writers
-// overflow the pipe buffer, so both repeatedly park in blockOnWrite at the same
-// time — the exact scenario that raced on the old shared writePoll member
-// array. Run under -race; a shared-array regression trips the detector here.
-func TestPoll_ConcurrentWrite_NoRace(t *testing.T) {
-	var fds [2]int
-	require.NoError(t, unix.Pipe2(fds[:], unix.O_CLOEXEC))
-	readFd, writeFd := fds[0], fds[1]
-
-	shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = unix.Close(shutdownFd) })
-
-	p, err := newPoll(writeFd, shutdownFd)
-	require.NoError(t, err)
-
-	const writers = 2
-	const perWriter = 4000
-	payload := make([]byte, 100)
-	total := writers * perWriter * len(payload)
-
-	// Reader: drain the read end (blocking) until every writer's bytes are
-	// consumed, so the writers keep making progress rather than wedging on a
-	// permanently full pipe.
-	readDone := make(chan struct{})
-	go func() {
-		defer close(readDone)
-		buf := make([]byte, 4096)
-		got := 0
-		for got < total {
-			n, rerr := unix.Read(readFd, buf)
-			got += n
-			if rerr != nil {
-				if rerr == unix.EINTR {
-					continue
-				}
-				return
-			}
-			if n == 0 { // EOF
-				return
-			}
-		}
-	}()
-
-	var wg sync.WaitGroup
-	for w := 0; w < writers; w++ {
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			for i := 0; i < perWriter; i++ {
-				if _, werr := p.Write(payload); werr != nil {
-					t.Errorf("write: %v", werr)
-					return
-				}
-			}
-		}()
-	}
-	wg.Wait()
-
-	select {
-	case <-readDone:
-	case <-time.After(10 * time.Second):
-		t.Fatal("reader did not drain")
-	}
-
-	require.NoError(t, p.Close())
-	_ = unix.Close(readFd)
-}
-
-// TestPoll_NewPoll_DoesNotCloseFdOnFailure pins the ownership rule: when
-// newPoll fails, it must leave fd open so the caller (pollQueueSet.Add's
-// callers in tun_linux.go) is the sole closer. If newPoll also closed fd,
-// the poll path would double-close on Add error. We force the failure with
-// an O_PATH descriptor: fcntl(F_SETFL) — which SetNonblock performs — is not
-// permitted on O_PATH fds and fails with EBADF, while the fd itself stays
-// open so we can observe that newPoll left it alone.
-func TestPoll_NewPoll_DoesNotCloseFdOnFailure(t *testing.T) {
-	fd, err := unix.Open("/", unix.O_PATH|unix.O_CLOEXEC, 0)
-	require.NoError(t, err)
-	t.Cleanup(func() { _ = unix.Close(fd) })
-
-	p, err := newPoll(fd, 1)
-	require.Error(t, err, "SetNonblock on an O_PATH fd should fail")
-	require.Nil(t, p)
-
-	// If newPoll had closed fd, F_GETFD would report it closed. It staying
-	// open proves newPoll left the fd for the caller to close exactly once.
-	require.True(t, fdOpen(t, fd), "newPoll must not close fd on failure; caller is the sole closer")
-}
-
-func TestPoll_Close_Idempotent(t *testing.T) {
-	tf, err := newPoll(newReadPipe(t), 1)
-	require.NoError(t, err)
-	if err := tf.Close(); err != nil {
-		t.Fatalf("first Close: %v", err)
-	}
-	if err := tf.Close(); err != nil {
-		t.Fatalf("second Close should be a no-op, got %v", err)
-	}
-}
-
-// fdOpen reports whether fd currently refers to an open file description.
-// A closed (or never-allocated) fd makes F_GETFD fail with EBADF.
-func fdOpen(t *testing.T, fd int) bool {
-	t.Helper()
-	_, err := unix.FcntlInt(uintptr(fd), unix.F_GETFD, 0)
-	if err == nil {
-		return true
-	}
-	if errors.Is(err, unix.EBADF) {
-		return false
-	}
-	t.Fatalf("unexpected fcntl(F_GETFD) error on fd %d: %v", fd, err)
-	return false
-}
-
-// TestPollQueueSet_Close_ClosesShutdownFd is the regression test for the
-// leaked shutdown eventfd: the container that owns shutdownFd must close it in
-// Close, and a second Close must be a safe no-op.
-func TestPollQueueSet_Close_ClosesShutdownFd(t *testing.T) {
-	qs, err := NewPollQueueSet()
-	require.NoError(t, err)
-	c, ok := qs.(*pollQueueSet)
-	require.True(t, ok)
-	require.NoError(t, qs.Add(newReadPipe(t)))
-
-	shutdownFd := c.shutdownFd
-	require.True(t, fdOpen(t, shutdownFd), "shutdown eventfd should be open before Close")
-
-	require.NoError(t, qs.Close())
-	require.False(t, fdOpen(t, shutdownFd), "shutdown eventfd should be closed after Close")
-
-	// Second Close must not touch fds (shutdownFd is now -1) and must return nil.
-	require.NoError(t, qs.Close())
-}

overlay/tun_android.go

@@ -13,7 +13,6 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 )
@@ -64,7 +63,7 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
 	return r
 }
 
-func (t *tun) Activate() error {
+func (t tun) Activate() error {
 	return nil
 }
 
@@ -97,6 +96,10 @@ func (t *tun) Name() string {
 	return "android"
 }
 
-func (t *tun) Queues(int) ([]tio.Queue, error) {
+func (t *tun) SupportsMultiqueue() bool {
-	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
+	return false
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for android")
 }

overlay/tun_darwin.go

@@ -6,6 +6,7 @@ package overlay
 import (
 	"errors"
 	"fmt"
+	"io"
 	"log/slog"
 	"net/netip"
 	"os"
@@ -15,7 +16,6 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	netroute "golang.org/x/net/route"
@@ -606,6 +606,10 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) Queues(int) ([]tio.Queue, error) {
+func (t *tun) SupportsMultiqueue() bool {
-	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
+	return false
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
 }

overlay/tun_disabled.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/rcrowley/go-metrics"
 	"github.com/slackhq/nebula/iputil"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 )
 
@@ -24,23 +23,6 @@ type disabledTun struct {
 	l  *slog.Logger
 }
 
-// Read hands the next queued packet to a reader, copying it into b. Reads
-// from concurrent queues are safe: the channel receive serializes them and
-// each queue copies into its own private scratch buffer.
-func (t *disabledTun) Read(b []byte) (int, error) {
-	r, ok := <-t.read
-	if !ok {
-		return 0, io.EOF
-	}
-
-	t.tx.Inc(1)
-	if t.l.Enabled(context.Background(), slog.LevelDebug) {
-		t.l.Debug("Write payload", "raw", prettyPacket(r))
-	}
-
-	return copy(b, r), nil
-}
-
 func newDisabledTun(vpnNetworks []netip.Prefix, queueLen int, metricsEnabled bool, l *slog.Logger) *disabledTun {
 	tun := &disabledTun{
 		vpnNetworks: vpnNetworks,
@@ -75,6 +57,24 @@ func (*disabledTun) Name() string {
 	return "disabled"
 }
 
+func (t *disabledTun) Read(b []byte) (int, error) {
+	r, ok := <-t.read
+	if !ok {
+		return 0, io.EOF
+	}
+
+	if len(r) > len(b) {
+		return 0, fmt.Errorf("packet larger than mtu: %d > %d bytes", len(r), len(b))
+	}
+
+	t.tx.Inc(1)
+	if t.l.Enabled(context.Background(), slog.LevelDebug) {
+		t.l.Debug("Write payload", "raw", prettyPacket(r))
+	}
+
+	return copy(b, r), nil
+}
+
 func (t *disabledTun) handleICMPEchoRequest(b []byte) bool {
 	out := make([]byte, len(b))
 	out = iputil.CreateICMPEchoResponse(b, out)
@@ -106,14 +106,12 @@ func (t *disabledTun) Write(b []byte) (int, error) {
 	return len(b), nil
 }
 
-func (t *disabledTun) Queues(n int) ([]tio.Queue, error) {
+func (t *disabledTun) SupportsMultiqueue() bool {
-	out := make([]tio.Queue, n)
+	return true
-	for i := range out {
+}
-		// NoClose: the shared channel and metrics are owned by the
+
-		// disabledTun; Close on the device tears them down once for everybody.
+func (t *disabledTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-		out[i] = tio.NewSingleQueueNoClose(t, defaultBatchBufSize)
+	return t, nil
-	}
-	return out, nil
 }
 
 func (t *disabledTun) Close() error {

overlay/tun_file_linux_test.go

@@ -0,0 +1,120 @@
+//go:build linux && !android && !e2e_testing
+// +build linux,!android,!e2e_testing
+
+package overlay
+
+import (
+	"errors"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// newReadPipe returns a read fd. The matching write fd is registered for cleanup.
+// The caller takes ownership of the read fd (pass it to newTunFd / newFriend).
+func newReadPipe(t *testing.T) int {
+	t.Helper()
+	var fds [2]int
+	if err := unix.Pipe2(fds[:], unix.O_CLOEXEC); err != nil {
+		t.Fatalf("pipe2: %v", err)
+	}
+	t.Cleanup(func() { _ = unix.Close(fds[1]) })
+	return fds[0]
+}
+
+func TestTunFile_WakeForShutdown_UnblocksRead(t *testing.T) {
+	tf, err := newTunFd(newReadPipe(t))
+	if err != nil {
+		t.Fatalf("newTunFd: %v", err)
+	}
+	t.Cleanup(func() { _ = tf.Close() })
+
+	done := make(chan error, 1)
+	go func() {
+		_, err := tf.Read(make([]byte, 64))
+		done <- err
+	}()
+
+	// Verify Read is actually blocked in poll.
+	select {
+	case err := <-done:
+		t.Fatalf("Read returned before shutdown signal: %v", err)
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	if err := tf.wakeForShutdown(); err != nil {
+		t.Fatalf("wakeForShutdown: %v", err)
+	}
+
+	select {
+	case err := <-done:
+		if !errors.Is(err, os.ErrClosed) {
+			t.Fatalf("expected os.ErrClosed, got %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("Read did not wake on shutdown")
+	}
+}
+
+func TestTunFile_WakeForShutdown_WakesFriends(t *testing.T) {
+	parent, err := newTunFd(newReadPipe(t))
+	if err != nil {
+		t.Fatalf("newTunFd: %v", err)
+	}
+	friend, err := parent.newFriend(newReadPipe(t))
+	if err != nil {
+		_ = parent.Close()
+		t.Fatalf("newFriend: %v", err)
+	}
+	t.Cleanup(func() {
+		_ = friend.Close()
+		_ = parent.Close()
+	})
+
+	readers := []*tunFile{parent, friend}
+	errs := make([]error, len(readers))
+	var wg sync.WaitGroup
+	for i, r := range readers {
+		wg.Add(1)
+		go func(i int, r *tunFile) {
+			defer wg.Done()
+			_, errs[i] = r.Read(make([]byte, 64))
+		}(i, r)
+	}
+
+	time.Sleep(50 * time.Millisecond)
+
+	if err := parent.wakeForShutdown(); err != nil {
+		t.Fatalf("wakeForShutdown: %v", err)
+	}
+
+	done := make(chan struct{})
+	go func() { wg.Wait(); close(done) }()
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("readers did not wake")
+	}
+
+	for i, err := range errs {
+		if !errors.Is(err, os.ErrClosed) {
+			t.Errorf("reader %d: expected os.ErrClosed, got %v", i, err)
+		}
+	}
+}
+
+func TestTunFile_Close_Idempotent(t *testing.T) {
+	tf, err := newTunFd(newReadPipe(t))
+	if err != nil {
+		t.Fatalf("newTunFd: %v", err)
+	}
+	if err := tf.Close(); err != nil {
+		t.Fatalf("first Close: %v", err)
+	}
+	if err := tf.Close(); err != nil {
+		t.Fatalf("second Close should be a no-op, got %v", err)
+	}
+}

overlay/tun_freebsd.go

@@ -7,6 +7,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"io"
 	"io/fs"
 	"log/slog"
 	"net/netip"
@@ -19,7 +20,7 @@ import (
 	"github.com/gaissmai/bart"
 
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
+
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	netroute "golang.org/x/net/route"
@@ -560,8 +561,12 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) Queues(int) ([]tio.Queue, error) {
+func (t *tun) SupportsMultiqueue() bool {
-	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
+	return false
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
 }
 
 func (t *tun) addRoutes(logErrors bool) error {

overlay/tun_ios.go

@@ -16,7 +16,6 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	"golang.org/x/sys/unix"
@@ -160,6 +159,10 @@ func (t *tun) Name() string {
 	return "iOS"
 }
 
-func (t *tun) Queues(int) ([]tio.Queue, error) {
+func (t *tun) SupportsMultiqueue() bool {
-	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
+	return false
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for ios")
 }

overlay/tun_linux.go

@@ -4,7 +4,9 @@
 package overlay
 
 import (
+	"encoding/binary"
 	"fmt"
+	"io"
 	"log/slog"
 	"net"
 	"net/netip"
@@ -17,15 +19,180 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
 )
 
+// tunFile wraps a TUN file descriptor with poll-based reads. The FD provided will be changed to non-blocking.
+// A shared eventfd allows Close to wake all readers blocked in poll.
+type tunFile struct {
+	fd         int
+	shutdownFd int
+	lastOne    bool
+	readPoll   [2]unix.PollFd
+	writePoll  [2]unix.PollFd
+	closed     bool
+}
+
+// newFriend makes a tunFile for a MultiQueueReader that copies the shutdown eventfd from the parent tun
+func (r *tunFile) newFriend(fd int) (*tunFile, error) {
+	if err := unix.SetNonblock(fd, true); err != nil {
+		return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
+	}
+	return &tunFile{
+		fd:         fd,
+		shutdownFd: r.shutdownFd,
+		readPoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLIN},
+			{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
+		},
+		writePoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLOUT},
+			{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
+		},
+	}, nil
+}
+
+func newTunFd(fd int) (*tunFile, error) {
+	if err := unix.SetNonblock(fd, true); err != nil {
+		return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
+	}
+
+	shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create eventfd: %w", err)
+	}
+
+	out := &tunFile{
+		fd:         fd,
+		shutdownFd: shutdownFd,
+		lastOne:    true,
+		readPoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLIN},
+			{Fd: int32(shutdownFd), Events: unix.POLLIN},
+		},
+		writePoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLOUT},
+			{Fd: int32(shutdownFd), Events: unix.POLLIN},
+		},
+	}
+
+	return out, nil
+}
+
+func (r *tunFile) blockOnRead() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(r.readPoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	//always reset these!
+	tunEvents := r.readPoll[0].Revents
+	shutdownEvents := r.readPoll[1].Revents
+	r.readPoll[0].Revents = 0
+	r.readPoll[1].Revents = 0
+	//do the err check before trusting the potentially bogus bits we just got
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	} else if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
+}
+
+func (r *tunFile) blockOnWrite() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(r.writePoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	//always reset these!
+	tunEvents := r.writePoll[0].Revents
+	shutdownEvents := r.writePoll[1].Revents
+	r.writePoll[0].Revents = 0
+	r.writePoll[1].Revents = 0
+	//do the err check before trusting the potentially bogus bits we just got
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	} else if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
+}
+
+func (r *tunFile) Read(buf []byte) (int, error) {
+	for {
+		if n, err := unix.Read(r.fd, buf); err == nil {
+			return n, nil
+		} else if err == unix.EAGAIN {
+			if err = r.blockOnRead(); err != nil {
+				return 0, err
+			}
+			continue
+		} else if err == unix.EINTR {
+			continue
+		} else if err == unix.EBADF {
+			return 0, os.ErrClosed
+		} else {
+			return 0, err
+		}
+	}
+}
+
+func (r *tunFile) Write(buf []byte) (int, error) {
+	for {
+		if n, err := unix.Write(r.fd, buf); err == nil {
+			return n, nil
+		} else if err == unix.EAGAIN {
+			if err = r.blockOnWrite(); err != nil {
+				return 0, err
+			}
+			continue
+		} else if err == unix.EINTR {
+			continue
+		} else if err == unix.EBADF {
+			return 0, os.ErrClosed
+		} else {
+			return 0, err
+		}
+	}
+}
+
+func (r *tunFile) wakeForShutdown() error {
+	var buf [8]byte
+	binary.NativeEndian.PutUint64(buf[:], 1)
+	_, err := unix.Write(int(r.readPoll[1].Fd), buf[:])
+	return err
+}
+
+func (r *tunFile) Close() error {
+	if r.closed { // avoid closing more than once. Technically a fd could get re-used, which would be a problem
+		return nil
+	}
+	r.closed = true
+	if r.lastOne {
+		_ = unix.Close(r.shutdownFd)
+	}
+	return unix.Close(r.fd)
+}
+
 type tun struct {
-	readers     tio.QueueSet
+	*tunFile
+	readers     []*tunFile
 	closeLock   sync.Mutex
 	Device      string
 	vpnNetworks []netip.Prefix
@@ -82,58 +249,45 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
 	return t, nil
 }
 
-// openTunDev opens /dev/net/tun, creating the device node first if it's
+func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) {
-// missing (docker containers occasionally omit it).
-func openTunDev() (int, error) {
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
-	if err == nil {
+	if err != nil {
-		return fd, nil
+		// If /dev/net/tun doesn't exist, try to create it (will happen in docker)
+		if os.IsNotExist(err) {
+			err = os.MkdirAll("/dev/net", 0755)
+			if err != nil {
+				return nil, fmt.Errorf("/dev/net/tun doesn't exist, failed to mkdir -p /dev/net: %w", err)
 			}
-	if !os.IsNotExist(err) {
+			err = unix.Mknod("/dev/net/tun", unix.S_IFCHR|0600, int(unix.Mkdev(10, 200)))
-		return -1, err
+			if err != nil {
-	}
+				return nil, fmt.Errorf("failed to create /dev/net/tun: %w", err)
-	if err = os.MkdirAll("/dev/net", 0755); err != nil {
-		return -1, fmt.Errorf("/dev/net/tun doesn't exist, failed to mkdir -p /dev/net: %w", err)
-	}
-	if err = unix.Mknod("/dev/net/tun", unix.S_IFCHR|0600, int(unix.Mkdev(10, 200))); err != nil {
-		return -1, fmt.Errorf("failed to create /dev/net/tun: %w", err)
 			}
+
 			fd, err = unix.Open("/dev/net/tun", os.O_RDWR, 0)
 			if err != nil {
-		return -1, fmt.Errorf("created /dev/net/tun, but still failed: %w", err)
+				return nil, fmt.Errorf("created /dev/net/tun, but still failed: %w", err)
 			}
-	return fd, nil
+		} else {
-}
-
-// tunSetIff runs TUNSETIFF with the given flags and returns the kernel-chosen
-// device name on success.
-func tunSetIff(fd int, name string, flags uint16) (string, error) {
-	var req ifReq
-	req.Flags = flags
-	copy(req.Name[:], name)
-	if err := ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
-		return "", err
-	}
-	return strings.Trim(string(req.Name[:]), "\x00"), nil
-}
-
-func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) {
-	baseFlags := uint16(unix.IFF_TUN | unix.IFF_NO_PI)
-	if multiqueue {
-		baseFlags |= unix.IFF_MULTI_QUEUE
-	}
-	nameStr := c.GetString("tun.dev", "")
-
-	fd, err := openTunDev()
-	if err != nil {
 			return nil, err
 		}
-	name, err := tunSetIff(fd, nameStr, baseFlags)
-	if err != nil {
-		_ = unix.Close(fd)
-		return nil, &NameError{Name: nameStr, Underlying: err}
 	}
 
+	var req ifReq
+	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI)
+	if multiqueue {
+		req.Flags |= unix.IFF_MULTI_QUEUE
+	}
+	nameStr := c.GetString("tun.dev", "")
+	copy(req.Name[:], nameStr)
+	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
+		_ = unix.Close(fd)
+		return nil, &NameError{
+			Name:       nameStr,
+			Underlying: err,
+		}
+	}
+	name := strings.Trim(string(req.Name[:]), "\x00")
+
 	t, err := newTunGeneric(c, l, fd, vpnNetworks)
 	if err != nil {
 		return nil, err
@@ -144,22 +298,16 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue
 	return t, nil
 }
 
-// newTunGeneric does all the stuff common to different tun initialization
+// newTunGeneric does all the stuff common to different tun initialization paths. It will close your files on error.
-// paths. It will close your files on error.
 func newTunGeneric(c *config.C, l *slog.Logger, fd int, vpnNetworks []netip.Prefix) (*tun, error) {
-	qs, err := tio.NewPollQueueSet()
+	tfd, err := newTunFd(fd)
 	if err != nil {
 		_ = unix.Close(fd)
 		return nil, err
 	}
-	err = qs.Add(fd)
-	if err != nil {
-		_ = unix.Close(fd)
-		return nil, err
-	}
-
 	t := &tun{
-		readers:                   qs,
+		tunFile:                   tfd,
+		readers:                   []*tunFile{tfd},
 		closeLock:                 sync.Mutex{},
 		vpnNetworks:               vpnNetworks,
 		TXQueueLen:                c.GetInt("tun.tx_queue", 500),
@@ -258,41 +406,36 @@ func (t *tun) reload(c *config.C, initial bool) error {
 	return nil
 }
 
-// Queues opens additional kernel multiqueue fds until the device has n
+func (t *tun) SupportsMultiqueue() bool {
-// queues, then returns them all. The first queue was opened by newTun.
+	return true
-func (t *tun) Queues(n int) ([]tio.Queue, error) {
-	for len(t.readers.Queues()) < n {
-		if err := t.addQueue(); err != nil {
-			return nil, err
-		}
-	}
-	return t.readers.Queues(), nil
 }
 
-// addQueue opens one more IFF_MULTI_QUEUE fd on the device and adds it to
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-// the queue set.
-func (t *tun) addQueue() error {
 	t.closeLock.Lock()
 	defer t.closeLock.Unlock()
 
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
 	if err != nil {
-		return err
+		return nil, err
 	}
 
-	flags := uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
+	var req ifReq
-	if _, err = tunSetIff(fd, t.Device, flags); err != nil {
+	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
+	copy(req.Name[:], t.Device)
+	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
 		_ = unix.Close(fd)
-		return err
+		return nil, err
 	}
 
-	err = t.readers.Add(fd)
+	out, err := t.tunFile.newFriend(fd)
 	if err != nil {
 		_ = unix.Close(fd)
-		return err
+		return nil, err
 	}
 
-	return nil
+	t.readers = append(t.readers, out)
+
+	return out, nil
 }
 
 func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
@@ -735,10 +878,32 @@ func (t *tun) Close() error {
 		t.routeChan = nil
 	}
 
+	// Signal all readers blocked in poll to wake up and exit
+	_ = t.tunFile.wakeForShutdown()
+
 	if t.ioctlFd > 0 {
 		_ = unix.Close(int(t.ioctlFd))
 		t.ioctlFd = 0
 	}
 
-	return t.readers.Close()
+	for i := range t.readers {
+		if i == 0 {
+			continue //we want to close the zeroth reader last
+		}
+		err := t.readers[i].Close()
+		if err != nil {
+			t.l.Error("error closing tun reader", "reader", i, "error", err)
+		} else {
+			t.l.Info("closed tun reader", "reader", i)
+		}
+	}
+
+	//this is t.readers[0] too
+	err := t.tunFile.Close()
+	if err != nil {
+		t.l.Error("error closing tun reader", "reader", 0, "error", err)
+	} else {
+		t.l.Info("closed tun reader", "reader", 0)
+	}
+	return err
 }

overlay/tun_netbsd.go

@@ -6,6 +6,7 @@ package overlay
 import (
 	"errors"
 	"fmt"
+	"io"
 	"log/slog"
 	"net/netip"
 	"os"
@@ -16,7 +17,6 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	netroute "golang.org/x/net/route"
@@ -390,8 +390,12 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) Queues(int) ([]tio.Queue, error) {
+func (t *tun) SupportsMultiqueue() bool {
-	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
+	return false
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for netbsd")
 }
 
 func (t *tun) addRoutes(logErrors bool) error {

overlay/tun_openbsd.go

@@ -6,6 +6,7 @@ package overlay
 import (
 	"errors"
 	"fmt"
+	"io"
 	"log/slog"
 	"net/netip"
 	"os"
@@ -16,7 +17,6 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	netroute "golang.org/x/net/route"
@@ -138,8 +138,8 @@ func tunWritev(fd int, iovecs []unix.Iovec) (n int, err error)
 //go:noescape
 func tunReadv(fd int, iovecs []unix.Iovec) (n int, err error)
 
-// Read pulls one IP packet off the tun device, scattering the 4 byte protocol header away from
+// Read pulls one IP packet off the tun device, scattering the 4 byte protocol header away from the
-// the packet so the payload lands directly in to.
+// packet so the payload lands directly in to.
 func (t *tun) Read(to []byte) (int, error) {
 	var head [4]byte
 
@@ -369,8 +369,12 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) Queues(int) ([]tio.Queue, error) {
+func (t *tun) SupportsMultiqueue() bool {
-	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
+	return false
+}
+
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for openbsd")
 }
 
 func (t *tun) addRoutes(logErrors bool) error {

overlay/tun_tester.go

@@ -14,7 +14,6 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/udp"
 )
@@ -178,6 +177,10 @@ func (t *TestTun) Read(b []byte) (int, error) {
 	return n, nil
 }
 
-func (t *TestTun) Queues(int) ([]tio.Queue, error) {
+func (t *TestTun) SupportsMultiqueue() bool {
-	return []tio.Queue{tio.NewSingleQueue(t, udp.MTU)}, nil
+	return false
+}
+
+func (t *TestTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented")
 }

overlay/tun_windows.go

@@ -6,6 +6,7 @@ package overlay
 import (
 	"crypto"
 	"fmt"
+	"io"
 	"log/slog"
 	"net/netip"
 	"os"
@@ -17,7 +18,6 @@ import (
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
 	"github.com/slackhq/nebula/wintun"
@@ -47,10 +47,6 @@ type winTun struct {
 	tun *wintun.NativeTun
 }
 
-func (t *winTun) Read(b []byte) (int, error) {
-	return t.tun.Read(b, 0)
-}
-
 func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (Device, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in Windows")
 }
@@ -259,12 +255,20 @@ func (t *winTun) Name() string {
 	return t.Device
 }
 
+func (t *winTun) Read(b []byte) (int, error) {
+	return t.tun.Read(b, 0)
+}
+
 func (t *winTun) Write(b []byte) (int, error) {
 	return t.tun.Write(b, 0)
 }
 
-func (t *winTun) Queues(int) ([]tio.Queue, error) {
+func (t *winTun) SupportsMultiqueue() bool {
-	return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil
+	return false
+}
+
+func (t *winTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	return nil, fmt.Errorf("TODO: multiqueue not implemented for windows")
 }
 
 func (t *winTun) Close() error {

overlay/user.go

@@ -6,7 +6,6 @@ import (
 	"net/netip"
 
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/routing"
 )
 
@@ -47,16 +46,12 @@ func (d *UserDevice) RoutesFor(ip netip.Addr) routing.Gateways {
 	return routing.Gateways{routing.NewGateway(ip, 1)}
 }
 
-func (d *UserDevice) Queues(n int) ([]tio.Queue, error) {
+func (d *UserDevice) SupportsMultiqueue() bool {
-	out := make([]tio.Queue, n)
+	return true
-	for i := range out {
+}
-		// All queues share the underlying pipes (the io.Pipe serializes
+
-		// concurrent callers) but each owns a private scratch buffer so
+func (d *UserDevice) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-		// concurrent Reads across queues never alias. NoClose: the pipes are
+	return d, nil
-		// owned by the UserDevice and torn down once by UserDevice.Close.
-		out[i] = tio.NewSingleQueueNoClose(d, defaultBatchBufSize)
-	}
-	return out, nil
 }
 
 func (d *UserDevice) Pipe() (*io.PipeReader, *io.PipeWriter) {
@@ -66,11 +61,9 @@ func (d *UserDevice) Pipe() (*io.PipeReader, *io.PipeWriter) {
 func (d *UserDevice) Read(p []byte) (n int, err error) {
 	return d.outboundReader.Read(p)
 }
-
 func (d *UserDevice) Write(p []byte) (n int, err error) {
 	return d.inboundWriter.Write(p)
 }
-
 func (d *UserDevice) Close() error {
 	d.inboundWriter.Close()
 	d.outboundWriter.Close()

overlay/user_test.go

@@ -1,163 +0,0 @@
-package overlay
-
-import (
-	"fmt"
-	"net/netip"
-	"sync"
-	"testing"
-
-	"github.com/slackhq/nebula/overlay/tio"
-)
-
-// newTestUserDevice returns the concrete *UserDevice so tests can reach Pipe()
-// and the internal queue plumbing.
-func newTestUserDevice(t *testing.T) *UserDevice {
-	t.Helper()
-	dev, err := NewUserDevice([]netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")})
-	if err != nil {
-		t.Fatalf("NewUserDevice: %v", err)
-	}
-	ud, ok := dev.(*UserDevice)
-	if !ok {
-		t.Fatalf("NewUserDevice returned %T, want *UserDevice", dev)
-	}
-	return ud
-}
-
-// TestUserDeviceReadersDistinctBuffers ensures each Queue is actually different
-func TestUserDeviceReadersDistinctBuffers(t *testing.T) {
-	d := newTestUserDevice(t)
-
-	readers, err := d.Queues(2)
-	if err != nil {
-		t.Fatalf("Queues: %v", err)
-	}
-	if len(readers) != 2 {
-		t.Fatalf("Queues(2) returned %d queues, want 2", len(readers))
-	}
-
-	// Distinct queue objects.
-	if readers[0] == readers[1] {
-		t.Fatal("Queues(2) returned the same queue object twice")
-	}
-
-	// Drive one packet through each queue and confirm the borrowed bytes from
-	// the first read are NOT clobbered by the second read. With a shared
-	// buffer, reading pkt1 into q1 would corrupt q0's still-borrowed slice.
-	_, ow := d.Pipe()
-
-	pkt0 := []byte("packet-zero-aaaaaaaa")
-	pkt1 := []byte("packet-one-bbbbbbbbb")
-
-	// The pipe is unbuffered, so writes block until a reader consumes them.
-	// Serialize: write pkt0 (read on q0), then write pkt1 (read on q1).
-	go func() {
-		if _, err := ow.Write(pkt0); err != nil {
-			t.Errorf("write pkt0: %v", err)
-		}
-		if _, err := ow.Write(pkt1); err != nil {
-			t.Errorf("write pkt1: %v", err)
-		}
-	}()
-
-	got0, err := readers[0].Read()
-	if err != nil {
-		t.Fatalf("q0.Read: %v", err)
-	}
-	if len(got0) != 1 || string(got0[0].Bytes) != string(pkt0) {
-		t.Fatalf("q0 first read = %q, want %q", firstBytes(got0), pkt0)
-	}
-	// Hold onto q0's borrowed slice across q1's read.
-	borrowed := got0[0].Bytes
-
-	got1, err := readers[1].Read()
-	if err != nil {
-		t.Fatalf("q1.Read: %v", err)
-	}
-	if len(got1) != 1 || string(got1[0].Bytes) != string(pkt1) {
-		t.Fatalf("q1 read = %q, want %q", firstBytes(got1), pkt1)
-	}
-
-	// q0's borrowed bytes must still hold pkt0 - a shared buffer would now
-	// show pkt1's contents.
-	if string(borrowed) != string(pkt0) {
-		t.Fatalf("q0 borrowed bytes were clobbered by q1's read: got %q, want %q", borrowed, pkt0)
-	}
-}
-
-// TestUserDeviceReadersConcurrentRace exercises two queues reading distinct
-// packets concurrently. Run it under `go test -race`: with the old
-// shared-buffer implementation the concurrent Reads raced on readBuf/batchRet
-// and corrupted each other's returned slices.
-func TestUserDeviceReadersConcurrentRace(t *testing.T) {
-	d := newTestUserDevice(t)
-	readers, err := d.Queues(2)
-	if err != nil {
-		t.Fatalf("Queues: %v", err)
-	}
-	_, ow := d.Pipe()
-
-	const iterations = 200
-
-	errs := make(chan error, 3)
-
-	// Each reader parks in Read on the shared outboundReader; io.Pipe hands
-	// each write to whichever reader is currently waiting. We only care that
-	// concurrent Reads into distinct buffers are race-free, so any parked
-	// reader may serve any write.
-	var wg sync.WaitGroup
-	run := func(idx int) {
-		defer wg.Done()
-		for i := 0; i < iterations; i++ {
-			pkts, err := readers[idx].Read()
-			if err != nil {
-				errs <- err
-				return
-			}
-			if len(pkts) != 1 {
-				errs <- fmt.Errorf("reader %d: got %d packets, want 1", idx, len(pkts))
-				return
-			}
-			// Touch every byte of the borrowed slice while the other reader
-			// may be mid-Read; a shared buffer would race here.
-			total := 0
-			for _, c := range pkts[0].Bytes {
-				total += int(c)
-			}
-			_ = total
-		}
-	}
-
-	wg.Add(2)
-	go run(0)
-	go run(1)
-
-	// Feed 2*iterations packets. io.Pipe copies each write straight into the
-	// waiting reader's private buffer, so reusing buf between writes is safe.
-	go func() {
-		buf := make([]byte, 32)
-		for i := 0; i < 2*iterations; i++ {
-			for j := range buf {
-				buf[j] = byte(i + j)
-			}
-			if _, err := ow.Write(buf); err != nil {
-				errs <- err
-				return
-			}
-		}
-	}()
-
-	wg.Wait()
-	select {
-	case err := <-errs:
-		t.Fatalf("concurrent reader failed: %v", err)
-	default:
-	}
-}
-
-func firstBytes(p []tio.Packet) []byte {
-	if len(p) == 0 {
-		return nil
-	}
-	return p[0].Bytes
-}

pki.go

@@ -1,6 +1,7 @@
 package nebula
 
 import (
+	"crypto/fips140"
 	"encoding/binary"
 	"encoding/json"
 	"errors"
@@ -237,10 +238,17 @@ func (cs *CertState) getCertificate(v cert.Version) cert.Certificate {
 	return nil
 }
 
-func newCipherSuite(curve cert.Curve, pkcs11backed bool, cipher string) (noise.CipherSuite, error) {
+// newCipherSuite builds the noise.CipherSuite for the given curve and cipher.
+// When fips140Enforced is true (FIPS 140-only mode), non-approved algorithms
+// (Curve25519 and ChaChaPoly) are rejected with an error. Callers pass
+// fips140.Enforced() for fips140Enforced.
+func newCipherSuite(curve cert.Curve, pkcs11backed bool, cipher string, fips140Enforced bool) (noise.CipherSuite, error) {
 	var dhFunc noise.DHFunc
 	switch curve {
 	case cert.Curve_CURVE25519:
+		if fips140Enforced {
+			return nil, errors.New("pki: use of Curve25519 is not allowed in FIPS 140-only mode")
+		}
 		dhFunc = noise.DH25519
 	case cert.Curve_P256:
 		if pkcs11backed {
@@ -253,6 +261,9 @@ func newCipherSuite(curve cert.Curve, pkcs11backed bool, cipher string) (noise.C
 	}
 
 	if cipher == "chachapoly" {
+		if fips140Enforced {
+			return nil, errors.New("pki: use of ChaChaPoly is not allowed in FIPS 140-only mode")
+		}
 		return noise.NewCipherSuite(dhFunc, noise.CipherChaChaPoly, noise.HashSHA256), nil
 	}
 	return noise.NewCipherSuite(dhFunc, noiseutil.CipherAESGCM, noise.HashSHA256), nil
@@ -326,6 +337,10 @@ func newCertStateFromConfig(c *config.C, cipher string) (*CertState, error) {
 			return nil, err
 		}
 
+		if fips140.Enforced() && crt.Curve() != cert.Curve_P256 {
+			return nil, fmt.Errorf("pki: use of %s is not allowed in FIPS 140-only mode", crt.Curve())
+		}
+
 		switch crt.Version() {
 		case cert.Version1:
 			if v1 != nil {
@@ -405,7 +420,7 @@ func newCertState(dv cert.Version, v1, v2 cert.Certificate, pkcs11backed bool, p
 			//NOTE: We do not currently have a method to verify a public private key pair when the private key is in an hsm
 		} else {
 			if err := v1.VerifyPrivateKey(privateKeyCurve, privateKey); err != nil {
-				return nil, fmt.Errorf("private key is not a pair with public key in nebula cert")
+				return nil, fmt.Errorf("private key is not a pair with public key in nebula cert: %w", err)
 			}
 		}
 
@@ -413,7 +428,7 @@ func newCertState(dv cert.Version, v1, v2 cert.Certificate, pkcs11backed bool, p
 		if err != nil {
 			return nil, fmt.Errorf("error marshalling v1 certificate for handshake: %w", err)
 		}
-		ncs, err := newCipherSuite(v1.Curve(), pkcs11backed, cipher)
+		ncs, err := newCipherSuite(v1.Curve(), pkcs11backed, cipher, fips140.Enforced())
 		if err != nil {
 			return nil, err
 		}
@@ -430,7 +445,7 @@ func newCertState(dv cert.Version, v1, v2 cert.Certificate, pkcs11backed bool, p
 			//NOTE: We do not currently have a method to verify a public private key pair when the private key is in an hsm
 		} else {
 			if err := v2.VerifyPrivateKey(privateKeyCurve, privateKey); err != nil {
-				return nil, fmt.Errorf("private key is not a pair with public key in nebula cert")
+				return nil, fmt.Errorf("private key is not a pair with public key in nebula cert: %w", err)
 			}
 		}
 
@@ -438,7 +453,7 @@ func newCertState(dv cert.Version, v1, v2 cert.Certificate, pkcs11backed bool, p
 		if err != nil {
 			return nil, fmt.Errorf("error marshalling v2 certificate for handshake: %w", err)
 		}
-		ncs, err := newCipherSuite(v2.Curve(), pkcs11backed, cipher)
+		ncs, err := newCipherSuite(v2.Curve(), pkcs11backed, cipher, fips140.Enforced())
 		if err != nil {
 			return nil, err
 		}

pki_test.go

@@ -0,0 +1,94 @@
+package nebula
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/slackhq/nebula/cert"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewCipherSuite(t *testing.T) {
+	tests := []struct {
+		name            string
+		curve           cert.Curve
+		cipher          string
+		fips140Enforced bool
+		wantErr         string
+		// wantName is the full expected CipherSuite name (<DH>_<Cipher>_<Hash>),
+		// only checked when wantErr is empty. Asserting the whole name makes both
+		// the curve and cipher selection load-bearing.
+		wantName string
+	}{
+		{
+			name:     "curve25519 aesgcm, not enforced",
+			curve:    cert.Curve_CURVE25519,
+			cipher:   "aesgcm",
+			wantName: "25519_AESGCM_SHA256",
+		},
+		{
+			name:     "curve25519 chachapoly, not enforced",
+			curve:    cert.Curve_CURVE25519,
+			cipher:   "chachapoly",
+			wantName: "25519_ChaChaPoly_SHA256",
+		},
+		{
+			name:     "p256 aesgcm, not enforced",
+			curve:    cert.Curve_P256,
+			cipher:   "aesgcm",
+			wantName: "P256_AESGCM_SHA256",
+		},
+		{
+			name:            "p256 aesgcm, enforced is allowed",
+			curve:           cert.Curve_P256,
+			cipher:          "aesgcm",
+			fips140Enforced: true,
+			wantName:        "P256_AESGCM_SHA256",
+		},
+		{
+			name:            "curve25519 rejected when enforced",
+			curve:           cert.Curve_CURVE25519,
+			cipher:          "aesgcm",
+			fips140Enforced: true,
+			wantErr:         "pki: use of Curve25519 is not allowed in FIPS 140-only mode",
+		},
+		{
+			name:            "chachapoly rejected when enforced",
+			curve:           cert.Curve_P256,
+			cipher:          "chachapoly",
+			fips140Enforced: true,
+			wantErr:         "pki: use of ChaChaPoly is not allowed in FIPS 140-only mode",
+		},
+		{
+			// Curve is checked before cipher, so a Curve25519+ChaChaPoly
+			// request reports the Curve25519 rejection.
+			name:            "curve25519 chachapoly rejected on curve when enforced",
+			curve:           cert.Curve_CURVE25519,
+			cipher:          "chachapoly",
+			fips140Enforced: true,
+			wantErr:         "pki: use of Curve25519 is not allowed in FIPS 140-only mode",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cs, err := newCipherSuite(tt.curve, false, tt.cipher, tt.fips140Enforced)
+			if tt.wantErr != "" {
+				require.EqualError(t, err, tt.wantErr)
+				assert.Nil(t, cs)
+				return
+			}
+			require.NoError(t, err)
+			require.NotNil(t, cs)
+			assert.Equal(t, tt.wantName, string(cs.Name()))
+		})
+	}
+}
+
+func TestNewCipherSuiteUnsupportedCurve(t *testing.T) {
+	cs, err := newCipherSuite(cert.Curve(99), false, "aesgcm", false)
+	require.Error(t, err)
+	assert.True(t, strings.HasPrefix(err.Error(), "unsupported curve:"), "got: %v", err)
+	assert.Nil(t, cs)
+}

stats.go

@@ -2,6 +2,7 @@ package nebula
 
 import (
 	"context"
+	"crypto/fips140"
 	"errors"
 	"fmt"
 	"log/slog"
@@ -291,6 +292,9 @@ func (s *statsServer) buildRuntime(cfg statsConfig) ([]func(), *http.Server) {
 				"version":         s.buildVersion,
 				"goversion":       runtime.Version(),
 				"boringcrypto":    strconv.FormatBool(boringEnabled()),
+				"fips140Version":  fips140.Version(),
+				"fips140Enabled":  strconv.FormatBool(fips140.Enabled()),
+				"fips140Enforced": strconv.FormatBool(fips140.Enforced()),
 			},
 		})
 		pr.MustRegister(g)

udp/netchange.go

@@ -0,0 +1,61 @@
+package udp
+
+import (
+	"context"
+	"log/slog"
+
+	"github.com/slackhq/nebula/config"
+)
+
+// NetworkChangeMonitor rebinds the udp listener when the local network moves out from under it.
+//
+// Detection lives here in the udp package, next to the socket it concerns and the platform matrix that already knows
+// which sockets go stale. What to do about a change — updating the lighthouse, requerying tunnels — is not the udp
+// package's business, so Start takes the reaction as a plain function. Passing it at Start rather than holding it
+// keeps this package from referencing whatever owns the rebind.
+//
+// On platforms whose sockets do not go stale, watchNetworkChanges hands back a nil channel and Start returns.
+type NetworkChangeMonitor struct {
+	l       *slog.Logger
+	ctx     context.Context
+	enabled bool
+}
+
+// NewNetworkChangeMonitor builds a monitor for local network changes. The returned monitor is always usable: Start
+// is safe to call unconditionally, it no-ops when disabled or on a platform that does not need it.
+func NewNetworkChangeMonitor(ctx context.Context, l *slog.Logger, c *config.C) *NetworkChangeMonitor {
+	return &NetworkChangeMonitor{
+		l:       l,
+		ctx:     ctx,
+		enabled: c.GetBool("listen.rebind_on_network_change", true),
+	}
+}
+
+// Start watches for network changes until the context is cancelled, calling rebind once per settled change. It
+// blocks, so callers run it in a goroutine, and it no-ops when disabled, unsupported, or with nothing to rebind.
+func (m *NetworkChangeMonitor) Start(rebind func()) {
+	if !m.enabled || rebind == nil || m.ctx.Err() != nil {
+		return
+	}
+
+	changes, err := watchNetworkChanges(m.ctx, m.l)
+	if err != nil {
+		// Not fatal. Everything else still works, we just won't notice a network change on our own.
+		m.l.Error("Failed to watch for network changes, will not rebind the udp listener when the network moves",
+			"error", err,
+		)
+		return
+	}
+
+	if changes == nil {
+		// This platform's sockets don't go stale, so there is nothing to watch for.
+		return
+	}
+
+	m.l.Info("Watching for network changes to rebind the udp listener")
+
+	for range changes {
+		m.l.Info("Local network changed, rebinding the udp listener")
+		rebind()
+	}
+}

udp/netchange_darwin.go

@@ -0,0 +1,164 @@
+//go:build darwin && !ios && !e2e_testing
+// +build darwin,!ios,!e2e_testing
+
+package udp
+
+import (
+	"context"
+	"encoding/binary"
+	"errors"
+	"log/slog"
+	"os"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+const (
+	// netChangeSettleWindow is how long we keep swallowing routing messages after the first interesting one. A
+	// single network change is never a single message, it is a burst: the link drops, addresses go away, new ones
+	// arrive, routes get rewritten. Reporting part way through that just means reporting again.
+	netChangeSettleWindow = time.Second
+
+	// netChangeReadBuffer is sized well past any rt_msghdr plus its addresses. A short read would be discarded by
+	// the kernel, so being generous here is how we avoid missing a message.
+	netChangeReadBuffer = 4096
+)
+
+// watchNetworkChanges reports when the local network moves out from under us, so the listener can be rebound.
+//
+// Darwin scopes a udp socket to whatever interface it came up on. Move between networks and we keep sending out an
+// interface that no longer has a route, which surfaces as an instant "no route to host" with no packet ever leaving
+// the box. Rebind clears that, but only if something notices the change and calls it. iOS has always been told by
+// the host app off NWPathMonitor. This is the equivalent for everything else that runs on darwin.
+//
+// The returned channel is buffered and coalescing: a send is dropped if one is already pending, since both mean the
+// same thing to a reader. It is closed when ctx is cancelled or the routing socket fails, so a caller can simply
+// range over it. Platforms whose sockets do not need rebinding return a nil channel and no error.
+func watchNetworkChanges(ctx context.Context, l *slog.Logger) (<-chan struct{}, error) {
+	sock, err := openRouteSocket()
+	if err != nil {
+		return nil, err
+	}
+
+	changes := make(chan struct{}, 1)
+
+	go func() {
+		defer close(changes)
+		defer func() { _ = sock.Close() }()
+
+		// Closing the socket is what unblocks the read in watchRouteSocket, so this turns cancellation into a
+		// close. It is scoped to this call so it cannot outlive the watch it belongs to.
+		done := make(chan struct{})
+		defer close(done)
+		go func() {
+			select {
+			case <-ctx.Done():
+				_ = sock.Close()
+			case <-done:
+			}
+		}()
+
+		watchRouteSocket(l, sock, changes)
+	}()
+
+	return changes, nil
+}
+
+// watchRouteSocket blocks reading the routing socket, reporting once per settled burst of changes. It returns when
+// the socket is closed, which is how cancellation gets us out of here.
+func watchRouteSocket(l *slog.Logger, sock *os.File, changes chan<- struct{}) {
+	buf := make([]byte, netChangeReadBuffer)
+
+	for {
+		n, err := sock.Read(buf)
+		if err != nil {
+			logRouteSocketError(l, err)
+			return
+		}
+
+		if !isNetworkChange(buf[:n]) {
+			continue
+		}
+
+		// Swallow the rest of the burst. The deadline is absolute and not extended by what arrives, so this always
+		// ends after the settle window no matter how chatty the socket is. Changes that land after the window
+		// simply produce another report, which is the correct outcome anyway.
+		deadline := time.Now().Add(netChangeSettleWindow)
+		for {
+			if err = sock.SetReadDeadline(deadline); err != nil {
+				logRouteSocketError(l, err)
+				return
+			}
+
+			if _, err = sock.Read(buf); err != nil {
+				if os.IsTimeout(err) {
+					break
+				}
+				logRouteSocketError(l, err)
+				return
+			}
+		}
+
+		if err = sock.SetReadDeadline(time.Time{}); err != nil {
+			logRouteSocketError(l, err)
+			return
+		}
+
+		select {
+		case changes <- struct{}{}:
+		default:
+			// One already pending, and a second "the network moved" tells the reader nothing new.
+		}
+	}
+}
+
+// logRouteSocketError reports a routing socket failure unless it is just us shutting the socket down.
+func logRouteSocketError(l *slog.Logger, err error) {
+	if errors.Is(err, os.ErrClosed) {
+		return
+	}
+
+	l.Error("Error reading the routing socket, will no longer notice local network changes", "error", err)
+}
+
+// openRouteSocket returns the routing socket as a non blocking os.File. Going through os.File puts reads on the go
+// poller, which buys us both a working read deadline and a Close that unblocks a read in progress.
+func openRouteSocket() (*os.File, error) {
+	fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = unix.SetNonblock(fd, true); err != nil {
+		_ = unix.Close(fd)
+		return nil, err
+	}
+
+	return os.NewFile(uintptr(fd), "route"), nil
+}
+
+// isNetworkChange reports whether a routing message means our local addressing may have moved out from under us.
+//
+// We read the header instead of parsing the message because the type is the only part we need, and a full parse can
+// fail on shapes we don't care about, which would turn "a message I can't parse" into "a change I missed".
+// rt_msghdr, if_msghdr and ifa_msghdr all begin with the same three fields, so this is the same for every type.
+func isNetworkChange(msg []byte) bool {
+	if len(msg) < 4 {
+		return false
+	}
+
+	// u_short msglen, u_char version, u_char type
+	if int(binary.NativeEndian.Uint16(msg[0:2])) > len(msg) || msg[2] != unix.RTM_VERSION {
+		return false
+	}
+
+	switch msg[3] {
+	case unix.RTM_NEWADDR, unix.RTM_DELADDR, unix.RTM_IFINFO:
+		// An address arrived or left, or a link changed state. Anything else on this socket is either a route
+		// churning underneath us, which a rebind doesn't help with, or unrelated traffic.
+		return true
+	default:
+		return false
+	}
+}

udp/netchange_darwin_test.go

@@ -0,0 +1,244 @@
+//go:build darwin && !ios && !e2e_testing
+// +build darwin,!ios,!e2e_testing
+
+package udp
+
+import (
+	"context"
+	"encoding/binary"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
+	"golang.org/x/sys/unix"
+)
+
+// routeMsg builds the first four bytes of a routing message, which is all isNetworkChange reads.
+func routeMsg(msgType uint8, extra int) []byte {
+	msg := make([]byte, 4+extra)
+	binary.NativeEndian.PutUint16(msg[0:2], uint16(len(msg)))
+	msg[2] = unix.RTM_VERSION
+	msg[3] = msgType
+	return msg
+}
+
+func TestIsNetworkChange(t *testing.T) {
+	// The three that mean our addressing may have moved
+	assert.True(t, isNetworkChange(routeMsg(unix.RTM_NEWADDR, 0)))
+	assert.True(t, isNetworkChange(routeMsg(unix.RTM_DELADDR, 0)))
+	assert.True(t, isNetworkChange(routeMsg(unix.RTM_IFINFO, 0)))
+
+	// Route churn is not something a rebind helps with
+	assert.False(t, isNetworkChange(routeMsg(unix.RTM_ADD, 0)))
+	assert.False(t, isNetworkChange(routeMsg(unix.RTM_DELETE, 0)))
+	assert.False(t, isNetworkChange(routeMsg(unix.RTM_GET, 0)))
+
+	// Garbage must not be mistaken for a change
+	assert.False(t, isNetworkChange(nil), "empty")
+	assert.False(t, isNetworkChange([]byte{0, 0, 0}), "short header")
+
+	wrongVersion := routeMsg(unix.RTM_NEWADDR, 0)
+	wrongVersion[2] = unix.RTM_VERSION + 1
+	assert.False(t, isNetworkChange(wrongVersion), "wrong rtm_version")
+
+	lying := routeMsg(unix.RTM_NEWADDR, 0)
+	binary.NativeEndian.PutUint16(lying[0:2], 512)
+	assert.False(t, isNetworkChange(lying), "msglen longer than what we read")
+}
+
+// socketPair returns a connected pair of datagram sockets, the first wrapped the same way the routing socket is. It
+// stands in for the kernel so the watch loop can be driven with synthetic messages.
+func socketPair(t *testing.T) (*os.File, int) {
+	t.Helper()
+
+	fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_DGRAM, 0)
+	require.NoError(t, err)
+	require.NoError(t, unix.SetNonblock(fds[0], true))
+
+	f := os.NewFile(uintptr(fds[0]), "route")
+	t.Cleanup(func() {
+		_ = f.Close()
+		_ = unix.Close(fds[1])
+	})
+
+	return f, fds[1]
+}
+
+func TestWatchRouteSocketCoalescesABurst(t *testing.T) {
+	sock, kernel := socketPair(t)
+	changes := make(chan struct{}, 1)
+
+	done := make(chan struct{})
+	go func() {
+		watchRouteSocket(test.NewLogger(), sock, changes)
+		close(done)
+	}()
+
+	// One network change is a burst of messages. All of these land inside the settle window, so they must produce
+	// exactly one report rather than one apiece.
+	for range 5 {
+		_, err := unix.Write(kernel, routeMsg(unix.RTM_NEWADDR, 8))
+		require.NoError(t, err)
+	}
+	// Uninteresting messages in the middle of a burst must not add a report of their own either.
+	_, err := unix.Write(kernel, routeMsg(unix.RTM_ADD, 8))
+	require.NoError(t, err)
+
+	select {
+	case <-changes:
+	case <-time.After(netChangeSettleWindow * 4):
+		t.Fatal("a burst should have reported a change")
+	}
+
+	// Nothing more from that burst
+	select {
+	case <-changes:
+		t.Fatal("a burst should report exactly once")
+	case <-time.After(netChangeSettleWindow):
+	}
+
+	// A change after the window has closed is a separate event and gets its own report.
+	_, err = unix.Write(kernel, routeMsg(unix.RTM_IFINFO, 8))
+	require.NoError(t, err)
+	select {
+	case <-changes:
+	case <-time.After(netChangeSettleWindow * 4):
+		t.Fatal("a later change should report again")
+	}
+
+	// Closing the socket is how the real thing shuts down
+	require.NoError(t, sock.Close())
+	select {
+	case <-done:
+	case <-time.After(time.Second * 5):
+		t.Fatal("watchRouteSocket did not return after the socket was closed")
+	}
+}
+
+func TestWatchRouteSocketIgnoresUninterestingMessages(t *testing.T) {
+	sock, kernel := socketPair(t)
+	changes := make(chan struct{}, 1)
+
+	done := make(chan struct{})
+	go func() {
+		watchRouteSocket(test.NewLogger(), sock, changes)
+		close(done)
+	}()
+
+	for _, msgType := range []uint8{unix.RTM_ADD, unix.RTM_DELETE, unix.RTM_GET, unix.RTM_MISS} {
+		_, err := unix.Write(kernel, routeMsg(msgType, 8))
+		require.NoError(t, err)
+	}
+
+	select {
+	case <-changes:
+		t.Fatal("route churn alone must not report a change")
+	case <-time.After(netChangeSettleWindow * 2):
+	}
+
+	require.NoError(t, sock.Close())
+	select {
+	case <-done:
+	case <-time.After(time.Second * 5):
+		t.Fatal("watchRouteSocket did not return after the socket was closed")
+	}
+}
+
+// TestWatchRouteSocketDropsRatherThanBlocks covers the coalescing send. A reader that is busy rebinding must not
+// wedge the watcher, and a second pending "the network moved" tells it nothing new anyway.
+func TestWatchRouteSocketDropsRatherThanBlocks(t *testing.T) {
+	sock, kernel := socketPair(t)
+	changes := make(chan struct{}, 1)
+
+	done := make(chan struct{})
+	go func() {
+		watchRouteSocket(test.NewLogger(), sock, changes)
+		close(done)
+	}()
+
+	// Nobody is reading changes, so after the first report the buffer is full for the rest of this test
+	for range 3 {
+		_, err := unix.Write(kernel, routeMsg(unix.RTM_NEWADDR, 8))
+		require.NoError(t, err)
+		time.Sleep(netChangeSettleWindow + time.Millisecond*250)
+	}
+
+	// The watcher must still be alive and responsive to a close
+	require.NoError(t, sock.Close())
+	select {
+	case <-done:
+	case <-time.After(time.Second * 5):
+		t.Fatal("watchRouteSocket wedged on a full channel")
+	}
+
+	assert.Len(t, changes, 1, "the pending report should have coalesced, not queued")
+}
+
+// TestWatchNetworkChangesStopsWithContext covers the detection path against a real routing socket, including that
+// cancelling the context closes the channel so a ranging caller falls out of its loop.
+func TestWatchNetworkChangesStopsWithContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	changes, err := watchNetworkChanges(ctx, test.NewLogger())
+	require.NoError(t, err)
+	require.NotNil(t, changes, "darwin should support watching")
+
+	drained := make(chan struct{})
+	go func() {
+		for range changes {
+		}
+		close(drained)
+	}()
+
+	cancel()
+	select {
+	case <-drained:
+	case <-time.After(time.Second * 5):
+		t.Fatal("cancelling the context should close the changes channel")
+	}
+}
+
+// TestNetworkChangeMonitorStopsWithContext drives the whole monitor against a real routing socket: Start must block
+// watching, and cancelling the context (which is all Control does on shutdown, it never stops the monitor directly)
+// must return it and clean up the watch goroutines.
+func TestNetworkChangeMonitorStopsWithContext(t *testing.T) {
+	// IgnoreCurrent because other tests in this package leave readers running; we only care about what this test
+	// leaks itself.
+	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	l := test.NewLogger()
+	c := config.NewC(l)
+	require.NoError(t, c.LoadString("listen:\n  rebind_on_network_change: true\n"))
+	m := NewNetworkChangeMonitor(ctx, l, c)
+
+	done := make(chan struct{})
+	go func() {
+		m.Start(func() {})
+		close(done)
+	}()
+
+	// Start should be sitting on the routing socket, not have fallen out. If it returned early it either failed to
+	// watch or no-op'd, both of which we want to catch.
+	select {
+	case <-done:
+		t.Fatal("Start returned instead of watching")
+	case <-time.After(time.Millisecond * 250):
+	}
+
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(time.Second * 5):
+		t.Fatal("Start did not return after the context was cancelled")
+	}
+
+	// Starting again after the context is dead must not open anything.
+	m.Start(func() {})
+}

udp/netchange_generic.go

@@ -0,0 +1,22 @@
+//go:build !darwin || ios || e2e_testing
+// +build !darwin ios e2e_testing
+
+package udp
+
+import (
+	"context"
+	"log/slog"
+)
+
+// watchNetworkChanges is a no-op outside of darwin.
+//
+// Darwin is the platform that scopes a udp socket to the interface it came up on, so it is the platform whose socket
+// goes stale when the local network changes. Everywhere else Rebind has nothing to do, so there is nothing to watch
+// for. iOS is excluded on purpose even though it is darwin: the host app already drives the rebind off NWPathMonitor,
+// and two things racing to rebind the same socket is worse than one.
+//
+// A nil channel means "not supported here", which callers must treat as "do not start a watcher" rather than
+// selecting on it, since a receive from a nil channel blocks forever.
+func watchNetworkChanges(_ context.Context, _ *slog.Logger) (<-chan struct{}, error) {
+	return nil, nil
+}

udp/netchange_test.go

@@ -0,0 +1,39 @@
+package udp
+
+import (
+	"context"
+	"testing"
+
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func newMonitor(t *testing.T, ctx context.Context, cfg string) *NetworkChangeMonitor {
+	t.Helper()
+	l := test.NewLogger()
+	c := config.NewC(l)
+	require.NoError(t, c.LoadString(cfg))
+	return NewNetworkChangeMonitor(ctx, l, c)
+}
+
+func TestNetworkChangeMonitorDefaultsOn(t *testing.T) {
+	// Says nothing about rebinding, so this covers the default.
+	m := newMonitor(t, context.Background(), "listen:\n  host: 0.0.0.0\n")
+	assert.True(t, m.enabled, "should default to on")
+}
+
+func TestNetworkChangeMonitorDisabledIsANoOp(t *testing.T) {
+	m := newMonitor(t, context.Background(), "listen:\n  rebind_on_network_change: false\n")
+	require.False(t, m.enabled)
+
+	// Must return without opening a socket. If it watched anything this would block.
+	m.Start(func() {})
+}
+
+func TestNetworkChangeMonitorNilRebindIsANoOp(t *testing.T) {
+	// Nothing to rebind, so there is no point watching, on any platform.
+	m := newMonitor(t, context.Background(), "listen:\n  rebind_on_network_change: true\n")
+	m.Start(nil)
+}

udp/udp_darwin.go

@@ -187,6 +187,9 @@ func (u *StdConn) SupportsMultipleReaders() bool {
 	return false
 }
 
+// Rebind clears the interface the kernel scoped this socket to, so that sends are routed against the current
+// routing table instead of the interface we happened to be on when the socket was created. Darwin pins sockets
+// this way on its own, which is what strands us after the underlying network changes.
 func (u *StdConn) Rebind() error {
 	var err error
 	if u.isV4 {
@@ -195,9 +198,5 @@ func (u *StdConn) Rebind() error {
 		err = syscall.SetsockoptInt(int(u.sysFd), syscall.IPPROTO_IPV6, syscall.IPV6_BOUND_IF, 0)
 	}
 
-	if err != nil {
+	return err
-		u.l.Error("Failed to rebind udp socket", "error", err)
-	}
-
-	return nil
 }

udp/udp_linux.go

@@ -4,12 +4,13 @@
 package udp
 
 import (
-	"context"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"log/slog"
 	"net"
 	"net/netip"
+	"sync/atomic"
 	"syscall"
 	"unsafe"
 
@@ -19,58 +20,51 @@ import (
 )
 
 type StdConn struct {
-	udpConn *net.UDPConn
+	sysFd  int
-	rawConn syscall.RawConn
+	closed atomic.Bool
 	isV4   bool
 	l      *slog.Logger
 	batch  int
 }
 
-func setReusePort(network, address string, c syscall.RawConn) error {
-	var opErr error
-	err := c.Control(func(fd uintptr) {
-		opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
-		//CloseOnExec already set by the runtime
-	})
-	if err != nil {
-		return err
-	}
-	return opErr
-}
-
 func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
-	listen := netip.AddrPortFrom(ip, uint16(port))
+	af := unix.AF_INET6
-	lc := net.ListenConfig{}
+	if ip.Is4() {
+		af = unix.AF_INET
+	}
+	syscall.ForkLock.RLock()
+	fd, err := unix.Socket(af, unix.SOCK_DGRAM, unix.IPPROTO_UDP)
+	if err == nil {
+		unix.CloseOnExec(fd)
+	}
+	syscall.ForkLock.RUnlock()
+	if err != nil {
+		return nil, fmt.Errorf("unable to open socket: %w", err)
+	}
+
 	if multi {
-		lc.Control = setReusePort
+		if err = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
+			_ = unix.Close(fd)
+			return nil, fmt.Errorf("unable to set SO_REUSEPORT: %w", err)
 		}
-	//this context is only used during the bind operation, you can't cancel it to kill the socket
-	pc, err := lc.ListenPacket(context.Background(), "udp", listen.String())
-	if err != nil {
-		return nil, fmt.Errorf("unable to open socket: %s", err)
-	}
-	udpConn := pc.(*net.UDPConn)
-	rawConn, err := udpConn.SyscallConn()
-	if err != nil {
-		_ = udpConn.Close()
-		return nil, err
-	}
-	//gotta find out if we got an AF_INET6 socket or not:
-	out := &StdConn{
-		udpConn: udpConn,
-		rawConn: rawConn,
-		l:       l,
-		batch:   batch,
 	}
 
-	af, err := out.getSockOptInt(unix.SO_DOMAIN)
+	var sa unix.Sockaddr
-	if err != nil {
+	if ip.Is4() {
-		_ = out.Close()
+		sa4 := &unix.SockaddrInet4{Port: port}
-		return nil, err
+		sa4.Addr = ip.As4()
+		sa = sa4
+	} else {
+		sa6 := &unix.SockaddrInet6{Port: port}
+		sa6.Addr = ip.As16()
+		sa = sa6
+	}
+	if err = unix.Bind(fd, sa); err != nil {
+		_ = unix.Close(fd)
+		return nil, fmt.Errorf("unable to bind to socket: %w", err)
 	}
-	out.isV4 = af == unix.AF_INET
 
-	return out, nil
+	return &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch}, nil
 }
 
 func (u *StdConn) SupportsMultipleReaders() bool {
@@ -81,133 +75,110 @@ func (u *StdConn) Rebind() error {
 	return nil
 }
 
-func (u *StdConn) getSockOptInt(opt int) (int, error) {
-	if u.rawConn == nil {
-		return 0, fmt.Errorf("no UDP connection")
-	}
-	var out int
-	var opErr error
-	err := u.rawConn.Control(func(fd uintptr) {
-		out, opErr = unix.GetsockoptInt(int(fd), unix.SOL_SOCKET, opt)
-	})
-	if err != nil {
-		return 0, err
-	}
-	return out, opErr
-}
-
-func (u *StdConn) setSockOptInt(opt int, n int) error {
-	if u.rawConn == nil {
-		return fmt.Errorf("no UDP connection")
-	}
-	var opErr error
-	err := u.rawConn.Control(func(fd uintptr) {
-		opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, opt, n)
-	})
-	if err != nil {
-		return err
-	}
-	return opErr
-}
-
 func (u *StdConn) SetRecvBuffer(n int) error {
-	return u.setSockOptInt(unix.SO_RCVBUFFORCE, n)
+	return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUFFORCE, n)
 }
 
 func (u *StdConn) SetSendBuffer(n int) error {
-	return u.setSockOptInt(unix.SO_SNDBUFFORCE, n)
+	return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUFFORCE, n)
 }
 
 func (u *StdConn) SetSoMark(mark int) error {
-	return u.setSockOptInt(unix.SO_MARK, mark)
+	return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK, mark)
 }
 
 func (u *StdConn) GetRecvBuffer() (int, error) {
-	return u.getSockOptInt(unix.SO_RCVBUF)
+	return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUF)
 }
 
 func (u *StdConn) GetSendBuffer() (int, error) {
-	return u.getSockOptInt(unix.SO_SNDBUF)
+	return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUF)
 }
 
 func (u *StdConn) GetSoMark() (int, error) {
-	return u.getSockOptInt(unix.SO_MARK)
+	return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK)
 }
 
 func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
-	a := u.udpConn.LocalAddr()
+	sa, err := unix.Getsockname(u.sysFd)
-
+	if err != nil {
-	switch v := a.(type) {
+		return netip.AddrPort{}, err
-	case *net.UDPAddr:
-		addr, ok := netip.AddrFromSlice(v.IP)
-		if !ok {
-			return netip.AddrPort{}, fmt.Errorf("LocalAddr returned invalid IP address: %s", v.IP)
 	}
-		return netip.AddrPortFrom(addr, uint16(v.Port)), nil
+	switch sa := sa.(type) {
-
+	case *unix.SockaddrInet4:
+		return netip.AddrPortFrom(netip.AddrFrom4(sa.Addr), uint16(sa.Port)), nil
+	case *unix.SockaddrInet6:
+		return netip.AddrPortFrom(netip.AddrFrom16(sa.Addr), uint16(sa.Port)), nil
 	default:
-		return netip.AddrPort{}, fmt.Errorf("LocalAddr returned: %#v", a)
+		return netip.AddrPort{}, fmt.Errorf("unsupported sock type: %T", sa)
 	}
 }
 
-func recvmmsg(fd uintptr, msgs []rawMessage) (int, bool, error) {
+// recvmmsg does one blocking recvmmsg (MSG_WAITFORONE), reading up to len(msgs) datagrams
-	var errno syscall.Errno
+func (u *StdConn) recvmmsg(msgs []rawMessage) (int, error) {
-	n, _, errno := unix.Syscall6(
+	r, _, errno := unix.Syscall6(
 		unix.SYS_RECVMMSG,
-		fd,
+		uintptr(u.sysFd),
 		uintptr(unsafe.Pointer(&msgs[0])),
 		uintptr(len(msgs)),
 		unix.MSG_WAITFORONE,
 		0,
 		0,
 	)
-	if errno == syscall.EAGAIN || errno == syscall.EWOULDBLOCK {
-		// No data available, block for I/O and try again.
-		return int(n), false, nil
-	}
 	if errno != 0 {
-		return int(n), true, &net.OpError{Op: "recvmmsg", Err: errno}
+		if u.closed.Load() {
+			return 0, net.ErrClosed
 		}
-	return int(n), true, nil
+		return 0, &net.OpError{Op: "recvmmsg", Err: errno}
+	}
+	n := int(r)
+	if (n == 0 || msgs[0].Len == 0) && u.closed.Load() {
+		return 0, net.ErrClosed
+	}
+	return n, nil
 }
 
-func (u *StdConn) listenOutSingle(r EncReader) error {
+// recvmsg does one blocking recvmsg into msgs[0]
-	var err error
+func (u *StdConn) recvmsg(msgs []rawMessage) (int, error) {
-	var n int
+	r, _, errno := unix.Syscall6(
-	var from netip.AddrPort
+		unix.SYS_RECVMSG,
-	buffer := make([]byte, MTU)
+		uintptr(u.sysFd),
-
+		uintptr(unsafe.Pointer(&msgs[0].Hdr)),
-	for {
+		0,
-		n, from, err = u.udpConn.ReadFromUDPAddrPort(buffer)
+		0,
-		if err != nil {
+		0,
-			return err
+		0,
+	)
+	if errno != 0 {
+		if u.closed.Load() {
+			return 0, net.ErrClosed
 		}
-		from = netip.AddrPortFrom(from.Addr().Unmap(), from.Port())
+		return 0, &net.OpError{Op: "recvmsg", Err: errno}
-		r(from, buffer[:n])
 	}
+	if r == 0 && u.closed.Load() {
+		return 0, net.ErrClosed
+	}
+	msgs[0].Len = uint32(r)
+	return 1, nil
 }
 
-func (u *StdConn) listenOutBatch(r EncReader) error {
+func (u *StdConn) ListenOut(r EncReader) error {
 	var ip netip.Addr
-	var n int
-	var operr error
-
 	msgs, buffers, names := u.PrepareRawMessages(u.batch)
-
+	read := u.recvmmsg
-	//reader needs to capture variables from this function, since it's used as a lambda with rawConn.Read
+	if u.batch == 1 {
-	//defining it outside the loop so it gets re-used
+		read = u.recvmsg
-	reader := func(fd uintptr) (done bool) {
-		n, done, operr = recvmmsg(fd, msgs)
-		return done
 	}
 
 	for {
-		err := u.rawConn.Read(reader)
+		n, err := read(msgs)
 		if err != nil {
-			return err
+			if errors.Is(err, unix.EINTR) {
+				continue // interrupted by a signal, retry the read
 			}
-		if operr != nil {
+			// net.ErrClosed after Close() is teardown, absorbed by the caller's
-			return operr
+			// closed flag like the other platforms; anything else is a real error.
+			return err
 		}
 
 		for i := 0; i < n; i++ {
@@ -222,26 +193,68 @@ func (u *StdConn) listenOutBatch(r EncReader) error {
 	}
 }
 
-func (u *StdConn) ListenOut(r EncReader) error {
+func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
-	if u.batch == 1 {
+	if u.isV4 {
-		return u.listenOutSingle(r)
+		return u.writeTo4(b, ip)
-	} else {
+	}
-		return u.listenOutBatch(r)
+	return u.writeTo6(b, ip)
+}
+
+func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
+	var rsa unix.RawSockaddrInet6
+	rsa.Family = unix.AF_INET6
+	rsa.Addr = ip.Addr().As16()
+	binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
+
+	for {
+		_, _, err := unix.Syscall6(
+			unix.SYS_SENDTO,
+			uintptr(u.sysFd),
+			uintptr(unsafe.Pointer(&b[0])),
+			uintptr(len(b)),
+			uintptr(0),
+			uintptr(unsafe.Pointer(&rsa)),
+			uintptr(unix.SizeofSockaddrInet6),
+		)
+		if err != 0 {
+			return &net.OpError{Op: "sendto", Err: err}
+		}
+		return nil
 	}
 }
 
-func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
+func (u *StdConn) writeTo4(b []byte, ip netip.AddrPort) error {
-	_, err := u.udpConn.WriteToUDPAddrPort(b, ip)
+	if !ip.Addr().Is4() {
-	return err
+		return ErrInvalidIPv6RemoteForSocket
+	}
+
+	var rsa unix.RawSockaddrInet4
+	rsa.Family = unix.AF_INET
+	rsa.Addr = ip.Addr().As4()
+	binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
+
+	for {
+		_, _, err := unix.Syscall6(
+			unix.SYS_SENDTO,
+			uintptr(u.sysFd),
+			uintptr(unsafe.Pointer(&b[0])),
+			uintptr(len(b)),
+			uintptr(0),
+			uintptr(unsafe.Pointer(&rsa)),
+			uintptr(unix.SizeofSockaddrInet4),
+		)
+		if err != 0 {
+			return &net.OpError{Op: "sendto", Err: err}
+		}
+		return nil
+	}
 }
 
 func (u *StdConn) ReloadConfig(c *config.C) {
 	b := c.GetInt("listen.read_buffer", 0)
 	if b > 0 {
-		err := u.SetRecvBuffer(b)
+		if err := u.SetRecvBuffer(b); err == nil {
-		if err == nil {
+			if s, err := u.GetRecvBuffer(); err == nil {
-			s, err := u.GetRecvBuffer()
-			if err == nil {
 				u.l.Info("listen.read_buffer was set", "size", s)
 			} else {
 				u.l.Warn("Failed to get listen.read_buffer", "error", err)
@@ -253,10 +266,8 @@ func (u *StdConn) ReloadConfig(c *config.C) {
 
 	b = c.GetInt("listen.write_buffer", 0)
 	if b > 0 {
-		err := u.SetSendBuffer(b)
+		if err := u.SetSendBuffer(b); err == nil {
-		if err == nil {
+			if s, err := u.GetSendBuffer(); err == nil {
-			s, err := u.GetSendBuffer()
-			if err == nil {
 				u.l.Info("listen.write_buffer was set", "size", s)
 			} else {
 				u.l.Warn("Failed to get listen.write_buffer", "error", err)
@@ -269,10 +280,8 @@ func (u *StdConn) ReloadConfig(c *config.C) {
 	b = c.GetInt("listen.so_mark", 0)
 	s, err := u.GetSoMark()
 	if b > 0 || (err == nil && s != 0) {
-		err := u.SetSoMark(b)
+		if err := u.SetSoMark(b); err == nil {
-		if err == nil {
+			if s, err := u.GetSoMark(); err == nil {
-			s, err := u.GetSoMark()
-			if err == nil {
 				u.l.Info("listen.so_mark was set", "mark", s)
 			} else {
 				u.l.Warn("Failed to get listen.so_mark", "error", err)
@@ -285,28 +294,20 @@ func (u *StdConn) ReloadConfig(c *config.C) {
 
 func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
 	var vallen uint32 = 4 * unix.SK_MEMINFO_VARS
-
+	_, _, err := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(u.sysFd), uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
-	if u.rawConn == nil {
+	if err != 0 {
-		return fmt.Errorf("no UDP connection")
-	}
-	var opErr error
-	err := u.rawConn.Control(func(fd uintptr) {
-		_, _, syserr := unix.Syscall6(unix.SYS_GETSOCKOPT, fd, uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
-		if syserr != 0 {
-			opErr = syserr
-		}
-	})
-	if err != nil {
 		return err
 	}
-	return opErr
+	return nil
 }
 
 func (u *StdConn) Close() error {
-	if u.udpConn != nil {
+	u.closed.Store(true)
-		return u.udpConn.Close()
+	// Wake the reader parked in recvmmsg/recvmsg. shutdown(2) on an unconnected socket
-	}
+	// returns ENOTCONN but still wakes it, so ignore the error.
-	return nil
+	// The reader then sees closed and stops touching the fd, making the Close below safe.
+	_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
+	return unix.Close(u.sysFd)
 }
 
 func NewUDPStatsEmitter(udpConns []Conn) func() {

udp/udp_linux_test.go

@@ -0,0 +1,179 @@
+//go:build linux && !android && !e2e_testing
+
+package udp
+
+import (
+	"errors"
+	"fmt"
+	"log/slog"
+	"net"
+	"net/netip"
+	"os"
+	"runtime"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+func testLogger() *slog.Logger {
+	return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))
+}
+
+// TestShutdownWakesAfterRx_Mechanism exercises the kernel quirk our teardown
+// relies on: once a socket has received a packet, shutdown(2) wakes a blocked
+// recvmmsg with n>=1/Len==0 (not n==0). recvmmsg must turn that into net.ErrClosed
+// once Close set closed, so a parked reader exits instead of spinning.
+func TestShutdownWakesAfterRx_Mechanism(t *testing.T) {
+	c, err := NewListener(testLogger(), netip.MustParseAddr("127.0.0.1"), 0, true, 64)
+	if err != nil {
+		t.Fatalf("NewListener: %v", err)
+	}
+	sc := c.(*StdConn)
+	addr, err := sc.LocalAddr()
+	if err != nil {
+		t.Fatalf("LocalAddr: %v", err)
+	}
+	msgs, _, _ := sc.PrepareRawMessages(sc.batch)
+
+	// Receive a real packet so the socket has carried data.
+	send, err := net.Dial("udp", addr.String())
+	if err != nil {
+		t.Fatalf("dial: %v", err)
+	}
+	if _, err := send.Write([]byte("hello")); err != nil {
+		t.Fatalf("write: %v", err)
+	}
+	time.Sleep(50 * time.Millisecond)
+	n, err := sc.recvmmsg(msgs)
+	t.Logf("drain of real packet: n=%d err=%v msgs[0].Len=%d", n, err, msgs[0].Len)
+	_ = send.Close()
+
+	// Block a reader on the now-empty queue, then tear down as Close() does.
+	// recvmmsg must return net.ErrClosed (not hang, not spin) even post-rx.
+	done := make(chan error, 1)
+	go func() {
+		_, err := sc.recvmmsg(msgs)
+		done <- err
+	}()
+	time.Sleep(150 * time.Millisecond) // let it park in recvmmsg
+
+	sc.closed.Store(true)
+	if serr := unix.Shutdown(sc.sysFd, unix.SHUT_RDWR); serr != nil {
+		t.Logf("shutdown returned %v (expected ENOTCONN on unconnected UDP)", serr)
+	}
+
+	select {
+	case err := <-done:
+		if !errors.Is(err, net.ErrClosed) {
+			t.Errorf("recvmmsg after post-rx shutdown returned %v, want net.ErrClosed", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatalf("HANG: recvmmsg did not return after shutdown following a received packet")
+	}
+	_ = unix.Close(sc.sysFd)
+}
+
+// TestListenOutTeardown_TrafficPatterns reproduces the field report: a blocking
+// reader must tear down cleanly on Close() regardless of what the socket has
+// carried. The three cases the report called out:
+//
+//	no traffic ever      -> works (shutdown wakes recvmmsg with n==0)
+//	ping once, then idle -> historically HUNG: once the socket has received a
+//	                        packet, shutdown(2) wakes recvmmsg with n>=1/Len==0,
+//	                        which an n==0-only teardown check misses
+//	continuous traffic   -> works (a real packet is always arriving)
+//
+// All three must return within the deadline; a hang dumps goroutines so the
+// stuck reader is visible.
+func TestListenOutTeardown_TrafficPatterns(t *testing.T) {
+	cases := []struct {
+		name    string
+		traffic func(send net.Conn, stop <-chan struct{})
+	}{
+		{"no_traffic_ever", func(net.Conn, <-chan struct{}) {}},
+		{"ping_once_then_idle", func(send net.Conn, _ <-chan struct{}) {
+			_, _ = send.Write([]byte("hello"))
+		}},
+		{"continuous", func(send net.Conn, stop <-chan struct{}) {
+			for {
+				select {
+				case <-stop:
+					return
+				default:
+					_, _ = send.Write([]byte("hello"))
+					time.Sleep(2 * time.Millisecond)
+				}
+			}
+		}},
+	}
+
+	// batch 1 exercises the recvmsg path, batch 64 the recvmmsg path; both must
+	// tear down cleanly.
+	for _, batch := range []int{1, 64} {
+		for _, tc := range cases {
+			t.Run(fmt.Sprintf("batch%d/%s", batch, tc.name), func(t *testing.T) {
+				runTeardownCase(t, batch, tc.name, tc.traffic)
+			})
+		}
+	}
+}
+
+func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net.Conn, stop <-chan struct{})) {
+	c, err := NewListener(testLogger(), netip.MustParseAddr("127.0.0.1"), 0, true, batch)
+	if err != nil {
+		t.Fatalf("NewListener: %v", err)
+	}
+	sc := c.(*StdConn)
+	addr, err := sc.LocalAddr()
+	if err != nil {
+		t.Fatalf("LocalAddr: %v", err)
+	}
+
+	var received atomic.Int64
+	loopDone := make(chan error, 1)
+	go func() {
+		loopDone <- sc.ListenOut(func(netip.AddrPort, []byte) {
+			received.Add(1)
+		})
+	}()
+
+	send, err := net.Dial("udp", addr.String())
+	if err != nil {
+		t.Fatalf("dial: %v", err)
+	}
+	defer send.Close()
+
+	stop := make(chan struct{})
+	trafficDone := make(chan struct{})
+	go func() {
+		traffic(send, stop)
+		close(trafficDone)
+	}()
+
+	// Let the pattern run and, for the idle case, the reader park again on an
+	// empty queue with the socket already having received a packet.
+	time.Sleep(500 * time.Millisecond)
+
+	start := time.Now()
+	if err := sc.Close(); err != nil {
+		t.Fatalf("Close: %v", err)
+	}
+	close(stop)
+
+	select {
+	case err := <-loopDone:
+		// Clean teardown surfaces as net.ErrClosed (propagated like the other
+		// platforms); the caller absorbs it via its closed flag.
+		if err != nil && !errors.Is(err, net.ErrClosed) {
+			t.Fatalf("%s: ListenOut returned unexpected error on teardown: %v", name, err)
+		}
+		t.Logf("%s: closed in %v (received %d packets)", name, time.Since(start), received.Load())
+	case <-time.After(3 * time.Second):
+		buf := make([]byte, 1<<20)
+		n := runtime.Stack(buf, true)
+		t.Fatalf("%s: HANG, ListenOut did not return within 3s of Close\n%s", name, buf[:n])
+	}
+	<-trafficDone
+}

udp/udp_tester.go

@@ -10,6 +10,7 @@ import (
 	"net/netip"
 	"os"
 	"sync"
+	"sync/atomic"
 
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
@@ -64,7 +65,9 @@ func acquirePacket() *Packet {
 }
 
 type TesterConn struct {
-	Addr netip.AddrPort
+	// addr is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
+	// move it mid-run to simulate roaming, so it is atomic rather than a plain field.
+	addr atomic.Pointer[netip.AddrPort]
 
 	RxPackets chan *Packet // Packets to receive into nebula
 	TxPackets chan *Packet // Packets transmitted outside by nebula
@@ -82,13 +85,24 @@ type TesterConn struct {
 }
 
 func NewListener(l *slog.Logger, ip netip.Addr, port int, _ bool, _ int) (Conn, error) {
-	return &TesterConn{
+	c := &TesterConn{
-		Addr:      netip.AddrPortFrom(ip, uint16(port)),
 		RxPackets: make(chan *Packet, 10),
 		TxPackets: make(chan *Packet, 10),
 		done:      make(chan struct{}),
 		l:         l,
-	}, nil
+	}
+	c.SetAddr(netip.AddrPortFrom(ip, uint16(port)))
+	return c, nil
+}
+
+// GetAddr returns the underlay address this conn currently sends from.
+func (u *TesterConn) GetAddr() netip.AddrPort {
+	return *u.addr.Load()
+}
+
+// SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network.
+func (u *TesterConn) SetAddr(addr netip.AddrPort) {
+	u.addr.Store(&addr)
 }
 
 // Send will place a UdpPacket onto the receive queue for nebula to consume
@@ -147,7 +161,7 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
 		p.Data = p.Data[:len(b)]
 	}
 	copy(p.Data, b)
-	p.From = u.Addr
+	p.From = u.GetAddr()
 	p.To = addr
 	select {
 	case <-u.done:
@@ -178,7 +192,7 @@ func NewUDPStatsEmitter(_ []Conn) func() {
 }
 
 func (u *TesterConn) LocalAddr() (netip.AddrPort, error) {
-	return u.Addr, nil
+	return u.GetAddr(), nil
 }
 
 func (u *TesterConn) SupportsMultipleReaders() bool {

util/cpupin_linux.go

@@ -1,43 +0,0 @@
-//go:build linux && !android && !e2e_testing
-
-package util
-
-import (
-	"runtime"
-
-	"golang.org/x/sys/unix"
-)
-
-// PinThreadToCPU restricts the calling OS thread to the given CPU via
-// sched_setaffinity(2). Combined with runtime.LockOSThread on the
-// goroutine, this prevents the kernel from migrating us across CPUs and
-// in turn keeps every UDP send from this goroutine going through the
-// same XPS-selected TX ring, eliminating the wire-side reorder that
-// otherwise fragments one nebula flow across multiple rings.
-func PinThreadToCPU(cpu int) error {
-	runtime.LockOSThread()
-	var set unix.CPUSet
-	set.Zero()
-	set.Set(cpu)
-	return unix.SchedSetaffinity(0, &set)
-}
-
-// AllowedCPUs returns the CPU IDs the calling process is currently allowed to
-// run on, as reported by sched_getaffinity(2). Under a cgroup cpuset or a
-// `taskset` mask the allowed IDs are frequently not the contiguous range
-// 0..NumCPU-1 (e.g. pinned to CPUs 4-7: NumCPU reports 4 while the valid IDs
-// are 4,5,6,7). Callers that need a real CPU to pin to must choose from this
-// set rather than assuming i % NumCPU is runnable, or every pin fails.
-func AllowedCPUs() ([]int, error) {
-	var set unix.CPUSet
-	if err := unix.SchedGetaffinity(0, &set); err != nil {
-		return nil, err
-	}
-	cpus := make([]int, 0, set.Count())
-	for cpu := 0; cpu < len(set)*64; cpu++ {
-		if set.IsSet(cpu) {
-			cpus = append(cpus, cpu)
-		}
-	}
-	return cpus, nil
-}

util/cpupin_other.go

@@ -1,18 +0,0 @@
-//go:build !linux || android || e2e_testing
-
-package util
-
-// PinThreadToCPU is a no-op outside Linux: only Linux exposes a stable
-// per-thread CPU affinity API and only Linux has XPS-driven TX ring
-// selection in the first place. On every other platform there's nothing
-// to fix here.
-func PinThreadToCPU(_ int) error {
-	return nil
-}
-
-// AllowedCPUs has no meaningful answer off Linux (no sched_getaffinity), so it
-// reports "unknown" by returning a nil slice and nil error. Callers treat an
-// empty result as "fall back to the default CPU choice".
-func AllowedCPUs() ([]int, error) {
-	return nil, nil
-}