use P256 as the default curve for fips140 mode

Co-authored-by: Jack Doan <me@jackdoan.com>

.github/workflows/release.yml

@@ -14,7 +14,7 @@ jobs:
 
       - uses: actions/setup-go@v7
         with:
-          go-version: '1.25'
+          go-version: '1.26'
           check-latest: true
 
       - name: Build
@@ -40,7 +40,7 @@ jobs:
 
       - uses: actions/setup-go@v7
         with:
-          go-version: '1.25'
+          go-version: '1.26'
           check-latest: true
 
       - name: Build
@@ -80,7 +80,7 @@ jobs:
 
       - uses: actions/setup-go@v7
         with:
-          go-version: '1.25'
+          go-version: '1.26'
           check-latest: true
 
       - name: Import certificates

.github/workflows/smoke-extra.yml

@@ -34,7 +34,7 @@ jobs:
 
     - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     - name: add hashicorp source
@@ -66,7 +66,7 @@ jobs:
 
     - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     - name: add hashicorp source
@@ -92,7 +92,7 @@ jobs:
 
     - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     # WSL2 + Ubuntu so the smoke can run a real linux peer with its own

.github/workflows/smoke.yml

@@ -22,42 +22,22 @@ jobs:
 
     - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
-    - name: build
+    - name: Smoke Docker
-      run: make bin-docker CGO_ENABLED=1 BUILD_ARGS=-race
+      run: make smoke-docker
 
-    - name: setup docker image
+    - name: Smoke Docker IPv6 overlay
-      working-directory: ./.github/workflows/smoke
+      run: make smoke-docker-ipv6
-      run: ./build.sh
 
-    - name: run smoke
+    - name: Smoke Relay Docker
-      working-directory: ./.github/workflows/smoke
+      run: make smoke-relay-docker
-      run: ./smoke.sh
 
-    - name: setup docker image ipv6
+    - name: Smoke Docker boringcrypto
-      working-directory: ./.github/workflows/smoke
+      run: make boringcrypto smoke-docker
-      run: SMOKE_OVERLAY_IPV6=1 ./build.sh
 
-    - name: run smoke ipv6
+    - name: Smoke Docker fips140
-      working-directory: ./.github/workflows/smoke
+      run: make fips140-all GOALS=smoke-docker
-      run: SMOKE_OVERLAY_IPV6=1 ./smoke.sh
-
-    - name: setup relay docker image
-      working-directory: ./.github/workflows/smoke
-      run: ./build-relay.sh
-
-    - name: run smoke relay
-      working-directory: ./.github/workflows/smoke
-      run: ./smoke-relay.sh
-
-    - name: setup docker image for P256
-      working-directory: ./.github/workflows/smoke
-      run: NAME="smoke-p256" CURVE=P256 ./build.sh
-
-    - name: run smoke-p256
-      working-directory: ./.github/workflows/smoke
-      run: NAME="smoke-p256" ./smoke.sh
 
     timeout-minutes: 10

.github/workflows/test.yml

@@ -22,7 +22,7 @@ jobs:
 
     - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     - name: Install goimports
@@ -42,7 +42,7 @@ jobs:
     - name: golangci-lint
       uses: golangci/golangci-lint-action@v9
       with:
-        version: v2.5
+        version: v2.12
 
   test:
     name: Test ${{ matrix.name }}
@@ -58,9 +58,14 @@ jobs:
           e2e-cmd: make e2evv
         - name: linux-boringcrypto
           os: ubuntu-latest
-          build-cmd: make bin-boringcrypto
+          build-cmd: make boringcrypto
-          test-cmd: make test-boringcrypto
+          test-cmd: make boringcrypto test
-          e2e-cmd: make e2e GOEXPERIMENT=boringcrypto CGO_ENABLED=1 TEST_ENV="TEST_LOGS=1" TEST_FLAGS="-v -ldflags -checklinkname=0"
+          e2e-cmd: make boringcrypto e2evv
+        - name: linux-fips140
+          os: ubuntu-latest
+          build-cmd: make fips140-all
+          test-cmd: make fips140-all GOALS=test
+          e2e-cmd: make fips140-all GOALS=e2evv
         - name: linux-pkcs11
           os: ubuntu-latest
           build-cmd: make bin-pkcs11
@@ -82,7 +87,7 @@ jobs:
 
     - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     - name: Build
@@ -127,7 +132,7 @@ jobs:
 
     - uses: actions/setup-go@v7
       with:
-        go-version: '1.25'
+        go-version: '1.26'
         check-latest: true
 
     - name: Build ${{ matrix.name }}

Makefile

@@ -72,6 +72,17 @@ ALL_CROSS_LINUX_ARM   = linux-arm-5 linux-arm-6 linux-arm-7 linux-arm64
 ALL_CROSS_LINUX_MIPS  = linux-mips linux-mipsle linux-mips64 linux-mips64le linux-mips-softfloat
 ALL_CROSS_LINUX_OTHER = linux-386 linux-ppc64le linux-riscv64 linux-loong64
 
+# Based on section 2.2 of the Go Cryptographic Module CVMP Security Policy #5247
+ALL_FIPS140 = linux-amd64-fips140 \
+	linux-arm64-fips140 \
+	windows-amd64-fips140 \
+	windows-arm64-fips140 \
+	darwin-arm64-fips140 \
+	freebsd-amd64-fips140 \
+	linux-arm-7-fips140 \
+	linux-mips64-fips140 \
+	linux-ppc64le-fips140
+
 e2e:
 	$(TEST_ENV) go test -tags=e2e_testing -count=1 $(TEST_FLAGS) ./e2e
 
@@ -137,6 +148,8 @@ release-netbsd: $(ALL_NETBSD:%=build/nebula-%.tar.gz)
 
 release-boringcrypto: build/nebula-linux-$(shell go env GOARCH)-boringcrypto.tar.gz
 
+release-fips140: $(ALL_FIPS140:%=build/nebula-%.tar.gz)
+
 BUILD_ARGS += -trimpath
 
 bin-windows: build/windows-amd64/nebula.exe build/windows-amd64/nebula-cert.exe
@@ -157,17 +170,20 @@ bin-freebsd-arm64: build/freebsd-arm64/nebula build/freebsd-arm64/nebula-cert
 bin-boringcrypto: build/linux-$(shell go env GOARCH)-boringcrypto/nebula build/linux-$(shell go env GOARCH)-boringcrypto/nebula-cert
 	mv $? .
 
+bin-fips140: build/linux-$(shell go env GOARCH)-fips140/nebula build/linux-$(shell go env GOARCH)-fips140/nebula-cert
+	mv $? .
+
 bin-pkcs11: BUILD_ARGS += -tags pkcs11
 bin-pkcs11: CGO_ENABLED = 1
 bin-pkcs11: bin
 
 bin:
-	go build $(BUILD_ARGS) -ldflags "$(LDFLAGS)" -o ./nebula${NEBULA_CMD_SUFFIX} ${NEBULA_CMD_PATH}
+	$(GOENV) go build $(BUILD_ARGS) -ldflags "$(LDFLAGS)" -o ./nebula${NEBULA_CMD_SUFFIX} ${NEBULA_CMD_PATH}
-	go build $(BUILD_ARGS) -ldflags "$(LDFLAGS)" -o ./nebula-cert${NEBULA_CMD_SUFFIX} ./cmd/nebula-cert
+	$(GOENV) go build $(BUILD_ARGS) -ldflags "$(LDFLAGS)" -o ./nebula-cert${NEBULA_CMD_SUFFIX} ./cmd/nebula-cert
 
 install:
-	go install $(BUILD_ARGS) -ldflags "$(LDFLAGS)" ${NEBULA_CMD_PATH}
+	$(GOENV) go install $(BUILD_ARGS) -ldflags "$(LDFLAGS)" ${NEBULA_CMD_PATH}
-	go install $(BUILD_ARGS) -ldflags "$(LDFLAGS)" ./cmd/nebula-cert
+	$(GOENV) go install $(BUILD_ARGS) -ldflags "$(LDFLAGS)" ./cmd/nebula-cert
 
 build/linux-arm-%: GOENV += GOARM=$(word 3, $(subst -, ,$*))
 build/linux-mips-%: GOENV += GOMIPS=$(word 3, $(subst -, ,$*))
@@ -178,8 +194,11 @@ build/linux-mips-softfloat/%: LDFLAGS += -s -w
 # boringcrypto
 build/linux-amd64-boringcrypto/%: GOENV += GOEXPERIMENT=boringcrypto CGO_ENABLED=1
 build/linux-arm64-boringcrypto/%: GOENV += GOEXPERIMENT=boringcrypto CGO_ENABLED=1
-build/linux-amd64-boringcrypto/%: LDFLAGS += -checklinkname=0
+
-build/linux-arm64-boringcrypto/%: LDFLAGS += -checklinkname=0
+# fips140
+FIPSVERSION = v1.0.0
+$(foreach _rule, $(ALL_FIPS140), build/$(_rule)/%): GOENV += GOFIPS140=$(FIPSVERSION)
+$(foreach _rule, $(ALL_FIPS140), build/$(_rule)/%): BUILD_ARGS += -tags fips140enforce
 
 build/%/nebula: .FORCE
 	GOOS=$(firstword $(subst -, , $*)) \
@@ -210,10 +229,7 @@ vet:
 	go vet $(VET_FLAGS) -v ./...
 
 test:
-	go test -v ./...
+	$(TEST_ENV) go test $(TEST_FLAGS) -v ./...
-
-test-boringcrypto:
-	GOEXPERIMENT=boringcrypto CGO_ENABLED=1 go test -ldflags "-checklinkname=0" -v ./...
 
 test-pkcs11:
 	CGO_ENABLED=1 go test -v -tags pkcs11 ./...
@@ -256,29 +272,72 @@ ifeq ($(words $(MAKECMDGOALS)),1)
 	@$(MAKE) service ${.DEFAULT_GOAL} --no-print-directory
 endif
 
+# Useful to chain together, like:
+# - make fips140 e2evv
+# - make fips140 smoke-docker
+# Use `release-fips140` to build release binaries
+fips140:
+	@echo > $(NULL_FILE)
+ifeq ($(strip $(GOFIPS140)),)
+	$(eval GOFIPS140 = $(FIPSVERSION))
+endif
+	$(eval GOENV += GOFIPS140=$(GOFIPS140))
+	$(eval BUILD_ARGS += -tags fips140enforce)
+	$(eval TEST_ENV += $(GOENV))
+	$(eval CURVE = P256)
+ifeq ($(words $(MAKECMDGOALS)),1)
+	@$(MAKE) fips140 GOFIPS140=$(GOFIPS140) ${.DEFAULT_GOAL} --no-print-directory
+endif
+
+# To test the future pending module, use like `make fips140-latest test`
+ALL_GOFIPS140 = v1.0.0 v1.26.0 latest
+define FIPS140_rule
+fips140-$(1): GOFIPS140 = $(1)
+fips140-$(1): fips140
+endef
+$(foreach _rule, $(ALL_GOFIPS140), $(eval $(call FIPS140_rule,$(_rule))))
+
+# Iterate and run the goals for all fips versions, like `make fips140-all GOALS=test`
+fips140-all:
+	@$(foreach _v,$(ALL_GOFIPS140),$(MAKE) fips140-$(_v) $(GOALS) &&) true
+
+# Useful to chain together, like:
+# - make boringcrypto e2evv
+# - make boringcrypto smoke-docker
+# Use `release-boringcrypto` or `bin-boringcrypto` to build release binaries
+boringcrypto:
+	@echo > $(NULL_FILE)
+	$(eval GOENV += GOEXPERIMENT=boringcrypto CGO_ENABLED=1)
+	$(eval TEST_ENV += $(GOENV))
+	$(eval CURVE = P256)
+ifeq ($(words $(MAKECMDGOALS)),1)
+	@$(MAKE) boringcrypto ${.DEFAULT_GOAL} --no-print-directory
+endif
+
 bin-docker: bin build/linux-amd64/nebula build/linux-amd64/nebula-cert
 
+smoke-docker: BUILD_ARGS += -race
+smoke-docker: GOENV += CGO_ENABLED=1
 smoke-docker: bin-docker
-	cd .github/workflows/smoke/ && ./build.sh
+	# This is so we can limit `fips140` smoke test to just P256 curve.
-	cd .github/workflows/smoke/ && ./smoke.sh
+	if [ "$(CURVE)" != "P256" ]; then cd .github/workflows/smoke/ && $(GOENV) ./build.sh; fi
-	cd .github/workflows/smoke/ && NAME="smoke-p256" CURVE="P256" ./build.sh
+	if [ "$(CURVE)" != "P256" ]; then cd .github/workflows/smoke/ && $(GOENV) ./smoke.sh; fi
-	cd .github/workflows/smoke/ && NAME="smoke-p256" ./smoke.sh
+	cd .github/workflows/smoke/ && $(GOENV) NAME="smoke-p256" CURVE="P256" ./build.sh
+	cd .github/workflows/smoke/ && $(GOENV) NAME="smoke-p256" ./smoke.sh
 
+smoke-relay-docker: BUILD_ARGS += -race
+smoke-relay-docker: GOENV += CGO_ENABLED=1
 smoke-relay-docker: bin-docker
-	cd .github/workflows/smoke/ && ./build-relay.sh
+	cd .github/workflows/smoke/ && $(GOENV) ./build-relay.sh
-	cd .github/workflows/smoke/ && ./smoke-relay.sh
+	cd .github/workflows/smoke/ && $(GOENV) ./smoke-relay.sh
 
 smoke-docker-ipv6: export SMOKE_OVERLAY_IPV6 = 1
 smoke-docker-ipv6: smoke-docker
 
-smoke-docker-race: BUILD_ARGS = -race
-smoke-docker-race: CGO_ENABLED = 1
-smoke-docker-race: smoke-docker
-
 smoke-vagrant/%: bin-docker build/%/nebula
 	cd .github/workflows/smoke/ && ./build.sh $*
 	cd .github/workflows/smoke/ && ./smoke-vagrant.sh $*
 
 .FORCE:
-.PHONY: all all-linux all-freebsd all-openbsd all-netbsd all-darwin all-windows all-cross-linux all-cross-linux-arm all-cross-linux-mips all-cross-linux-other all-cross-darwin all-cross-windows bench bench-cpu bench-cpu-long bin build-test-mobile e2e e2ev e2evv e2evvv e2evvvv proto release service smoke-docker smoke-docker-race test test-cov-html smoke-vagrant/%
+.PHONY: all all-linux all-freebsd all-openbsd all-netbsd all-darwin all-windows all-cross-linux all-cross-linux-arm all-cross-linux-mips all-cross-linux-other all-cross-darwin all-cross-windows bench bench-cpu bench-cpu-long bin bin-windows bin-windows-arm64 bin-darwin bin-freebsd bin-freebsd-arm64 bin-boringcrypto bin-fips140 bin-pkcs11 bin-docker boringcrypto build-test-mobile docker e2e e2ev e2evv e2evvv e2evvvv e2e-bench fips140 fips140-all $(ALL_GOFIPS140:%=fips140-%) install proto release release-linux release-freebsd release-openbsd release-netbsd release-boringcrypto release-fips140 service smoke-docker smoke-relay-docker smoke-docker-ipv6 test test-pkcs11 test-cov-html vet smoke-vagrant/%
 .DEFAULT_GOAL := bin

README.md

@@ -145,17 +145,27 @@ To build nebula for a specific platform (ex, Windows):
 
 See the [Makefile](Makefile) for more details on build targets
 
-## Curve P256 and BoringCrypto
+## Curve P256 and FIPS 140-3 mode
 
 The default curve used for cryptographic handshakes and signatures is Curve25519. This is the recommended setting for most users. If your deployment has certain compliance requirements, you have the option of creating your CA using `nebula-cert ca -curve P256` to use NIST Curve P256. The CA will then sign certificates using ECDSA P256, and any hosts using these certificates will use P256 for ECDH handshakes.
 
-In addition, Nebula can be built using the [BoringCrypto GOEXPERIMENT](https://github.com/golang/go/blob/go1.20/src/crypto/internal/boring/README.md) by running either of the following make targets:
+Nebula can be built to support the [FIPS 140-3](https://go.dev/doc/security/fips140) mode of Go by running either of the following make targets. (This sets GOFIPS140=v1.0.0, which must be done at compile time so that the correct AES-GCM can be used for FIPS 140-3 enforcement mode).
+
+```sh
+make fips140
+make fips140 test
+make release-fips140
+```
+
+Nebula can also be built using the [BoringCrypto GOEXPERIMENT](https://github.com/golang/go/blob/go1.20/src/crypto/internal/boring/README.md) by running either of the following make targets.
 
 ```sh
 make bin-boringcrypto
 make release-boringcrypto
 ```
 
+NOTE: boringcrypto support is deprecated and will be removed in the next release. Users should migrate to the native FIPS 140-3 mode described above.
+
 This is not the recommended default deployment, but may be useful based on your compliance requirements.
 
 ## Credits

cmd/nebula-cert/ca.go

@@ -3,6 +3,7 @@ package main
 import (
 	"crypto/ecdsa"
 	"crypto/elliptic"
+	"crypto/fips140"
 	"crypto/rand"
 	"flag"
 	"fmt"
@@ -43,6 +44,13 @@ type caFlags struct {
 	subnets *string
 }
 
+func defaultCurve() string {
+	if fips140.Enforced() {
+		return "P256"
+	}
+	return "25519"
+}
+
 func newCaFlags() *caFlags {
 	cf := caFlags{set: flag.NewFlagSet("ca", flag.ContinueOnError)}
 	cf.set.Usage = func() {}
@@ -59,7 +67,7 @@ func newCaFlags() *caFlags {
 	cf.argonParallelism = cf.set.Uint("argon-parallelism", 4, "Optional: Argon2 parallelism parameter used for encrypted private key passphrase")
 	cf.argonIterations = cf.set.Uint("argon-iterations", 1, "Optional: Argon2 iterations parameter used for encrypted private key passphrase")
 	cf.encryption = cf.set.Bool("encrypt", false, "Optional: prompt for passphrase and write out-key in an encrypted format")
-	cf.curve = cf.set.String("curve", "25519", "EdDSA/ECDSA Curve (25519, P256)")
+	cf.curve = cf.set.String("curve", defaultCurve(), "EdDSA/ECDSA Curve (25519, P256)")
 	cf.p11url = p11Flag(cf.set)
 
 	cf.ips = cf.set.String("ips", "", "Deprecated, see -networks")

cmd/nebula-cert/fips140enforce.go

@@ -0,0 +1,5 @@
+//go:build fips140enforce
+
+//go:debug fips140=only
+
+package main

cmd/nebula-cert/keygen.go

@@ -24,7 +24,7 @@ func newKeygenFlags() *keygenFlags {
 	cf.set.Usage = func() {}
 	cf.outPubPath = cf.set.String("out-pub", "", "Required: path to write the public key to")
 	cf.outKeyPath = cf.set.String("out-key", "", "Required: path to write the private key to")
-	cf.curve = cf.set.String("curve", "25519", "ECDH Curve (25519, P256)")
+	cf.curve = cf.set.String("curve", defaultCurve(), "ECDH Curve (25519, P256)")
 	cf.p11url = p11Flag(cf.set)
 	return &cf
 }

cmd/nebula-service/fips140enforce.go

@@ -0,0 +1,5 @@
+//go:build fips140enforce
+
+//go:debug fips140=only
+
+package main

cmd/nebula/fips140enforce.go

@@ -0,0 +1,5 @@
+//go:build fips140enforce
+
+//go:debug fips140=only
+
+package main

connection_manager.go

@@ -44,11 +44,6 @@ type connectionManager struct {
 	inactivityTimeout       atomic.Int64
 	dropInactive            atomic.Bool
 
-	// Wake-from-sleep handling, sampled once per tick in Start
-	wakeDetector       *wakeDetector
-	clearOnWake        atomic.Bool
-	wakeClearThreshold atomic.Int64
-
 	l *slog.Logger
 }
 
@@ -59,7 +54,6 @@ func newConnectionManagerFromConfig(l *slog.Logger, c *config.C, hm *HostMap, p
 		punchy:        p,
 		relayUsed:     make(map[uint32]struct{}),
 		relayUsedLock: &sync.RWMutex{},
-		wakeDetector:  newWakeDetector(),
 	}
 
 	cm.reload(c, true)
@@ -104,38 +98,12 @@ func (cm *connectionManager) reload(c *config.C, initial bool) {
 			)
 		}
 	}
-
-	if initial || c.HasChanged("tunnels.clear_on_wake") {
-		old := cm.clearOnWake.Load()
-		cm.clearOnWake.Store(c.GetBool("tunnels.clear_on_wake", true))
-		if !initial {
-			cm.l.Info("Clear on wake setting has changed",
-				"oldBool", old,
-				"newBool", cm.clearOnWake.Load(),
-			)
-		}
-	}
-
-	if initial || c.HasChanged("tunnels.wake_clear_threshold") {
-		old := cm.getWakeClearThreshold()
-		cm.wakeClearThreshold.Store((int64)(c.GetDuration("tunnels.wake_clear_threshold", 30*time.Second)))
-		if !initial {
-			cm.l.Info("Wake clear threshold has changed",
-				"oldDuration", old,
-				"newDuration", cm.getWakeClearThreshold(),
-			)
-		}
-	}
 }
 
 func (cm *connectionManager) getInactivityTimeout() time.Duration {
 	return (time.Duration)(cm.inactivityTimeout.Load())
 }
 
-func (cm *connectionManager) getWakeClearThreshold() time.Duration {
-	return (time.Duration)(cm.wakeClearThreshold.Load())
-}
-
 func (cm *connectionManager) In(h *HostInfo) {
 	h.in.Store(true)
 }
@@ -168,73 +136,6 @@ func (cm *connectionManager) getAndResetTrafficCheck(h *HostInfo, now time.Time)
 	return in, out
 }
 
-// checkWake runs once per tick and clears every tunnel when the machine has just returned from system sleep.
-// Tunnels rarely survive a suspend: our NAT mappings have expired and our address has usually changed, so every
-// established hostinfo is a corpse that will eat 15-20s of traffic checks before the wheel declares it dead.
-// Clearing now means the first packet after wake starts a fresh handshake immediately.
-//
-// The suspend itself costs nothing here: the ticker driving us is frozen with the rest of the process and this
-// fires within one tick of resume.
-func (cm *connectionManager) checkWake() {
-	slept, ok := cm.wakeDetector.Sample()
-	if !ok || slept == 0 {
-		return
-	}
-
-	// The clock pair is read non-atomically, so scheduling jitter shows up as tiny sub-millisecond "sleeps".
-	// Keep the floor well above that so a zero/nonsense threshold can't clear tunnels on every tick.
-	threshold := max(cm.getWakeClearThreshold(), time.Second)
-
-	if slept < threshold {
-		// Short suspends (lid closed and quickly reopened) often come back before NAT state expires; those
-		// tunnels may well be alive, leave them to the normal traffic checks.
-		if slept >= time.Second {
-			cm.l.Debug("Woke from sleep below the clear threshold, leaving tunnels alone",
-				"sleptFor", slept,
-				"threshold", threshold,
-			)
-		}
-		return
-	}
-
-	if !cm.clearOnWake.Load() {
-		cm.l.Info("Woke from sleep, tunnels.clear_on_wake is disabled so tunnels are left to the normal traffic checks", "sleptFor", slept)
-		return
-	}
-
-	closed := cm.clearAllTunnels()
-	cm.l.Info("Woke from sleep, cleared tunnels", "sleptFor", slept, "tunnelsCleared", closed)
-
-	// Our public address almost certainly changed; get it to the lighthouses as soon as possible so peers can
-	// find us again. The update rides over a fresh lighthouse handshake. If the network isn't back up yet these
-	// sends fail harmlessly and the periodic update worker retries within lighthouse.interval.
-	cm.intf.lightHouse.TriggerUpdate()
-}
-
-// clearAllTunnels closes every tunnel in the hostmap locally, without notifying the remotes. It is the wake-from-
-// sleep counterpart to Control.CloseAllTunnels: after a suspend the remotes stopped hearing from us long ago, and
-// close packets fired into a network that may not even be up yet are wasted, so we only tear down our own state
-// and let the next packet to each host start a fresh handshake.
-func (cm *connectionManager) clearAllTunnels() int {
-	cm.hostMap.RLock()
-	hostinfos := make([]*HostInfo, 0, len(cm.hostMap.Indexes))
-	for _, h := range cm.hostMap.Indexes {
-		hostinfos = append(hostinfos, h)
-	}
-	cm.hostMap.RUnlock()
-
-	for _, h := range hostinfos {
-		cm.intf.closeTunnel(h)
-	}
-
-	// With every tunnel gone no relay can be in use, drop the usage tracking wholesale.
-	cm.relayUsedLock.Lock()
-	clear(cm.relayUsed)
-	cm.relayUsedLock.Unlock()
-
-	return len(hostinfos)
-}
-
 func (cm *connectionManager) Start(ctx context.Context) {
 	clockSource := time.NewTicker(cm.trafficTimer.t.tickDuration)
 	defer clockSource.Stop()
@@ -249,7 +150,6 @@ func (cm *connectionManager) Start(ctx context.Context) {
 			return
 
 		case now := <-clockSource.C:
-			cm.checkWake()
 			cm.trafficTimer.Advance(now)
 			for {
 				localIndex, has := cm.trafficTimer.Purge()

connection_manager_test.go

@@ -25,6 +25,7 @@ func newTestLighthouse() *LightHouse {
 	lighthouses := []netip.Addr{}
 	staticList := map[netip.Addr]struct{}{}
 
+	lh.localAddrsFn = func(*LocalAllowList) []netip.Addr { return nil }
 	lh.lighthouses.Store(&lighthouses)
 	lh.staticList.Store(&staticList)
 
@@ -501,85 +502,3 @@ func (d *dummyCert) MarshalJSON() ([]byte, error) {
 func (d *dummyCert) Copy() cert.Certificate {
 	return d
 }
-
-func TestConnectionManager_WakeClear(t *testing.T) {
-	l := test.NewLogger()
-	localrange := netip.MustParsePrefix("10.1.1.1/24")
-	vpnIp := netip.MustParseAddr("172.1.1.2")
-	preferredRanges := []netip.Prefix{localrange}
-
-	// Very incomplete mock objects
-	hostMap := newHostMap(l)
-	hostMap.preferredRanges.Store(&preferredRanges)
-
-	cs := &CertState{
-		initiatingVersion: cert.Version1,
-		privateKey:        []byte{},
-		v1Cert:            &dummyCert{version: cert.Version1},
-		v1Credential:      nil,
-	}
-
-	lh := newTestLighthouse()
-	ifce := &Interface{
-		hostMap:          hostMap,
-		inside:           &overlaytest.NoopTun{},
-		outside:          &udp.NoopConn{},
-		firewall:         &Firewall{},
-		lightHouse:       lh,
-		pki:              &PKI{},
-		handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
-		l:                l,
-	}
-	ifce.pki.cs.Store(cs)
-
-	// Create manager
-	conf := config.NewC(test.NewLogger())
-	punchy := NewPunchyFromConfig(test.NewLogger(), conf, nil)
-	nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy)
-	nc.intf = ifce
-
-	// Drive the wake detector from a fake clock pair
-	suspended := time.Duration(0)
-	nc.wakeDetector = &wakeDetector{read: func() (time.Duration, bool) { return suspended, true }}
-	nc.checkWake() // primes the baseline
-
-	addTunnel := func(localIndex uint32) *HostInfo {
-		hostinfo := &HostInfo{
-			vpnAddrs:      []netip.Addr{vpnIp},
-			localIndexId:  localIndex,
-			remoteIndexId: 9901,
-		}
-		hostinfo.ConnectionState = &ConnectionState{
-			myCert: &dummyCert{version: cert.Version1},
-		}
-		nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
-		return hostinfo
-	}
-
-	addTunnel(1099)
-	nc.RelayUsed(5000)
-
-	// No suspend, nothing happens
-	nc.checkWake()
-	assert.Contains(t, nc.hostMap.Indexes, uint32(1099))
-
-	// A suspend below the threshold leaves tunnels alone
-	suspended += 5 * time.Second
-	nc.checkWake()
-	assert.Contains(t, nc.hostMap.Indexes, uint32(1099))
-
-	// A suspend past the threshold clears everything, including relay usage tracking
-	suspended += time.Hour
-	nc.checkWake()
-	assert.Empty(t, nc.hostMap.Indexes)
-	assert.Empty(t, nc.hostMap.Hosts)
-	assert.Empty(t, nc.relayUsed)
-
-	// With clear_on_wake disabled the tunnels survive a long suspend
-	addTunnel(1100)
-	nc.clearOnWake.Store(false)
-	suspended += time.Hour
-	nc.checkWake()
-	assert.Contains(t, nc.hostMap.Indexes, uint32(1100))
-	assert.Contains(t, nc.hostMap.Hosts, vpnIp)
-}

control.go

@@ -53,6 +53,7 @@ type Control struct {
 	statsStart             func()
 	dnsStart               func()
 	lighthouseStart        func()
+	networkChangeStart     func(rebind func())
 	connectionManagerStart func(context.Context)
 }
 
@@ -104,6 +105,9 @@ func (c *Control) Start() error {
 	if c.dnsStart != nil {
 		go c.dnsStart()
 	}
+	if c.networkChangeStart != nil {
+		go c.networkChangeStart(c.RebindUDPServer)
+	}
 	if c.connectionManagerStart != nil {
 		go c.connectionManagerStart(c.ctx)
 	}
@@ -198,7 +202,11 @@ func (c *Control) RebindUDPServer() {
 		return
 	}
 
-	_ = c.f.outside.Rebind()
+	// A failure here means we are likely still pinned to the interface we came up on, so the rest of this is
+	// unlikely to help. Say so instead of silently carrying on as if we rebound.
+	if err := c.f.outside.Rebind(); err != nil {
+		c.l.Error("Failed to rebind udp socket", "error", err)
+	}
 
 	// Trigger a lighthouse update, useful for mobile clients that should have an update interval of 0
 	c.f.lightHouse.SendUpdate()

control_tester.go

@@ -108,7 +108,19 @@ func (c *Control) GetVpnAddrs() []netip.Addr {
 }
 
 func (c *Control) GetUDPAddr() netip.AddrPort {
-	return c.f.outside.(*udp.TesterConn).Addr
+	return c.f.outside.(*udp.TesterConn).GetAddr()
+}
+
+// SetUDPAddr moves this node to a new underlay address, standing in for a laptop waking up on a different
+// network. Register the new address with the router as well or nothing will route back.
+func (c *Control) SetUDPAddr(addr netip.AddrPort) {
+	c.f.outside.(*udp.TesterConn).SetAddr(addr)
+}
+
+// SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of
+// whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker.
+func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {
+	c.f.lightHouse.localAddrsFn = fn
 }
 
 func (c *Control) KillPendingTunnel(vpnIp netip.Addr) bool {

e2e/rebind_test.go

@@ -0,0 +1,225 @@
+//go:build e2e_testing
+// +build e2e_testing
+
+package e2e
+
+import (
+	"net/netip"
+	"testing"
+	"time"
+
+	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/cert_test"
+	"github.com/slackhq/nebula/e2e/router"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/udp"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// reportedAddrs is what the lighthouse would hand a peer asking where vpnAddr is.
+func reportedAddrs(t *testing.T, lh *nebula.Control, vpnAddr netip.Addr) []netip.AddrPort {
+	t.Helper()
+	cm := lh.QueryLighthouse(vpnAddr)
+	if cm == nil {
+		return nil
+	}
+	var out []netip.AddrPort
+	for _, c := range *cm {
+		out = append(out, c.Reported...)
+		out = append(out, c.Learned...)
+	}
+	return out
+}
+
+// waitForLighthouseMsg routes until a lighthouse message lands on lh, or gives up. Reports whether one arrived.
+func waitForLighthouseMsg(t *testing.T, r *router.R, lh *nebula.Control, wait time.Duration) bool {
+	t.Helper()
+	h := &header.H{}
+	return r.RouteForAllExitFuncOrTimeout(wait, func(p *udp.Packet, c *nebula.Control) router.ExitType {
+		if c != lh {
+			return router.KeepRouting
+		}
+		// Punches are a single byte and never parse, they are just not what we are after
+		if err := h.Parse(p.Data); err != nil {
+			return router.KeepRouting
+		}
+		if h.Type == header.LightHouse {
+			return router.RouteAndExit
+		}
+		return router.KeepRouting
+	})
+}
+
+// A laptop that changes networks has to tell the lighthouse promptly, otherwise the lighthouse keeps handing peers
+// the old address and their punches land nowhere. On a long lighthouse interval the only thing that closes that
+// window is the rebind, which on darwin the network change monitor drives. The e2e build compiles the monitor out,
+// so we call RebindUDPServer directly, which is the same thing the monitor does.
+func TestRebindSendsLighthouseUpdate(t *testing.T) {
+	t.Parallel()
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
+	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
+		"lighthouse": m{"am_lighthouse": true},
+	})
+
+	// 600s interval, so nothing scheduled can send an update during this test. A rebind is the only thing that can.
+	myControl, _, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{
+		"lighthouse": m{
+			"hosts":    []any{lhVpnIpNet[0].Addr().String()},
+			"interval": 600,
+		},
+		"static_host_map": m{
+			lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
+		},
+	})
+
+	r := router.NewR(t, lhControl, myControl)
+	defer r.RenderFlow()
+
+	lhControl.Start()
+	myControl.Start()
+
+	// Let the startup registration finish, then clear everything it left behind
+	require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an initial registration")
+	r.RouteFor(time.Millisecond * 400)
+
+	// Nothing should be talking to the lighthouse on its own now
+	require.False(t, waitForLighthouseMsg(t, r, lhControl, time.Millisecond*200),
+		"nothing should reach the lighthouse before the rebind")
+
+	myControl.RebindUDPServer()
+
+	assert.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5),
+		"a rebind should push an update to the lighthouse rather than waiting out the interval")
+
+	lhControl.Stop()
+	myControl.Stop()
+}
+
+// The other half of a rebind: every live tunnel requeries the lighthouse on its next send. That query is what makes
+// the lighthouse tell the peer to punch toward our new address, which is the part that actually revives a tunnel
+// whose remote NAT state died while we were on a different network.
+func TestRebindRequeriesPeersOnNextSend(t *testing.T) {
+	t.Parallel()
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
+	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
+		"lighthouse": m{"am_lighthouse": true},
+	})
+
+	lhCfg := m{
+		"lighthouse": m{
+			"hosts":    []any{lhVpnIpNet[0].Addr().String()},
+			"interval": 600,
+			// Without this the peers advertise this machine's real addresses and then try to punch at them,
+			// which the router has no route for.
+			"local_allow_list": m{
+				"10.0.0.0/24": true,
+				"::/0":        false,
+			},
+		},
+		"static_host_map": m{
+			lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
+		},
+	}
+
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", lhCfg)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.3/24", lhCfg)
+
+	r := router.NewR(t, lhControl, myControl, theirControl)
+	defer r.RenderFlow()
+
+	lhControl.Start()
+	myControl.Start()
+	theirControl.Start()
+	r.RouteFor(time.Millisecond * 500)
+
+	// Point the peers at each other directly, this test is about the rebind and not about lighthouse discovery
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("initial")))
+	r.RouteFor(time.Second)
+	require.NotNil(t, myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false), "expected a tunnel to them")
+	r.RouteFor(time.Millisecond * 300)
+
+	// Assert on what the peer sees rather than on lighthouse traffic. A query for them makes the lighthouse send
+	// them a punch notification, which is the whole point. Our own update to the lighthouse sends them nothing,
+	// so this cannot be satisfied by the update the rebind itself pushes.
+	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("quiet")))
+	require.False(t, waitForLighthouseMsg(t, r, theirControl, time.Millisecond*300),
+		"an ordinary send should not requery the lighthouse")
+
+	myControl.RebindUDPServer()
+	r.RouteFor(time.Millisecond * 300) // let the update the rebind itself sends pass by
+
+	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("after rebind")))
+	assert.True(t, waitForLighthouseMsg(t, r, theirControl, time.Second*5),
+		"the first send after a rebind should requery the lighthouse, which then tells the peer to punch at us")
+
+	lhControl.Stop()
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+// The scenario this whole thing exists for: a laptop sleeps at the office and wakes up at home on a new address.
+// Until it tells the lighthouse, the lighthouse keeps handing peers the office address, so their punches land
+// nowhere and the tunnel stays dead. On a long interval the rebind is the only thing that closes that window.
+func TestRebindAdvertisesNewAddressAfterMove(t *testing.T) {
+	t.Parallel()
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
+	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
+		"lighthouse": m{"am_lighthouse": true},
+	})
+
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{
+		"lighthouse": m{
+			"hosts":    []any{lhVpnIpNet[0].Addr().String()},
+			"interval": 600,
+		},
+		"static_host_map": m{
+			lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
+		},
+	})
+
+	// Advertise wherever we currently are rather than this machine's real NICs, read fresh each time so a move
+	// is picked up.
+	myControl.SetLocalAddrsFn(func(*nebula.LocalAllowList) []netip.Addr {
+		return []netip.Addr{myControl.GetUDPAddr().Addr()}
+	})
+
+	r := router.NewR(t, lhControl, myControl)
+	defer r.RenderFlow()
+
+	lhControl.Start()
+	myControl.Start()
+
+	require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an initial registration")
+	r.RouteFor(time.Millisecond * 400)
+
+	require.Contains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), myUdpAddr,
+		"the lighthouse should know the address we started on")
+
+	// Wake up somewhere else
+	newAddr := netip.MustParseAddrPort("10.0.0.99:4242")
+	myControl.SetUDPAddr(newAddr)
+	r.AddRoute(newAddr.Addr(), newAddr.Port(), myControl)
+
+	// Nothing has told the lighthouse, and with interval 600 nothing scheduled will
+	r.RouteFor(time.Millisecond * 400)
+	require.NotContains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), newAddr,
+		"the lighthouse should still be handing out the old address before the rebind")
+
+	myControl.RebindUDPServer()
+	require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an update after the rebind")
+	r.RouteFor(time.Millisecond * 400)
+
+	assert.Contains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), newAddr,
+		"after the rebind the lighthouse should hand peers our new address")
+
+	lhControl.Stop()
+	myControl.Stop()
+}

e2e/router/router.go

@@ -114,6 +114,28 @@ type packet struct {
 	packet *udp.Packet
 	tun    bool // a packet pulled off a tun device
 	rx     bool // the packet was received by a udp device
+
+	// h is the nebula header, parsed once when the packet is recorded. parseErr says why there isn't one, which
+	// the flow log reports rather than hiding. Punchy sends a single byte, so an unparseable packet is normal.
+	h        header.H
+	parseErr error
+}
+
+// fromAddr and toAddr are the addresses this packet actually travelled between. Reading them off the control
+// instead would misreport the whole history once a test moves a node. Tun packets are synthesized without
+// addresses, so they fall back to the control.
+func (p *packet) fromAddr() netip.AddrPort {
+	if p.tun || !p.packet.From.IsValid() {
+		return p.from.GetUDPAddr()
+	}
+	return p.packet.From
+}
+
+func (p *packet) toAddr() netip.AddrPort {
+	if p.tun || !p.packet.To.IsValid() {
+		return p.to.GetUDPAddr()
+	}
+	return p.packet.To
 }
 
 func (p *packet) WasReceived() {
@@ -249,7 +271,7 @@ func (r *R) renderFlow() {
 			continue
 		}
 
-		addr := e.packet.from.GetUDPAddr()
+		addr := e.packet.fromAddr()
 		if _, ok := participants[addr]; ok {
 			continue
 		}
@@ -268,7 +290,6 @@ func (r *R) renderFlow() {
 	}
 
 	// Print packets
-	h := &header.H{}
 	for _, e := range r.flow {
 		if e.packet == nil {
 			//fmt.Fprintf(f, "    note over %s: %s\n", strings.Join(participantsVals, ", "), e.note)
@@ -280,21 +301,22 @@ func (r *R) renderFlow() {
 			fmt.Fprintln(f, r.formatUdpPacket(p))
 
 		} else {
-			if err := h.Parse(p.packet.Data); err != nil {
-				panic(err)
-			}
-
 			line := "--x"
 			if p.rx {
 				line = "->>"
 			}
 
-			fmt.Fprintf(f,
+			detail := fmt.Sprintf("%s(%s), index %v, counter: %v",
-				"    %s%s%s: %s(%s), index %v, counter: %v\n",
+				p.h.TypeName(), p.h.SubTypeName(), p.h.RemoteIndex, p.h.MessageCounter)
-				normalizeName(p.from.GetUDPAddr().String()),
+			if p.parseErr != nil {
+				detail = fmt.Sprintf("unparsed, %v (%d bytes)", p.parseErr, len(p.packet.Data))
+			}
+
+			fmt.Fprintf(f, "    %s%s%s: %s\n",
+				normalizeName(p.fromAddr().String()),
 				line,
-				normalizeName(p.to.GetUDPAddr().String()),
+				normalizeName(p.toAddr().String()),
-				h.TypeName(), h.SubTypeName(), h.RemoteIndex, h.MessageCounter,
+				detail,
 			)
 		}
 	}
@@ -408,29 +430,34 @@ func (r *R) unlockedInjectFlow(from, to *nebula.Control, p *udp.Packet, tun bool
 
 	r.renderHostmaps(fmt.Sprintf("Packet %v", len(r.flow)))
 
-	if len(r.ignoreFlows) > 0 {
+	var h header.H
-		var h header.H
+	var parseErr error
-		err := h.Parse(p.Data)
+	if !tun {
-		if err != nil {
+		parseErr = h.Parse(p.Data)
-			panic(err)
+	}
-		}
 
-		for _, i := range r.ignoreFlows {
+	// Decide before copying, the copy comes from a freelist and an ignored packet would never be released
-			if !tun {
+	for _, i := range r.ignoreFlows {
-				if i.messageType == h.Type && i.subType == h.Subtype {
+		if tun {
-					return nil
+			if i.tun.HasValue && i.tun.IsTrue {
-				}
-			} else if i.tun.HasValue && i.tun.IsTrue {
 				return nil
 			}
+			continue
+		}
+
+		// A packet we could not parse has no type to match against, so no rule can ignore it
+		if parseErr == nil && i.messageType == h.Type && i.subType == h.Subtype {
+			return nil
 		}
 	}
 
 	fp := &packet{
-		from:   from,
+		from:     from,
-		to:     to,
+		to:       to,
-		packet: p.Copy(),
+		packet:   p.Copy(),
-		tun:    tun,
+		tun:      tun,
+		h:        h,
+		parseErr: parseErr,
 	}
 
 	r.flow = append(r.flow, flowEntry{packet: fp})
@@ -690,6 +717,81 @@ func (r *R) RouteUntilAfterMsgType(sender *nebula.Control, msgType header.Messag
 	})
 }
 
+// RouteFor routes everything that shows up for the given duration and then returns. Use it to let a test settle
+// deterministically rather than sleeping and hoping: a single FlushAll races a completing handshake, which queues
+// more packets right behind it.
+func (r *R) RouteFor(d time.Duration) {
+	r.RouteForAllExitFuncOrTimeout(d, func(*udp.Packet, *nebula.Control) ExitType {
+		return KeepRouting
+	})
+}
+
+// RouteForAllExitFuncOrTimeout is RouteForAllExitFunc with a deadline, reporting whether whatDo asked to exit
+// before time ran out. The unbounded version blocks forever on a quiet network, so this is what a test needs to
+// assert that something does NOT happen, or to route for a fixed settling period.
+func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc) bool {
+	sc := make([]reflect.SelectCase, 0, len(r.controls)+1)
+	cm := make([]*nebula.Control, 0, len(r.controls))
+
+	for _, c := range r.controls {
+		sc = append(sc, reflect.SelectCase{
+			Dir:  reflect.SelectRecv,
+			Chan: reflect.ValueOf(c.GetUDPTxChan()),
+			Send: reflect.Value{},
+		})
+		cm = append(cm, c)
+	}
+
+	timer := time.NewTimer(timeout)
+	defer timer.Stop()
+	sc = append(sc, reflect.SelectCase{
+		Dir:  reflect.SelectRecv,
+		Chan: reflect.ValueOf(timer.C),
+		Send: reflect.Value{},
+	})
+
+	for {
+		x, rx, _ := reflect.Select(sc)
+		if x == len(cm) {
+			return false
+		}
+
+		r.Lock()
+		p := rx.Interface().(*udp.Packet)
+		receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
+		if receiver == nil {
+			r.Unlock()
+			panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String())
+		}
+
+		e := whatDo(p, receiver)
+		switch e {
+		case ExitNow:
+			r.Unlock()
+			p.Release()
+			return true
+
+		case RouteAndExit:
+			fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
+			receiver.InjectUDPPacket(p)
+			fp.WasReceived()
+			r.Unlock()
+			p.Release()
+			return true
+
+		case KeepRouting:
+			fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
+			receiver.InjectUDPPacket(p)
+			fp.WasReceived()
+
+		default:
+			panic(fmt.Sprintf("Unknown exitFunc return: %v", e))
+		}
+		r.Unlock()
+		p.Release()
+	}
+}
+
 func (r *R) RouteForAllUntilAfterMsgTypeTo(receiver *nebula.Control, msgType header.MessageType, subType header.MessageSubType) {
 	h := &header.H{}
 	r.RouteForAllExitFunc(func(p *udp.Packet, r *nebula.Control) ExitType {

examples/config.yml

@@ -146,6 +146,14 @@ listen:
   # Default true; set to false to leave WDF in charge of inbound decisions on the listener port. Not reloadable.
   #windows_bypass_wdf: true
 
+  # On macOS only
+  # macOS scopes the udp socket to the interface it was created on, so moving between networks (wifi to wired,
+  # office to home) leaves Nebula sending out an interface that no longer has a route. When true, Nebula watches
+  # the routing socket and rebinds the listener once the change settles.
+  # iOS does not use this, the host app drives the same rebind itself.
+  # Default true. Not reloadable.
+  #rebind_on_network_change: true
+
   # By default, Nebula replies to packets it has no tunnel for with a "recv_error" packet. This packet helps speed up reconnection
   # in the case that Nebula on either side did not shut down cleanly. This response can be abused as a way to discover if Nebula is running
   # on a host though. This option lets you configure if you want to send "recv_error" packets always, never, or only to private network remotes.
@@ -390,20 +398,6 @@ logging:
   # This setting is reloadable
   #inactivity_timeout: 10m
 
-  # clear_on_wake controls whether all tunnels are immediately torn down (locally, without notifying the remotes)
-  # when the machine detects it has just woken from system sleep. Tunnels rarely survive a suspend: NAT mappings
-  # expire and the machine's address usually changes, so waiting for the normal liveness checks costs 15-20 seconds
-  # of black-holed traffic per tunnel after wake. Clearing them means the first packet after wake starts a fresh
-  # handshake right away.
-  # This setting is reloadable
-  #clear_on_wake: true
-
-  # wake_clear_threshold is the minimum time the machine must have been suspended for clear_on_wake to act.
-  # Suspends shorter than this often come back before NAT state expires, so those tunnels may still be alive and
-  # are left to the normal liveness checks. Values below 1s are treated as 1s.
-  # This setting is reloadable
-  #wake_clear_threshold: 30s
-
 # Nebula security group configuration
 firewall:
   # Action to take when a packet is not allowed by the firewall rules.

go.mod

@@ -1,6 +1,6 @@
 module github.com/slackhq/nebula
 
-go 1.25.0
+go 1.26.0
 
 require (
 	dario.cat/mergo v1.0.2

handshake_manager.go

@@ -295,7 +295,13 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 		hm.messageMetrics.Tx(header.Handshake, hh.machine.Subtype(), 1)
 		err := hm.outside.WriteTo(stage0, addr)
 		if err != nil {
-			hostinfo.logger(hm.l).Error("Failed to send handshake message",
+			// These repeat every attempt, so match the success log below and only shout when the remotes changed
+			level := slog.LevelDebug
+			if remotesHaveChanged {
+				level = slog.LevelError
+			}
+
+			hostinfo.logger(hm.l).Log(context.Background(), level, "Failed to send handshake message",
 				"udpAddr", addr,
 				"initiatorIndex", hostinfo.localIndexId,
 				"handshake", hsFields,

inside.go

@@ -408,7 +408,7 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 		if err != nil {
 			hostinfo.logger(f.l).Error("Failed to write outgoing packet",
 				"error", err,
-				"udpAddr", remote,
+				"udpAddr", hr,
 			)
 		}
 	} else {

interface.go

@@ -2,6 +2,7 @@ package nebula
 
 import (
 	"context"
+	"crypto/fips140"
 	"errors"
 	"fmt"
 	"io"
@@ -238,6 +239,9 @@ func (f *Interface) activate() error {
 		"build", f.version,
 		"udpAddr", addr,
 		"boringcrypto", boringEnabled(),
+		"fips140Version", fips140.Version(),
+		"fips140Enabled", fips140.Enabled(),
+		"fips140Enforced", fips140.Enforced(),
 	)
 
 	if f.routines > 1 {

lighthouse.go

@@ -36,6 +36,10 @@ type LightHouse struct {
 	myVpnNetworksTable *bart.Lite
 	punchy             *Punchy
 
+	// localAddrsFn enumerates the underlay addresses we advertise. It is a field so tests can supply simulated
+	// addresses rather than whatever this machine's NICs happen to be. Set it before Start.
+	localAddrsFn func(*LocalAllowList) []netip.Addr
+
 	// Local cache of answers from light houses
 	// map of vpn addr to answers
 	addrMap map[netip.Addr]*RemoteList
@@ -107,6 +111,10 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
 		queryChan:          make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)),
 		l:                  l,
 	}
+	h.localAddrsFn = func(al *LocalAllowList) []netip.Addr {
+		return localAddrs(h.l, al)
+	}
+
 	lighthouses := make([]netip.Addr, 0)
 	h.lighthouses.Store(&lighthouses)
 	staticList := make(map[netip.Addr]struct{})
@@ -918,7 +926,7 @@ func (lh *LightHouse) SendUpdate() {
 	}
 
 	lal := lh.GetLocalAllowList()
-	for _, e := range localAddrs(lh.l, lal) {
+	for _, e := range lh.localAddrsFn(lal) {
 		if lh.myVpnNetworksTable.Contains(e) {
 			continue
 		}

main.go

@@ -268,6 +268,8 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 
 	attachCommands(l, c, ssh, ifce)
 
+	networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c)
+
 	return &Control{
 		state:                  StateReady,
 		f:                      ifce,
@@ -278,6 +280,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 		statsStart:             stats.Start,
 		dnsStart:               ds.Start,
 		lighthouseStart:        lightHouse.StartUpdateWorker,
+		networkChangeStart:     networkChanges.Start,
 		connectionManagerStart: connManager.Start,
 	}, nil
 }

noiseutil/boring.go

@@ -4,77 +4,16 @@
 package noiseutil
 
 import (
-	"crypto/aes"
+	"crypto/boring"
-	"crypto/cipher"
-	"encoding/binary"
-
-	// unsafe needed for go:linkname
-	_ "unsafe"
 
 	"github.com/flynn/noise"
 )
 
+var CipherAESGCM noise.CipherFunc = CipherAESGCMFIPS140
+
 // EncryptLockNeeded indicates if calls to Encrypt need a lock
 // This is true for boringcrypto because the Seal function verifies that the
 // nonce is strictly increasing.
 const EncryptLockNeeded = true
 
-// NewGCMTLS is no longer exposed in go1.19+, so we need to link it in
+var boringEnabled = boring.Enabled()
-// See: https://github.com/golang/go/issues/56326
-//
-// NewGCMTLS is the internal method used with boringcrypto that provides a
-// validated mode of AES-GCM which enforces the nonce is strictly
-// monotonically increasing.  This is the TLS 1.2 specification for nonce
-// generation (which also matches the method used by the Noise Protocol)
-//
-// - https://github.com/golang/go/blob/go1.19/src/crypto/tls/cipher_suites.go#L520-L522
-// - https://github.com/golang/go/blob/go1.19/src/crypto/internal/boring/aes.go#L235-L237
-// - https://github.com/golang/go/blob/go1.19/src/crypto/internal/boring/aes.go#L250
-// - https://github.com/google/boringssl/blob/ae223d6138807a13006342edfeef32e813246b39/include/openssl/aead.h#L379-L381
-// - https://github.com/google/boringssl/blob/ae223d6138807a13006342edfeef32e813246b39/crypto/fipsmodule/cipher/e_aes.c#L1082-L1093
-//
-//go:linkname newGCMTLS crypto/internal/boring.NewGCMTLS
-func newGCMTLS(c cipher.Block) (cipher.AEAD, error)
-
-type cipherFn struct {
-	fn   func([32]byte) noise.Cipher
-	name string
-}
-
-func (c cipherFn) Cipher(k [32]byte) noise.Cipher { return c.fn(k) }
-func (c cipherFn) CipherName() string             { return c.name }
-
-// CipherAESGCM is the AES256-GCM AEAD cipher (using NewGCMTLS when GoBoring is present)
-var CipherAESGCM noise.CipherFunc = cipherFn{cipherAESGCMBoring, "AESGCM"}
-
-func cipherAESGCMBoring(k [32]byte) noise.Cipher {
-	c, err := aes.NewCipher(k[:])
-	if err != nil {
-		panic(err)
-	}
-	gcm, err := newGCMTLS(c)
-	if err != nil {
-		panic(err)
-	}
-	return aeadCipher{
-		gcm,
-		func(n uint64) []byte {
-			var nonce [12]byte
-			binary.BigEndian.PutUint64(nonce[4:], n)
-			return nonce[:]
-		},
-	}
-}
-
-type aeadCipher struct {
-	cipher.AEAD
-	nonce func(uint64) []byte
-}
-
-func (c aeadCipher) Encrypt(out []byte, n uint64, ad, plaintext []byte) []byte {
-	return c.Seal(out, c.nonce(n), plaintext, ad)
-}
-
-func (c aeadCipher) Decrypt(out []byte, n uint64, ad, ciphertext []byte) ([]byte, error) {
-	return c.Open(out, c.nonce(n), ciphertext, ad)
-}

noiseutil/boring_test.go

@@ -4,8 +4,6 @@
 package noiseutil
 
 import (
-	"crypto/boring"
-	"encoding/hex"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -14,33 +12,3 @@ import (
 func TestEncryptLockNeeded(t *testing.T) {
 	assert.True(t, EncryptLockNeeded)
 }
-
-// Ensure NewGCMTLS validates the nonce is non-repeating
-func TestNewGCMTLS(t *testing.T) {
-	assert.True(t, boring.Enabled())
-
-	// Test Case 16 from GCM Spec:
-	//  - (now dead link): http://csrc.nist.gov/groups/ST/toolkit/BCM/documents/proposedmodes/gcm/gcm-spec.pdf
-	//  - as listed in boringssl tests: https://github.com/google/boringssl/blob/fips-20220613/crypto/cipher_extra/test/cipher_tests.txt#L412-L418
-	key, _ := hex.DecodeString("feffe9928665731c6d6a8f9467308308feffe9928665731c6d6a8f9467308308")
-	iv, _ := hex.DecodeString("cafebabefacedbaddecaf888")
-	plaintext, _ := hex.DecodeString("d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39")
-	aad, _ := hex.DecodeString("feedfacedeadbeeffeedfacedeadbeefabaddad2")
-	expected, _ := hex.DecodeString("522dc1f099567d07f47f37a32a84427d643a8cdcbfe5c0c97598a2bd2555d1aa8cb08e48590dbb3da7b08b1056828838c5f61e6393ba7a0abcc9f662")
-	expectedTag, _ := hex.DecodeString("76fc6ece0f4e1768cddf8853bb2d551b")
-
-	expected = append(expected, expectedTag...)
-
-	var keyArray [32]byte
-	copy(keyArray[:], key)
-	c := CipherAESGCM.Cipher(keyArray)
-	aead := c.(aeadCipher).AEAD
-
-	dst := aead.Seal([]byte{}, iv, plaintext, aad)
-	assert.Equal(t, expected, dst)
-
-	// We expect this to fail since we are re-encrypting with a repeat IV
-	assert.PanicsWithError(t, "boringcrypto: EVP_AEAD_CTX_seal failed", func() {
-		dst = aead.Seal([]byte{}, iv, plaintext, aad)
-	})
-}

noiseutil/cipher_state.go

@@ -29,8 +29,11 @@ type CipherState interface {
 // NewCipherState wraps the post-handshake noise.CipherState in the per-cipher type that matches cipherFunc.
 // cipherFunc must be the same cipher used to build the noise CipherSuite that produced s.
 func NewCipherState(s *noise.CipherState, cipherFunc noise.CipherFunc) CipherState {
+	if cs, ok := s.Cipher().(CipherState); ok {
+		return cs
+	}
 	switch cipherFunc.CipherName() {
-	case CipherAESGCM.CipherName():
+	case noise.CipherAESGCM.CipherName():
 		return NewCipherStateAESGCM(s)
 	case noise.CipherChaChaPoly.CipherName():
 		return NewCipherStateChaChaPoly(s)

noiseutil/cipher_state_test.go

@@ -1,6 +1,7 @@
 package noiseutil
 
 import (
+	"crypto/fips140"
 	"testing"
 
 	"github.com/flynn/noise"
@@ -10,24 +11,30 @@ import (
 
 func TestCipherStateAESGCMRoundtrip(t *testing.T) {
 	enc, dec := buildCipherStates(t, CipherAESGCM)
-	roundtrip(t, NewCipherStateAESGCM(enc), NewCipherStateAESGCM(dec))
+	roundtrip(t, NewCipherState(enc, CipherAESGCM), NewCipherState(dec, CipherAESGCM))
 }
 
 func TestCipherStateChaChaPolyRoundtrip(t *testing.T) {
 	enc, dec := buildCipherStates(t, noise.CipherChaChaPoly)
-	roundtrip(t, NewCipherStateChaChaPoly(enc), NewCipherStateChaChaPoly(dec))
+	roundtrip(t, NewCipherState(enc, noise.CipherChaChaPoly), NewCipherState(dec, noise.CipherChaChaPoly))
 }
 
 func TestNewCipherStateDispatch(t *testing.T) {
 	encA, _ := buildCipherStates(t, CipherAESGCM)
 	encC, _ := buildCipherStates(t, noise.CipherChaChaPoly)
 
-	assert.IsType(t, &CipherStateAESGCM{}, NewCipherState(encA, CipherAESGCM))
+	if !boringEnabled && !fips140.Enabled() {
+		assert.IsType(t, &CipherStateAESGCM{}, NewCipherState(encA, CipherAESGCM))
+	} else {
+		// fips140
+		assert.IsType(t, encA.Cipher(), NewCipherState(encA, CipherAESGCM))
+	}
+
 	assert.IsType(t, &CipherStateChaChaPoly{}, NewCipherState(encC, noise.CipherChaChaPoly))
 }
 
 func TestNewCipherStateUnsupportedPanics(t *testing.T) {
-	enc, _ := buildCipherStates(t, CipherAESGCM)
+	enc, _ := buildCipherStates(t, noise.CipherChaChaPoly)
 	assert.Panics(t, func() {
 		NewCipherState(enc, fakeCipher{})
 	})

noiseutil/fips140.go

@@ -0,0 +1,179 @@
+package noiseutil
+
+import (
+	"bytes"
+	"crypto/cipher"
+	"crypto/fips140"
+	"encoding/binary"
+	"fmt"
+	"reflect"
+	"runtime"
+	"unsafe"
+
+	// unsafe needed for go:linkname
+	_ "crypto/tls"
+	_ "unsafe"
+
+	"github.com/flynn/noise"
+)
+
+// TODO: Use NewGCMWithCounterNonce or NewGCMForQUIC once available:
+// - https://github.com/golang/go/issues/73110
+// - https://github.com/golang/go/issues/79219
+// Using tls.aeadAESGCMTLS13 gives us the TLS 1.3 GCM, which also verifies
+// that the nonce is strictly increasing. This works for both boringcrypto
+// and fips140.
+//
+//go:linkname aeadAESGCMTLS13 crypto/tls.aeadAESGCMTLS13
+func aeadAESGCMTLS13(key, noncePrefix []byte) cipher.AEAD
+
+type cipherFn struct {
+	fn   func([32]byte) noise.Cipher
+	name string
+}
+
+func (c cipherFn) Cipher(k [32]byte) noise.Cipher { return c.fn(k) }
+func (c cipherFn) CipherName() string             { return c.name }
+
+// CipherAESGCMFIPS140 is the AES256-GCM AEAD cipher (using tls.aeadAESGCMTLS13, for both boringcrypto and fips140)
+var CipherAESGCMFIPS140 noise.CipherFunc = cipherFn{cipherAESGCMFIPS140, "AESGCM"}
+
+// tls.aeadAESGCMTLS13 uses a 4 byte static prefix and an 8 byte XOR mask
+var emptyNonce = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
+
+func cipherAESGCMFIPS140(k [32]byte) noise.Cipher {
+	gcm := aeadAESGCMTLS13(k[:], emptyNonce)
+	gcm = extractFIPSAEAD(gcm)
+	return &aeadGCMFIPS140Cipher{
+		AEAD: gcm,
+	}
+}
+
+type aeadGCMFIPS140Cipher struct {
+	cipher.AEAD
+	ready bool
+}
+
+// Extract the internal FIPS GCM implementation from the tls wrapper. The TLS
+// wrapper is not thread safe around Open, so instead of locking around it we
+// can grab the internal implementation that is thread safe. This is the FIPS
+// module implementation: `crypto/internal/fips140/aes/gcm.GCMWithXORCounterNonce`
+//
+// - https://github.com/golang/go/blob/go1.26.4/src/crypto/internal/fips140/aes/gcm/gcm_nonces.go#L212-L287
+//
+// The wrapper is struct `crypto/tls.xorNonceAEAD` , with field `aead`:
+//
+// - https://github.com/golang/go/blob/go1.26.4/src/crypto/tls/cipher_suites.go#L482-L487
+//
+// This can be cleaned up once these FIPS implementations are exposed directly:
+//
+// - https://github.com/golang/go/issues/73110
+func extractFIPSAEAD(xorNonceAEAD cipher.AEAD) cipher.AEAD {
+	r := reflect.ValueOf(xorNonceAEAD)
+	v := r.Elem().FieldByName("aead")
+	if !v.IsValid() {
+		// The internal crypto/tls.xorNonceAEAD struct no longer has an `aead`
+		// field. This can only happen on a Go version this code was not built
+		// against; the package init() self-test guards against ever reaching
+		// this at runtime, so this is a defensive fail-fast.
+		panic(fmt.Sprintf("noiseutil: could not extract FIPS AEAD from %T on %s: no `aead` field (incompatible Go version)", xorNonceAEAD, runtime.Version()))
+	}
+	v2 := reflect.NewAt(v.Type(), unsafe.Pointer(v.UnsafeAddr())).Elem()
+	aead, ok := v2.Interface().(cipher.AEAD)
+	if !ok {
+		panic(fmt.Sprintf("noiseutil: extracted FIPS `aead` field is %s, not a cipher.AEAD, on %s (incompatible Go version)", v2.Type(), runtime.Version()))
+	}
+	return aead
+}
+
+func (c *aeadGCMFIPS140Cipher) init(nonce []byte) {
+	// GCMWithXORCounterNonce expects that the first call to Seal
+	// is with a counter of `0`, this is how it extracts the nonce mask.
+	// We can clean this up in the future when NewGCMWithCounterNonce or
+	// NewGCMForQUIC are available:
+	if !bytes.Equal(emptyNonce, nonce) {
+		c.AEAD.Seal([]byte{}, emptyNonce, []byte{}, []byte{})
+	}
+	c.ready = true
+}
+
+func (c *aeadGCMFIPS140Cipher) Seal(dst, nonce, plaintext, additionalData []byte) []byte {
+	if !c.ready {
+		c.init(nonce)
+	}
+	return c.AEAD.Seal(dst, nonce, plaintext, additionalData)
+}
+
+func (c *aeadGCMFIPS140Cipher) Encrypt(out []byte, n uint64, ad, plaintext []byte) []byte {
+	return c.Seal(out, aeadGCMFIPS140CipherNonce(n), plaintext, ad)
+}
+
+func (c *aeadGCMFIPS140Cipher) Decrypt(out []byte, n uint64, ad, ciphertext []byte) ([]byte, error) {
+	return c.Open(out, aeadGCMFIPS140CipherNonce(n), ciphertext, ad)
+}
+
+func (c *aeadGCMFIPS140Cipher) EncryptDanger(out, ad, plaintext []byte, n uint64, nb []byte) ([]byte, error) {
+	binary.BigEndian.PutUint64(nb[4:], n)
+	out = c.Seal(out, nb, plaintext, ad)
+	return out, nil
+}
+
+func (c *aeadGCMFIPS140Cipher) DecryptDanger(out, ad, ciphertext []byte, n uint64, nb []byte) ([]byte, error) {
+	binary.BigEndian.PutUint64(nb[4:], n)
+	return c.Open(out, nb, ciphertext, ad)
+}
+
+func aeadGCMFIPS140CipherNonce(n uint64) []byte {
+	// GCMWithXORCounterNonce uses a 4 byte static prefix and an 8 byte nonce
+	var nonce [12]byte
+	binary.BigEndian.PutUint64(nonce[4:], n)
+	return nonce[:]
+}
+
+// init validates the go:linkname + reflection extraction and the nonce-reuse
+// protection at startup, in every build. cipherAESGCMFIPS140 relies on unexported
+// crypto/tls and crypto/internal/fips140 internals; if a future Go version changes
+// those, this fails fast with a clear message instead of panicking per-handshake
+// (or, worse, silently losing the strictly-increasing nonce check that is the whole
+// point of using this cipher). Because this file has no build tag, this self-test
+// runs even in non-FIPS builds, so the default CI lane catches an incompatible Go.
+func init() {
+	var key [32]byte
+	c := cipherAESGCMFIPS140(key)
+
+	// Verify the extracted AEAD produces a working encrypt/decrypt roundtrip.
+	plaintext := []byte("nebula fips140 self-test")
+	ad := []byte("ad")
+	ct := c.Encrypt(nil, 1, ad, plaintext)
+	pt, err := c.Decrypt(nil, 1, ad, ct)
+	if err != nil {
+		panic(fmt.Sprintf("noiseutil: FIPS AES-GCM self-test roundtrip failed on %s: %v", runtime.Version(), err))
+	}
+	if !bytes.Equal(pt, plaintext) {
+		panic(fmt.Sprintf("noiseutil: FIPS AES-GCM self-test roundtrip returned wrong plaintext on %s", runtime.Version()))
+	}
+
+	// Verify the nonce-reuse protection still fires: re-encrypting with the same
+	// counter must panic. This is the guarantee we depend on for nonce safety, so
+	// if the extraction ever silently yields an AEAD without it, refuse to start.
+	// The strictly-increasing nonce check only exists under boringcrypto/fips140;
+	// in a plain build aeadAESGCMTLS13 wraps a standard GCM that does not enforce
+	// it (and CipherAESGCMFIPS140 is unused there anyway), so only assert it when
+	// one of those modes is active.
+	if (boringEnabled || fips140.Enabled()) && !reusePanics(c) {
+		panic(fmt.Sprintf("noiseutil: FIPS AES-GCM self-test did not reject a reused nonce on %s; nonce-reuse protection is missing (incompatible Go version)", runtime.Version()))
+	}
+}
+
+// reusePanics reports whether re-encrypting with an already-used counter panics,
+// as GCMWithXORCounterNonce is expected to.
+func reusePanics(c noise.Cipher) (panicked bool) {
+	c.Encrypt(nil, 2, nil, nil)
+	defer func() {
+		if recover() != nil {
+			panicked = true
+		}
+	}()
+	c.Encrypt(nil, 2, nil, nil)
+	return false
+}

noiseutil/fips140_test.go

@@ -0,0 +1,48 @@
+package noiseutil
+
+import (
+	"crypto/cipher"
+	"crypto/fips140"
+	"encoding/hex"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+// Ensure NewAESGCM validates the nonce is non-repeating
+func TestNewAESGCM(t *testing.T) {
+	if !boringEnabled && !fips140.Enabled() {
+		t.Skip("TestNewAESGCM is only for fips140/boringcrypto")
+	}
+
+	key, _ := hex.DecodeString("feffe9928665731c6d6a8f9467308308feffe9928665731c6d6a8f9467308308")
+	iv, _ := hex.DecodeString("00000000facedbaddecaf888")
+	plaintext, _ := hex.DecodeString("d9313225f88406e5a55909c5aff5269a86a7a9531534f7da2e4c303d8a318a721c3c0c95956809532fcf0e2449a6b525b16aedf5aa0de657ba637b39")
+	aad, _ := hex.DecodeString("feedfacedeadbeeffeedfacedeadbeefabaddad2")
+	expected, _ := hex.DecodeString("6a65c2edd45bd63c7e29f40e3d2ed8ba2b99f4c83135383d5676652f255059ceb24863ff10afb1089db701245da87fb88d3acd5f9dd0770cac220c3c04145caf25e190aeb775e7080401c628")
+
+	var keyArray [32]byte
+	copy(keyArray[:], key)
+	c := CipherAESGCM.Cipher(keyArray)
+	aead := c.(cipher.AEAD)
+
+	dst := aead.Seal([]byte{}, iv, plaintext, aad)
+	t.Logf("%x", dst)
+	assert.Equal(t, expected, dst)
+
+	// We expect this to fail since we are re-encrypting with a repeat IV
+	switch {
+	case boringEnabled:
+		assert.PanicsWithError(t, "boringcrypto: EVP_AEAD_CTX_seal failed", func() {
+			dst = aead.Seal([]byte{}, iv, plaintext, aad)
+		})
+	case fips140.Version() == "v1.0.0":
+		assert.PanicsWithValue(t, "crypto/cipher: counter decreased", func() {
+			dst = aead.Seal([]byte{}, iv, plaintext, aad)
+		})
+	default:
+		assert.PanicsWithValue(t, "crypto/cipher: counter decreased or remained the same", func() {
+			dst = aead.Seal([]byte{}, iv, plaintext, aad)
+		})
+	}
+}

noiseutil/fips140enforce.go

@@ -0,0 +1,13 @@
+//go:build fips140enforce
+
+package noiseutil
+
+import (
+	"crypto/fips140"
+)
+
+func init() {
+	if !fips140.Enforced() {
+		panic("Nebula compiled with fips140 expects FIPS140 to be enforced. Do not set GODEBUG=fips140, or if you do it must be set as GODEBUG=fips140=only")
+	}
+}

noiseutil/notboring.go

@@ -1,14 +1,25 @@
 //go:build !boringcrypto
-// +build !boringcrypto
 
 package noiseutil
 
 import (
+	"crypto/fips140"
+
 	"github.com/flynn/noise"
 )
 
 // EncryptLockNeeded indicates if calls to Encrypt need a lock
-const EncryptLockNeeded = false
+var EncryptLockNeeded = fips140.Enabled()
 
-// CipherAESGCM is the standard noise.CipherAESGCM when boringcrypto is not enabled
+var CipherAESGCM noise.CipherFunc = initAESGCM()
-var CipherAESGCM noise.CipherFunc = noise.CipherAESGCM
+
+func initAESGCM() noise.CipherFunc {
+	if fips140.Enabled() {
+		return CipherAESGCMFIPS140
+	} else {
+		return noise.CipherAESGCM
+	}
+
+}
+
+var boringEnabled = false

noiseutil/notboring_test.go

@@ -1,14 +0,0 @@
-//go:build !boringcrypto
-// +build !boringcrypto
-
-package noiseutil
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestEncryptLockNeeded(t *testing.T) {
-	assert.False(t, EncryptLockNeeded)
-}

pki.go

@@ -1,6 +1,7 @@
 package nebula
 
 import (
+	"crypto/fips140"
 	"encoding/binary"
 	"encoding/json"
 	"errors"
@@ -237,10 +238,17 @@ func (cs *CertState) getCertificate(v cert.Version) cert.Certificate {
 	return nil
 }
 
-func newCipherSuite(curve cert.Curve, pkcs11backed bool, cipher string) (noise.CipherSuite, error) {
+// newCipherSuite builds the noise.CipherSuite for the given curve and cipher.
+// When fips140Enforced is true (FIPS 140-only mode), non-approved algorithms
+// (Curve25519 and ChaChaPoly) are rejected with an error. Callers pass
+// fips140.Enforced() for fips140Enforced.
+func newCipherSuite(curve cert.Curve, pkcs11backed bool, cipher string, fips140Enforced bool) (noise.CipherSuite, error) {
 	var dhFunc noise.DHFunc
 	switch curve {
 	case cert.Curve_CURVE25519:
+		if fips140Enforced {
+			return nil, errors.New("pki: use of Curve25519 is not allowed in FIPS 140-only mode")
+		}
 		dhFunc = noise.DH25519
 	case cert.Curve_P256:
 		if pkcs11backed {
@@ -253,6 +261,9 @@ func newCipherSuite(curve cert.Curve, pkcs11backed bool, cipher string) (noise.C
 	}
 
 	if cipher == "chachapoly" {
+		if fips140Enforced {
+			return nil, errors.New("pki: use of ChaChaPoly is not allowed in FIPS 140-only mode")
+		}
 		return noise.NewCipherSuite(dhFunc, noise.CipherChaChaPoly, noise.HashSHA256), nil
 	}
 	return noise.NewCipherSuite(dhFunc, noiseutil.CipherAESGCM, noise.HashSHA256), nil
@@ -326,6 +337,10 @@ func newCertStateFromConfig(c *config.C, cipher string) (*CertState, error) {
 			return nil, err
 		}
 
+		if fips140.Enforced() && crt.Curve() != cert.Curve_P256 {
+			return nil, fmt.Errorf("pki: use of %s is not allowed in FIPS 140-only mode", crt.Curve())
+		}
+
 		switch crt.Version() {
 		case cert.Version1:
 			if v1 != nil {
@@ -405,7 +420,7 @@ func newCertState(dv cert.Version, v1, v2 cert.Certificate, pkcs11backed bool, p
 			//NOTE: We do not currently have a method to verify a public private key pair when the private key is in an hsm
 		} else {
 			if err := v1.VerifyPrivateKey(privateKeyCurve, privateKey); err != nil {
-				return nil, fmt.Errorf("private key is not a pair with public key in nebula cert")
+				return nil, fmt.Errorf("private key is not a pair with public key in nebula cert: %w", err)
 			}
 		}
 
@@ -413,7 +428,7 @@ func newCertState(dv cert.Version, v1, v2 cert.Certificate, pkcs11backed bool, p
 		if err != nil {
 			return nil, fmt.Errorf("error marshalling v1 certificate for handshake: %w", err)
 		}
-		ncs, err := newCipherSuite(v1.Curve(), pkcs11backed, cipher)
+		ncs, err := newCipherSuite(v1.Curve(), pkcs11backed, cipher, fips140.Enforced())
 		if err != nil {
 			return nil, err
 		}
@@ -430,7 +445,7 @@ func newCertState(dv cert.Version, v1, v2 cert.Certificate, pkcs11backed bool, p
 			//NOTE: We do not currently have a method to verify a public private key pair when the private key is in an hsm
 		} else {
 			if err := v2.VerifyPrivateKey(privateKeyCurve, privateKey); err != nil {
-				return nil, fmt.Errorf("private key is not a pair with public key in nebula cert")
+				return nil, fmt.Errorf("private key is not a pair with public key in nebula cert: %w", err)
 			}
 		}
 
@@ -438,7 +453,7 @@ func newCertState(dv cert.Version, v1, v2 cert.Certificate, pkcs11backed bool, p
 		if err != nil {
 			return nil, fmt.Errorf("error marshalling v2 certificate for handshake: %w", err)
 		}
-		ncs, err := newCipherSuite(v2.Curve(), pkcs11backed, cipher)
+		ncs, err := newCipherSuite(v2.Curve(), pkcs11backed, cipher, fips140.Enforced())
 		if err != nil {
 			return nil, err
 		}

pki_test.go

@@ -0,0 +1,94 @@
+package nebula
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/slackhq/nebula/cert"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewCipherSuite(t *testing.T) {
+	tests := []struct {
+		name            string
+		curve           cert.Curve
+		cipher          string
+		fips140Enforced bool
+		wantErr         string
+		// wantName is the full expected CipherSuite name (<DH>_<Cipher>_<Hash>),
+		// only checked when wantErr is empty. Asserting the whole name makes both
+		// the curve and cipher selection load-bearing.
+		wantName string
+	}{
+		{
+			name:     "curve25519 aesgcm, not enforced",
+			curve:    cert.Curve_CURVE25519,
+			cipher:   "aesgcm",
+			wantName: "25519_AESGCM_SHA256",
+		},
+		{
+			name:     "curve25519 chachapoly, not enforced",
+			curve:    cert.Curve_CURVE25519,
+			cipher:   "chachapoly",
+			wantName: "25519_ChaChaPoly_SHA256",
+		},
+		{
+			name:     "p256 aesgcm, not enforced",
+			curve:    cert.Curve_P256,
+			cipher:   "aesgcm",
+			wantName: "P256_AESGCM_SHA256",
+		},
+		{
+			name:            "p256 aesgcm, enforced is allowed",
+			curve:           cert.Curve_P256,
+			cipher:          "aesgcm",
+			fips140Enforced: true,
+			wantName:        "P256_AESGCM_SHA256",
+		},
+		{
+			name:            "curve25519 rejected when enforced",
+			curve:           cert.Curve_CURVE25519,
+			cipher:          "aesgcm",
+			fips140Enforced: true,
+			wantErr:         "pki: use of Curve25519 is not allowed in FIPS 140-only mode",
+		},
+		{
+			name:            "chachapoly rejected when enforced",
+			curve:           cert.Curve_P256,
+			cipher:          "chachapoly",
+			fips140Enforced: true,
+			wantErr:         "pki: use of ChaChaPoly is not allowed in FIPS 140-only mode",
+		},
+		{
+			// Curve is checked before cipher, so a Curve25519+ChaChaPoly
+			// request reports the Curve25519 rejection.
+			name:            "curve25519 chachapoly rejected on curve when enforced",
+			curve:           cert.Curve_CURVE25519,
+			cipher:          "chachapoly",
+			fips140Enforced: true,
+			wantErr:         "pki: use of Curve25519 is not allowed in FIPS 140-only mode",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			cs, err := newCipherSuite(tt.curve, false, tt.cipher, tt.fips140Enforced)
+			if tt.wantErr != "" {
+				require.EqualError(t, err, tt.wantErr)
+				assert.Nil(t, cs)
+				return
+			}
+			require.NoError(t, err)
+			require.NotNil(t, cs)
+			assert.Equal(t, tt.wantName, string(cs.Name()))
+		})
+	}
+}
+
+func TestNewCipherSuiteUnsupportedCurve(t *testing.T) {
+	cs, err := newCipherSuite(cert.Curve(99), false, "aesgcm", false)
+	require.Error(t, err)
+	assert.True(t, strings.HasPrefix(err.Error(), "unsupported curve:"), "got: %v", err)
+	assert.Nil(t, cs)
+}

stats.go

@@ -2,6 +2,7 @@ package nebula
 
 import (
 	"context"
+	"crypto/fips140"
 	"errors"
 	"fmt"
 	"log/slog"
@@ -288,9 +289,12 @@ func (s *statsServer) buildRuntime(cfg statsConfig) ([]func(), *http.Server) {
 			Name:      "info",
 			Help:      "Version information for the Nebula binary",
 			ConstLabels: prometheus.Labels{
-				"version":      s.buildVersion,
+				"version":         s.buildVersion,
-				"goversion":    runtime.Version(),
+				"goversion":       runtime.Version(),
-				"boringcrypto": strconv.FormatBool(boringEnabled()),
+				"boringcrypto":    strconv.FormatBool(boringEnabled()),
+				"fips140Version":  fips140.Version(),
+				"fips140Enabled":  strconv.FormatBool(fips140.Enabled()),
+				"fips140Enforced": strconv.FormatBool(fips140.Enforced()),
 			},
 		})
 		pr.MustRegister(g)

udp/netchange.go

@@ -0,0 +1,61 @@
+package udp
+
+import (
+	"context"
+	"log/slog"
+
+	"github.com/slackhq/nebula/config"
+)
+
+// NetworkChangeMonitor rebinds the udp listener when the local network moves out from under it.
+//
+// Detection lives here in the udp package, next to the socket it concerns and the platform matrix that already knows
+// which sockets go stale. What to do about a change — updating the lighthouse, requerying tunnels — is not the udp
+// package's business, so Start takes the reaction as a plain function. Passing it at Start rather than holding it
+// keeps this package from referencing whatever owns the rebind.
+//
+// On platforms whose sockets do not go stale, watchNetworkChanges hands back a nil channel and Start returns.
+type NetworkChangeMonitor struct {
+	l       *slog.Logger
+	ctx     context.Context
+	enabled bool
+}
+
+// NewNetworkChangeMonitor builds a monitor for local network changes. The returned monitor is always usable: Start
+// is safe to call unconditionally, it no-ops when disabled or on a platform that does not need it.
+func NewNetworkChangeMonitor(ctx context.Context, l *slog.Logger, c *config.C) *NetworkChangeMonitor {
+	return &NetworkChangeMonitor{
+		l:       l,
+		ctx:     ctx,
+		enabled: c.GetBool("listen.rebind_on_network_change", true),
+	}
+}
+
+// Start watches for network changes until the context is cancelled, calling rebind once per settled change. It
+// blocks, so callers run it in a goroutine, and it no-ops when disabled, unsupported, or with nothing to rebind.
+func (m *NetworkChangeMonitor) Start(rebind func()) {
+	if !m.enabled || rebind == nil || m.ctx.Err() != nil {
+		return
+	}
+
+	changes, err := watchNetworkChanges(m.ctx, m.l)
+	if err != nil {
+		// Not fatal. Everything else still works, we just won't notice a network change on our own.
+		m.l.Error("Failed to watch for network changes, will not rebind the udp listener when the network moves",
+			"error", err,
+		)
+		return
+	}
+
+	if changes == nil {
+		// This platform's sockets don't go stale, so there is nothing to watch for.
+		return
+	}
+
+	m.l.Info("Watching for network changes to rebind the udp listener")
+
+	for range changes {
+		m.l.Info("Local network changed, rebinding the udp listener")
+		rebind()
+	}
+}

udp/netchange_darwin.go

@@ -0,0 +1,164 @@
+//go:build darwin && !ios && !e2e_testing
+// +build darwin,!ios,!e2e_testing
+
+package udp
+
+import (
+	"context"
+	"encoding/binary"
+	"errors"
+	"log/slog"
+	"os"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+const (
+	// netChangeSettleWindow is how long we keep swallowing routing messages after the first interesting one. A
+	// single network change is never a single message, it is a burst: the link drops, addresses go away, new ones
+	// arrive, routes get rewritten. Reporting part way through that just means reporting again.
+	netChangeSettleWindow = time.Second
+
+	// netChangeReadBuffer is sized well past any rt_msghdr plus its addresses. A short read would be discarded by
+	// the kernel, so being generous here is how we avoid missing a message.
+	netChangeReadBuffer = 4096
+)
+
+// watchNetworkChanges reports when the local network moves out from under us, so the listener can be rebound.
+//
+// Darwin scopes a udp socket to whatever interface it came up on. Move between networks and we keep sending out an
+// interface that no longer has a route, which surfaces as an instant "no route to host" with no packet ever leaving
+// the box. Rebind clears that, but only if something notices the change and calls it. iOS has always been told by
+// the host app off NWPathMonitor. This is the equivalent for everything else that runs on darwin.
+//
+// The returned channel is buffered and coalescing: a send is dropped if one is already pending, since both mean the
+// same thing to a reader. It is closed when ctx is cancelled or the routing socket fails, so a caller can simply
+// range over it. Platforms whose sockets do not need rebinding return a nil channel and no error.
+func watchNetworkChanges(ctx context.Context, l *slog.Logger) (<-chan struct{}, error) {
+	sock, err := openRouteSocket()
+	if err != nil {
+		return nil, err
+	}
+
+	changes := make(chan struct{}, 1)
+
+	go func() {
+		defer close(changes)
+		defer func() { _ = sock.Close() }()
+
+		// Closing the socket is what unblocks the read in watchRouteSocket, so this turns cancellation into a
+		// close. It is scoped to this call so it cannot outlive the watch it belongs to.
+		done := make(chan struct{})
+		defer close(done)
+		go func() {
+			select {
+			case <-ctx.Done():
+				_ = sock.Close()
+			case <-done:
+			}
+		}()
+
+		watchRouteSocket(l, sock, changes)
+	}()
+
+	return changes, nil
+}
+
+// watchRouteSocket blocks reading the routing socket, reporting once per settled burst of changes. It returns when
+// the socket is closed, which is how cancellation gets us out of here.
+func watchRouteSocket(l *slog.Logger, sock *os.File, changes chan<- struct{}) {
+	buf := make([]byte, netChangeReadBuffer)
+
+	for {
+		n, err := sock.Read(buf)
+		if err != nil {
+			logRouteSocketError(l, err)
+			return
+		}
+
+		if !isNetworkChange(buf[:n]) {
+			continue
+		}
+
+		// Swallow the rest of the burst. The deadline is absolute and not extended by what arrives, so this always
+		// ends after the settle window no matter how chatty the socket is. Changes that land after the window
+		// simply produce another report, which is the correct outcome anyway.
+		deadline := time.Now().Add(netChangeSettleWindow)
+		for {
+			if err = sock.SetReadDeadline(deadline); err != nil {
+				logRouteSocketError(l, err)
+				return
+			}
+
+			if _, err = sock.Read(buf); err != nil {
+				if os.IsTimeout(err) {
+					break
+				}
+				logRouteSocketError(l, err)
+				return
+			}
+		}
+
+		if err = sock.SetReadDeadline(time.Time{}); err != nil {
+			logRouteSocketError(l, err)
+			return
+		}
+
+		select {
+		case changes <- struct{}{}:
+		default:
+			// One already pending, and a second "the network moved" tells the reader nothing new.
+		}
+	}
+}
+
+// logRouteSocketError reports a routing socket failure unless it is just us shutting the socket down.
+func logRouteSocketError(l *slog.Logger, err error) {
+	if errors.Is(err, os.ErrClosed) {
+		return
+	}
+
+	l.Error("Error reading the routing socket, will no longer notice local network changes", "error", err)
+}
+
+// openRouteSocket returns the routing socket as a non blocking os.File. Going through os.File puts reads on the go
+// poller, which buys us both a working read deadline and a Close that unblocks a read in progress.
+func openRouteSocket() (*os.File, error) {
+	fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = unix.SetNonblock(fd, true); err != nil {
+		_ = unix.Close(fd)
+		return nil, err
+	}
+
+	return os.NewFile(uintptr(fd), "route"), nil
+}
+
+// isNetworkChange reports whether a routing message means our local addressing may have moved out from under us.
+//
+// We read the header instead of parsing the message because the type is the only part we need, and a full parse can
+// fail on shapes we don't care about, which would turn "a message I can't parse" into "a change I missed".
+// rt_msghdr, if_msghdr and ifa_msghdr all begin with the same three fields, so this is the same for every type.
+func isNetworkChange(msg []byte) bool {
+	if len(msg) < 4 {
+		return false
+	}
+
+	// u_short msglen, u_char version, u_char type
+	if int(binary.NativeEndian.Uint16(msg[0:2])) > len(msg) || msg[2] != unix.RTM_VERSION {
+		return false
+	}
+
+	switch msg[3] {
+	case unix.RTM_NEWADDR, unix.RTM_DELADDR, unix.RTM_IFINFO:
+		// An address arrived or left, or a link changed state. Anything else on this socket is either a route
+		// churning underneath us, which a rebind doesn't help with, or unrelated traffic.
+		return true
+	default:
+		return false
+	}
+}

udp/netchange_darwin_test.go

@@ -0,0 +1,244 @@
+//go:build darwin && !ios && !e2e_testing
+// +build darwin,!ios,!e2e_testing
+
+package udp
+
+import (
+	"context"
+	"encoding/binary"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
+	"golang.org/x/sys/unix"
+)
+
+// routeMsg builds the first four bytes of a routing message, which is all isNetworkChange reads.
+func routeMsg(msgType uint8, extra int) []byte {
+	msg := make([]byte, 4+extra)
+	binary.NativeEndian.PutUint16(msg[0:2], uint16(len(msg)))
+	msg[2] = unix.RTM_VERSION
+	msg[3] = msgType
+	return msg
+}
+
+func TestIsNetworkChange(t *testing.T) {
+	// The three that mean our addressing may have moved
+	assert.True(t, isNetworkChange(routeMsg(unix.RTM_NEWADDR, 0)))
+	assert.True(t, isNetworkChange(routeMsg(unix.RTM_DELADDR, 0)))
+	assert.True(t, isNetworkChange(routeMsg(unix.RTM_IFINFO, 0)))
+
+	// Route churn is not something a rebind helps with
+	assert.False(t, isNetworkChange(routeMsg(unix.RTM_ADD, 0)))
+	assert.False(t, isNetworkChange(routeMsg(unix.RTM_DELETE, 0)))
+	assert.False(t, isNetworkChange(routeMsg(unix.RTM_GET, 0)))
+
+	// Garbage must not be mistaken for a change
+	assert.False(t, isNetworkChange(nil), "empty")
+	assert.False(t, isNetworkChange([]byte{0, 0, 0}), "short header")
+
+	wrongVersion := routeMsg(unix.RTM_NEWADDR, 0)
+	wrongVersion[2] = unix.RTM_VERSION + 1
+	assert.False(t, isNetworkChange(wrongVersion), "wrong rtm_version")
+
+	lying := routeMsg(unix.RTM_NEWADDR, 0)
+	binary.NativeEndian.PutUint16(lying[0:2], 512)
+	assert.False(t, isNetworkChange(lying), "msglen longer than what we read")
+}
+
+// socketPair returns a connected pair of datagram sockets, the first wrapped the same way the routing socket is. It
+// stands in for the kernel so the watch loop can be driven with synthetic messages.
+func socketPair(t *testing.T) (*os.File, int) {
+	t.Helper()
+
+	fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_DGRAM, 0)
+	require.NoError(t, err)
+	require.NoError(t, unix.SetNonblock(fds[0], true))
+
+	f := os.NewFile(uintptr(fds[0]), "route")
+	t.Cleanup(func() {
+		_ = f.Close()
+		_ = unix.Close(fds[1])
+	})
+
+	return f, fds[1]
+}
+
+func TestWatchRouteSocketCoalescesABurst(t *testing.T) {
+	sock, kernel := socketPair(t)
+	changes := make(chan struct{}, 1)
+
+	done := make(chan struct{})
+	go func() {
+		watchRouteSocket(test.NewLogger(), sock, changes)
+		close(done)
+	}()
+
+	// One network change is a burst of messages. All of these land inside the settle window, so they must produce
+	// exactly one report rather than one apiece.
+	for range 5 {
+		_, err := unix.Write(kernel, routeMsg(unix.RTM_NEWADDR, 8))
+		require.NoError(t, err)
+	}
+	// Uninteresting messages in the middle of a burst must not add a report of their own either.
+	_, err := unix.Write(kernel, routeMsg(unix.RTM_ADD, 8))
+	require.NoError(t, err)
+
+	select {
+	case <-changes:
+	case <-time.After(netChangeSettleWindow * 4):
+		t.Fatal("a burst should have reported a change")
+	}
+
+	// Nothing more from that burst
+	select {
+	case <-changes:
+		t.Fatal("a burst should report exactly once")
+	case <-time.After(netChangeSettleWindow):
+	}
+
+	// A change after the window has closed is a separate event and gets its own report.
+	_, err = unix.Write(kernel, routeMsg(unix.RTM_IFINFO, 8))
+	require.NoError(t, err)
+	select {
+	case <-changes:
+	case <-time.After(netChangeSettleWindow * 4):
+		t.Fatal("a later change should report again")
+	}
+
+	// Closing the socket is how the real thing shuts down
+	require.NoError(t, sock.Close())
+	select {
+	case <-done:
+	case <-time.After(time.Second * 5):
+		t.Fatal("watchRouteSocket did not return after the socket was closed")
+	}
+}
+
+func TestWatchRouteSocketIgnoresUninterestingMessages(t *testing.T) {
+	sock, kernel := socketPair(t)
+	changes := make(chan struct{}, 1)
+
+	done := make(chan struct{})
+	go func() {
+		watchRouteSocket(test.NewLogger(), sock, changes)
+		close(done)
+	}()
+
+	for _, msgType := range []uint8{unix.RTM_ADD, unix.RTM_DELETE, unix.RTM_GET, unix.RTM_MISS} {
+		_, err := unix.Write(kernel, routeMsg(msgType, 8))
+		require.NoError(t, err)
+	}
+
+	select {
+	case <-changes:
+		t.Fatal("route churn alone must not report a change")
+	case <-time.After(netChangeSettleWindow * 2):
+	}
+
+	require.NoError(t, sock.Close())
+	select {
+	case <-done:
+	case <-time.After(time.Second * 5):
+		t.Fatal("watchRouteSocket did not return after the socket was closed")
+	}
+}
+
+// TestWatchRouteSocketDropsRatherThanBlocks covers the coalescing send. A reader that is busy rebinding must not
+// wedge the watcher, and a second pending "the network moved" tells it nothing new anyway.
+func TestWatchRouteSocketDropsRatherThanBlocks(t *testing.T) {
+	sock, kernel := socketPair(t)
+	changes := make(chan struct{}, 1)
+
+	done := make(chan struct{})
+	go func() {
+		watchRouteSocket(test.NewLogger(), sock, changes)
+		close(done)
+	}()
+
+	// Nobody is reading changes, so after the first report the buffer is full for the rest of this test
+	for range 3 {
+		_, err := unix.Write(kernel, routeMsg(unix.RTM_NEWADDR, 8))
+		require.NoError(t, err)
+		time.Sleep(netChangeSettleWindow + time.Millisecond*250)
+	}
+
+	// The watcher must still be alive and responsive to a close
+	require.NoError(t, sock.Close())
+	select {
+	case <-done:
+	case <-time.After(time.Second * 5):
+		t.Fatal("watchRouteSocket wedged on a full channel")
+	}
+
+	assert.Len(t, changes, 1, "the pending report should have coalesced, not queued")
+}
+
+// TestWatchNetworkChangesStopsWithContext covers the detection path against a real routing socket, including that
+// cancelling the context closes the channel so a ranging caller falls out of its loop.
+func TestWatchNetworkChangesStopsWithContext(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+
+	changes, err := watchNetworkChanges(ctx, test.NewLogger())
+	require.NoError(t, err)
+	require.NotNil(t, changes, "darwin should support watching")
+
+	drained := make(chan struct{})
+	go func() {
+		for range changes {
+		}
+		close(drained)
+	}()
+
+	cancel()
+	select {
+	case <-drained:
+	case <-time.After(time.Second * 5):
+		t.Fatal("cancelling the context should close the changes channel")
+	}
+}
+
+// TestNetworkChangeMonitorStopsWithContext drives the whole monitor against a real routing socket: Start must block
+// watching, and cancelling the context (which is all Control does on shutdown, it never stops the monitor directly)
+// must return it and clean up the watch goroutines.
+func TestNetworkChangeMonitorStopsWithContext(t *testing.T) {
+	// IgnoreCurrent because other tests in this package leave readers running; we only care about what this test
+	// leaks itself.
+	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	l := test.NewLogger()
+	c := config.NewC(l)
+	require.NoError(t, c.LoadString("listen:\n  rebind_on_network_change: true\n"))
+	m := NewNetworkChangeMonitor(ctx, l, c)
+
+	done := make(chan struct{})
+	go func() {
+		m.Start(func() {})
+		close(done)
+	}()
+
+	// Start should be sitting on the routing socket, not have fallen out. If it returned early it either failed to
+	// watch or no-op'd, both of which we want to catch.
+	select {
+	case <-done:
+		t.Fatal("Start returned instead of watching")
+	case <-time.After(time.Millisecond * 250):
+	}
+
+	cancel()
+	select {
+	case <-done:
+	case <-time.After(time.Second * 5):
+		t.Fatal("Start did not return after the context was cancelled")
+	}
+
+	// Starting again after the context is dead must not open anything.
+	m.Start(func() {})
+}

udp/netchange_generic.go

@@ -0,0 +1,22 @@
+//go:build !darwin || ios || e2e_testing
+// +build !darwin ios e2e_testing
+
+package udp
+
+import (
+	"context"
+	"log/slog"
+)
+
+// watchNetworkChanges is a no-op outside of darwin.
+//
+// Darwin is the platform that scopes a udp socket to the interface it came up on, so it is the platform whose socket
+// goes stale when the local network changes. Everywhere else Rebind has nothing to do, so there is nothing to watch
+// for. iOS is excluded on purpose even though it is darwin: the host app already drives the rebind off NWPathMonitor,
+// and two things racing to rebind the same socket is worse than one.
+//
+// A nil channel means "not supported here", which callers must treat as "do not start a watcher" rather than
+// selecting on it, since a receive from a nil channel blocks forever.
+func watchNetworkChanges(_ context.Context, _ *slog.Logger) (<-chan struct{}, error) {
+	return nil, nil
+}

udp/netchange_test.go

@@ -0,0 +1,39 @@
+package udp
+
+import (
+	"context"
+	"testing"
+
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func newMonitor(t *testing.T, ctx context.Context, cfg string) *NetworkChangeMonitor {
+	t.Helper()
+	l := test.NewLogger()
+	c := config.NewC(l)
+	require.NoError(t, c.LoadString(cfg))
+	return NewNetworkChangeMonitor(ctx, l, c)
+}
+
+func TestNetworkChangeMonitorDefaultsOn(t *testing.T) {
+	// Says nothing about rebinding, so this covers the default.
+	m := newMonitor(t, context.Background(), "listen:\n  host: 0.0.0.0\n")
+	assert.True(t, m.enabled, "should default to on")
+}
+
+func TestNetworkChangeMonitorDisabledIsANoOp(t *testing.T) {
+	m := newMonitor(t, context.Background(), "listen:\n  rebind_on_network_change: false\n")
+	require.False(t, m.enabled)
+
+	// Must return without opening a socket. If it watched anything this would block.
+	m.Start(func() {})
+}
+
+func TestNetworkChangeMonitorNilRebindIsANoOp(t *testing.T) {
+	// Nothing to rebind, so there is no point watching, on any platform.
+	m := newMonitor(t, context.Background(), "listen:\n  rebind_on_network_change: true\n")
+	m.Start(nil)
+}

udp/udp_darwin.go

@@ -187,6 +187,9 @@ func (u *StdConn) SupportsMultipleReaders() bool {
 	return false
 }
 
+// Rebind clears the interface the kernel scoped this socket to, so that sends are routed against the current
+// routing table instead of the interface we happened to be on when the socket was created. Darwin pins sockets
+// this way on its own, which is what strands us after the underlying network changes.
 func (u *StdConn) Rebind() error {
 	var err error
 	if u.isV4 {
@@ -195,9 +198,5 @@ func (u *StdConn) Rebind() error {
 		err = syscall.SetsockoptInt(int(u.sysFd), syscall.IPPROTO_IPV6, syscall.IPV6_BOUND_IF, 0)
 	}
 
-	if err != nil {
+	return err
-		u.l.Error("Failed to rebind udp socket", "error", err)
-	}
-
-	return nil
 }

udp/udp_tester.go

@@ -10,6 +10,7 @@ import (
 	"net/netip"
 	"os"
 	"sync"
+	"sync/atomic"
 
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
@@ -64,7 +65,9 @@ func acquirePacket() *Packet {
 }
 
 type TesterConn struct {
-	Addr netip.AddrPort
+	// addr is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
+	// move it mid-run to simulate roaming, so it is atomic rather than a plain field.
+	addr atomic.Pointer[netip.AddrPort]
 
 	RxPackets chan *Packet // Packets to receive into nebula
 	TxPackets chan *Packet // Packets transmitted outside by nebula
@@ -82,13 +85,24 @@ type TesterConn struct {
 }
 
 func NewListener(l *slog.Logger, ip netip.Addr, port int, _ bool, _ int) (Conn, error) {
-	return &TesterConn{
+	c := &TesterConn{
-		Addr:      netip.AddrPortFrom(ip, uint16(port)),
 		RxPackets: make(chan *Packet, 10),
 		TxPackets: make(chan *Packet, 10),
 		done:      make(chan struct{}),
 		l:         l,
-	}, nil
+	}
+	c.SetAddr(netip.AddrPortFrom(ip, uint16(port)))
+	return c, nil
+}
+
+// GetAddr returns the underlay address this conn currently sends from.
+func (u *TesterConn) GetAddr() netip.AddrPort {
+	return *u.addr.Load()
+}
+
+// SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network.
+func (u *TesterConn) SetAddr(addr netip.AddrPort) {
+	u.addr.Store(&addr)
 }
 
 // Send will place a UdpPacket onto the receive queue for nebula to consume
@@ -147,7 +161,7 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
 		p.Data = p.Data[:len(b)]
 	}
 	copy(p.Data, b)
-	p.From = u.Addr
+	p.From = u.GetAddr()
 	p.To = addr
 	select {
 	case <-u.done:
@@ -178,7 +192,7 @@ func NewUDPStatsEmitter(_ []Conn) func() {
 }
 
 func (u *TesterConn) LocalAddr() (netip.AddrPort, error) {
-	return u.Addr, nil
+	return u.GetAddr(), nil
 }
 
 func (u *TesterConn) SupportsMultipleReaders() bool {

wake_clock_darwin.go

@@ -1,29 +0,0 @@
-//go:build darwin
-
-package nebula
-
-import (
-	"time"
-
-	"golang.org/x/sys/unix"
-)
-
-// suspendClockDelta returns CLOCK_MONOTONIC - CLOCK_UPTIME_RAW. On macOS CLOCK_MONOTONIC keeps counting across
-// system sleep while CLOCK_UPTIME_RAW (mach_absolute_time) pauses, so the difference grows by time spent asleep.
-//
-// The pausing clock is read first so scheduling jitter between the two reads biases the delta positive; the
-// wakeDetector clamps out the noise.
-//
-// Caveat: on Apple Silicon the hardware timebase keeps ticking through sleep, which can make both clocks advance
-// and the spread stay flat, leaving this detector blind. That fails safe (no clears, behavior as before); IOKit
-// power notifications are the follow-up for full coverage on those machines.
-func suspendClockDelta() (time.Duration, bool) {
-	var uptime, mono unix.Timespec
-	if err := unix.ClockGettime(unix.CLOCK_UPTIME_RAW, &uptime); err != nil {
-		return 0, false
-	}
-	if err := unix.ClockGettime(unix.CLOCK_MONOTONIC, &mono); err != nil {
-		return 0, false
-	}
-	return time.Duration(mono.Nano() - uptime.Nano()), true
-}

wake_clock_generic.go

@@ -1,11 +0,0 @@
-//go:build !linux && !darwin && !windows
-
-package nebula
-
-import "time"
-
-// suspendClockDelta reports that this platform has no usable clock pair for detecting system sleep; the wake
-// detector stays dormant and dead tunnels are left to the normal traffic checks.
-func suspendClockDelta() (time.Duration, bool) {
-	return 0, false
-}

wake_clock_linux.go

@@ -1,26 +0,0 @@
-//go:build linux
-
-package nebula
-
-import (
-	"time"
-
-	"golang.org/x/sys/unix"
-)
-
-// suspendClockDelta returns CLOCK_BOOTTIME - CLOCK_MONOTONIC. CLOCK_MONOTONIC pauses while the system is suspended
-// and CLOCK_BOOTTIME does not, so the difference only ever grows, and only by time spent suspended. Both reads are
-// vDSO calls, cheap enough for a hot ticker.
-//
-// The pausing clock is read first so scheduling jitter between the two reads biases the delta positive; the
-// wakeDetector clamps out the noise.
-func suspendClockDelta() (time.Duration, bool) {
-	var mono, boot unix.Timespec
-	if err := unix.ClockGettime(unix.CLOCK_MONOTONIC, &mono); err != nil {
-		return 0, false
-	}
-	if err := unix.ClockGettime(unix.CLOCK_BOOTTIME, &boot); err != nil {
-		return 0, false
-	}
-	return time.Duration(boot.Nano() - mono.Nano()), true
-}

wake_clock_windows.go

@@ -1,41 +0,0 @@
-//go:build windows
-
-package nebula
-
-import (
-	"sync"
-	"time"
-	"unsafe"
-
-	"golang.org/x/sys/windows"
-)
-
-var (
-	procQueryInterruptTime         = windows.NewLazySystemDLL("kernelbase.dll").NewProc("QueryInterruptTime")
-	procQueryUnbiasedInterruptTime = windows.NewLazySystemDLL("kernel32.dll").NewProc("QueryUnbiasedInterruptTime")
-
-	// QueryInterruptTime needs Windows 10; probe once and stay dormant on anything older.
-	wakeClockAvailable = sync.OnceValue(func() bool {
-		return procQueryInterruptTime.Find() == nil && procQueryUnbiasedInterruptTime.Find() == nil
-	})
-)
-
-// suspendClockDelta returns interrupt time minus unbiased interrupt time, both in 100ns units. The unbiased count
-// excludes time the system spends suspended while the biased one includes it, so the difference grows by exactly
-// the time spent asleep.
-//
-// The pausing (unbiased) clock is read first so scheduling jitter between the two reads biases the delta positive;
-// the wakeDetector clamps out the noise.
-func suspendClockDelta() (time.Duration, bool) {
-	if !wakeClockAvailable() {
-		return 0, false
-	}
-
-	var unbiased, biased uint64
-	if r1, _, _ := procQueryUnbiasedInterruptTime.Call(uintptr(unsafe.Pointer(&unbiased))); r1 == 0 {
-		return 0, false
-	}
-	// Returns void, cannot fail once resolved.
-	_, _, _ = procQueryInterruptTime.Call(uintptr(unsafe.Pointer(&biased)))
-	return time.Duration(int64(biased-unbiased)) * 100, true
-}

wake_detector.go

@@ -1,47 +0,0 @@
-package nebula
-
-import "time"
-
-// wakeDetector notices when the machine has returned from system sleep and measures how long it was suspended.
-//
-// It samples the spread between two kernel clocks: one that pauses across a suspend and one that keeps counting
-// (suspendClockDelta, per platform). While the machine is awake the spread is constant no matter how starved,
-// stopped, or stepped this process is — SIGSTOP, debugger pauses, scheduler starvation, and NTP adjustments move
-// both clocks together or neither, so none of them can fake a wake. A true suspend is the only thing that grows
-// the spread, and it grows by exactly the time spent suspended.
-//
-// Sample is intended to piggyback on a ticker the caller already runs; it costs two clock reads. It is not safe
-// for concurrent use.
-type wakeDetector struct {
-	// read returns the current spread between the two clocks, false if this platform can't provide one.
-	read   func() (time.Duration, bool)
-	last   time.Duration
-	primed bool
-}
-
-func newWakeDetector() *wakeDetector {
-	return &wakeDetector{read: suspendClockDelta}
-}
-
-// Sample returns how long the machine was suspended since the previous call, 0 if it wasn't, and false if the
-// platform has no way to tell. The first call primes the baseline and always reports 0.
-func (w *wakeDetector) Sample() (time.Duration, bool) {
-	delta, ok := w.read()
-	if !ok {
-		return 0, false
-	}
-
-	if !w.primed {
-		w.primed = true
-		w.last = delta
-		return 0, true
-	}
-
-	slept := delta - w.last
-	w.last = delta
-	if slept < 0 {
-		// The clock pair is read non-atomically so tiny negative jitter is possible; it is never a wake.
-		slept = 0
-	}
-	return slept, true
-}

wake_detector_test.go

@@ -1,67 +0,0 @@
-package nebula
-
-import (
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestWakeDetector(t *testing.T) {
-	delta := time.Duration(0)
-	ok := true
-	w := &wakeDetector{read: func() (time.Duration, bool) { return delta, ok }}
-
-	// The first sample primes the baseline and never reports a wake, even with a pre-existing spread
-	delta = 3 * time.Hour
-	slept, sok := w.Sample()
-	assert.True(t, sok)
-	assert.Equal(t, time.Duration(0), slept)
-
-	// A stable spread means the machine never slept
-	slept, sok = w.Sample()
-	assert.True(t, sok)
-	assert.Equal(t, time.Duration(0), slept)
-
-	// The spread grows by exactly the time spent suspended
-	delta += 42 * time.Second
-	slept, sok = w.Sample()
-	assert.True(t, sok)
-	assert.Equal(t, 42*time.Second, slept)
-
-	// A wake is reported once, then the baseline moves with it
-	slept, sok = w.Sample()
-	assert.True(t, sok)
-	assert.Equal(t, time.Duration(0), slept)
-
-	// Negative jitter from the non-atomic clock pair reads clamps to zero
-	delta -= time.Microsecond
-	slept, sok = w.Sample()
-	assert.True(t, sok)
-	assert.Equal(t, time.Duration(0), slept)
-
-	// Consecutive suspends both report; the clamped jitter moved the baseline so it is not double-counted
-	delta += time.Minute
-	slept, _ = w.Sample()
-	assert.Equal(t, time.Minute, slept)
-	delta += time.Hour
-	slept, _ = w.Sample()
-	assert.Equal(t, time.Hour, slept)
-
-	// An unsupported platform read reports not-ok
-	ok = false
-	_, sok = w.Sample()
-	assert.False(t, sok)
-}
-
-// TestWakeDetectorPlatformClock smoke tests the real clock pair: two samples close together must not report a
-// wake on a machine that isn't suspending mid-test.
-func TestWakeDetectorPlatformClock(t *testing.T) {
-	w := newWakeDetector()
-	if _, ok := w.Sample(); !ok {
-		t.Skip("platform has no suspend clock pair")
-	}
-	slept, ok := w.Sample()
-	assert.True(t, ok)
-	assert.Less(t, slept, time.Second)
-}