Experimenting with fw event reporting

Bumps the zx2c4-dependencies group with 1 update: golang.zx2c4.com/wireguard/windows.


Updates `golang.zx2c4.com/wireguard/windows` from 0.5.3 to 0.6.1

---
updated-dependencies:
- dependency-name: golang.zx2c4.com/wireguard/windows
  dependency-version: 0.6.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: zx2c4-dependencies
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/release.yml

@@ -209,10 +209,11 @@ jobs:
         id: create_release
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_REF_NAME: ${{ github.ref_name }}
         run: |
           cd artifacts
           gh release create \
             --verify-tag \
-            --title "Release ${{ github.ref_name }}" \
+            --title "Release ${GITHUB_REF_NAME}" \
-            "${{ github.ref_name }}" \
+            "${GITHUB_REF_NAME}" \
             SHASUM256.txt *-latest/*.zip *-latest/*.tar.gz

.github/workflows/smoke-extra.yml

@@ -18,6 +18,8 @@ jobs:
     if: github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'smoke-test-extra')
     name: Run extra smoke tests
     runs-on: ubuntu-latest
+    env:
+      VAGRANT_DEFAULT_PROVIDER: libvirt
     steps:
 
     - uses: actions/checkout@v6
@@ -30,11 +32,13 @@ jobs:
     - name: add hashicorp source
       run: wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg && echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
 
-    - name: workaround AMD-V issue  # https://github.com/cri-o/packaging/pull/306
+    - name: install vagrant and libvirt
-      run: sudo rmmod kvm_amd
+      run: |
-
+        sudo apt-get update && sudo apt-get install -y vagrant libvirt-daemon-system libvirt-dev
-    - name: install vagrant
+        sudo chmod 666 /dev/kvm
-      run: sudo apt-get update && sudo apt-get install -y vagrant virtualbox
+        sudo usermod -aG libvirt $(whoami)
+        sudo chmod 666 /var/run/libvirt/libvirt-sock
+        vagrant plugin install vagrant-libvirt
 
     - name: freebsd-amd64
       run: make smoke-vagrant/freebsd-amd64
@@ -45,10 +49,19 @@ jobs:
     - name: netbsd-amd64
       run: make smoke-vagrant/netbsd-amd64
 
-    - name: linux-386
-      run: make smoke-vagrant/linux-386
-
     - name: linux-amd64-ipv6disable
       run: make smoke-vagrant/linux-amd64-ipv6disable
 
+    # linux-386 runs last because it requires disabling KVM to use VirtualBox,
+    # which prevents libvirt (used by the other tests) from working after this point.
+    - name: install virtualbox for i386 test
+      run: |
+        sudo apt-get install -y virtualbox
+        sudo rmmod kvm_amd kvm_intel kvm 2>/dev/null || true
+
+    - name: linux-386
+      env:
+        VAGRANT_DEFAULT_PROVIDER: virtualbox
+      run: make smoke-vagrant/linux-386
+
     timeout-minutes: 30

.github/workflows/smoke/build-relay.sh

@@ -16,8 +16,10 @@ relay:
   am_relay: true
 EOF
 
-    export LIGHTHOUSES="192.168.100.1 172.17.0.2:4242"
+    # TEST-NET-3 placeholder IPs; smoke-relay.sh seds them to real container IPs.
-    export REMOTE_ALLOW_LIST='{"172.17.0.4/32": false, "172.17.0.5/32": false}'
+    # Mapping: .2 lighthouse1, .3 host2, .4 host3, .5 host4.
+    export LIGHTHOUSES="192.168.100.1 203.0.113.2:4242"
+    export REMOTE_ALLOW_LIST='{"203.0.113.4/32": false, "203.0.113.5/32": false}'
 
     HOST="host2" ../genconfig.sh >host2.yml <<EOF
 relay:
@@ -25,7 +27,7 @@ relay:
     - 192.168.100.1
 EOF
 
-    export REMOTE_ALLOW_LIST='{"172.17.0.3/32": false}'
+    export REMOTE_ALLOW_LIST='{"203.0.113.3/32": false}'
 
     HOST="host3" ../genconfig.sh >host3.yml
 

.github/workflows/smoke/build.sh

@@ -5,9 +5,15 @@ set -e -x
 rm -rf ./build
 mkdir ./build
 
-# TODO: Assumes your docker bridge network is a /24, and the first container that launches will be .1
+# Smoke containers run on a dedicated docker network whose subnet is allocated
-# - We could make this better by launching the lighthouse first and then fetching what IP it is.
+# at smoke time, not known at build time. Configs are written with TEST-NET-3
-NET="$(docker network inspect bridge -f '{{ range .IPAM.Config }}{{ .Subnet }}{{ end }}' | cut -d. -f1-3)"
+# placeholder IPs (RFC 5737) and smoke.sh / smoke-vagrant.sh / smoke-relay.sh
+# sed the real container IPs in before starting nebula.
+#
+# Placeholder mapping (last octet == fixed container slot):
+#   203.0.113.2 -> lighthouse1, 203.0.113.3 -> host2,
+#   203.0.113.4 -> host3,       203.0.113.5 -> host4.
+LIGHTHOUSE_IP="203.0.113.2"
 
 (
     cd build
@@ -25,16 +31,16 @@ NET="$(docker network inspect bridge -f '{{ range .IPAM.Config }}{{ .Subnet }}{{
         ../genconfig.sh >lighthouse1.yml
 
     HOST="host2" \
-        LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
+        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
         ../genconfig.sh >host2.yml
 
     HOST="host3" \
-        LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
+        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
         INBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
         ../genconfig.sh >host3.yml
 
     HOST="host4" \
-        LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
+        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
         OUTBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
         ../genconfig.sh >host4.yml
 

.github/workflows/smoke/smoke-relay.sh

@@ -6,6 +6,8 @@ set -o pipefail
 
 mkdir -p logs
 
+NETWORK="nebula-smoke-relay"
+
 cleanup() {
     echo
     echo " *** cleanup"
@@ -16,22 +18,53 @@ cleanup() {
     then
         docker kill lighthouse1 host2 host3 host4
     fi
+    docker network rm "$NETWORK" >/dev/null 2>&1
 }
 
 trap cleanup EXIT
 
-docker run --name lighthouse1 --rm nebula:smoke-relay -config lighthouse1.yml -test
+# Create a dedicated smoke network with an explicit subnet (required for --ip
-docker run --name host2 --rm nebula:smoke-relay -config host2.yml -test
+# below). Probe a short list of candidates so a locally-used range doesn't
-docker run --name host3 --rm nebula:smoke-relay -config host3.yml -test
+# fail the whole test — we only need one to be free.
-docker run --name host4 --rm nebula:smoke-relay -config host4.yml -test
+docker network rm "$NETWORK" >/dev/null 2>&1 || true
+for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
+    if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
+        break
+    fi
+done
+if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
+    echo "failed to create $NETWORK: every candidate subnet is in use" >&2
+    exit 1
+fi
 
-docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
+# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
+# .3 host2, .4 host3, .5 host4 — matches the placeholders in build-relay.sh.
+SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
+PREFIX="${SUBNET%/*}"
+PREFIX="${PREFIX%.*}"
+LIGHTHOUSE_IP="$PREFIX.2"
+HOST2_IP="$PREFIX.3"
+HOST3_IP="$PREFIX.4"
+HOST4_IP="$PREFIX.5"
+
+# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
+for f in build/host2.yml build/host3.yml build/host4.yml; do
+    sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
+    mv "$f.tmp" "$f"
+done
+
+docker run --name lighthouse1 --rm nebula:smoke-relay -config lighthouse1.yml -test
+docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" nebula:smoke-relay -config host2.yml -test
+docker run --name host3 --rm -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" nebula:smoke-relay -config host3.yml -test
+docker run --name host4 --rm -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" nebula:smoke-relay -config host4.yml -test
+
+docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
 sleep 1
-docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
+docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
 sleep 1
-docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
+docker run --name host3 --network "$NETWORK" --ip "$HOST3_IP" -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
 sleep 1
-docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
+docker run --name host4 --network "$NETWORK" --ip "$HOST4_IP" -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
 sleep 1
 
 set +x
@@ -76,7 +109,13 @@ docker exec host4 sh -c 'kill 1'
 docker exec host3 sh -c 'kill 1'
 docker exec host2 sh -c 'kill 1'
 docker exec lighthouse1 sh -c 'kill 1'
-sleep 5
+
+# Wait up to 30s for all backgrounded jobs to exit rather than relying on a
+# fixed sleep.
+for _ in $(seq 1 30); do
+    [ -z "$(jobs -r)" ] && break
+    sleep 1
+done
 
 if [ "$(jobs -r)" ]
 then

.github/workflows/smoke/smoke-vagrant.sh

@@ -8,6 +8,8 @@ export VAGRANT_CWD="$PWD/vagrant-$1"
 
 mkdir -p logs
 
+NETWORK="nebula-smoke"
+
 cleanup() {
     echo
     echo " *** cleanup"
@@ -19,21 +21,51 @@ cleanup() {
         docker kill lighthouse1 host2
     fi
     vagrant destroy -f
+    docker network rm "$NETWORK" >/dev/null 2>&1
 }
 
 trap cleanup EXIT
 
+# Create a dedicated smoke network with an explicit subnet (required for --ip
+# below). Probe a short list of candidates so a locally-used range doesn't
+# fail the whole test — we only need one to be free.
+docker network rm "$NETWORK" >/dev/null 2>&1 || true
+for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
+    if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
+        break
+    fi
+done
+if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
+    echo "failed to create $NETWORK: every candidate subnet is in use" >&2
+    exit 1
+fi
+
+# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
+# .3 host2 — matches the placeholders in build.sh.
+SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
+PREFIX="${SUBNET%/*}"
+PREFIX="${PREFIX%.*}"
+LIGHTHOUSE_IP="$PREFIX.2"
+HOST2_IP="$PREFIX.3"
+
+# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
+# This must happen before `vagrant up` rsyncs build/ into the VM for host3.
+for f in build/host2.yml build/host3.yml; do
+    sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
+    mv "$f.tmp" "$f"
+done
+
 CONTAINER="nebula:${NAME:-smoke}"
 
 docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test
-docker run --name host2 --rm "$CONTAINER" -config host2.yml -test
+docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" "$CONTAINER" -config host2.yml -test
 
 vagrant up
 vagrant ssh -c "cd /nebula && /nebula/$1-nebula -config host3.yml -test" -- -T
 
-docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
+docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
 sleep 1
-docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
+docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
 sleep 1
 vagrant ssh -c "cd /nebula && sudo sh -c 'echo \$\$ >/nebula/pid && exec /nebula/$1-nebula -config host3.yml'" 2>&1 -- -T | tee logs/host3 | sed -u 's/^/  [host3]  /' &
 sleep 15
@@ -96,7 +128,14 @@ vagrant ssh -c "ping -c1 192.168.100.2" -- -T
 vagrant ssh -c "sudo xargs kill </nebula/pid" -- -T
 docker exec host2 sh -c 'kill 1'
 docker exec lighthouse1 sh -c 'kill 1'
-sleep 1
+
+# Wait up to 30s for all backgrounded jobs to exit. vagrant ssh in particular
+# takes a beat to tear down after nebula exits on the VM, so a fixed sleep is
+# racy.
+for _ in $(seq 1 30); do
+    [ -z "$(jobs -r)" ] && break
+    sleep 1
+done
 
 if [ "$(jobs -r)" ]
 then

.github/workflows/smoke/smoke.sh

@@ -6,6 +6,8 @@ set -o pipefail
 
 mkdir -p logs
 
+NETWORK="nebula-smoke"
+
 cleanup() {
     echo
     echo " *** cleanup"
@@ -16,24 +18,56 @@ cleanup() {
     then
         docker kill lighthouse1 host2 host3 host4
     fi
+    docker network rm "$NETWORK" >/dev/null 2>&1
 }
 
 trap cleanup EXIT
 
+# Create a dedicated smoke network with an explicit subnet (required for --ip
+# below). Probe a short list of candidates so a locally-used range doesn't
+# fail the whole test — we only need one to be free.
+docker network rm "$NETWORK" >/dev/null 2>&1 || true
+for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
+    if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
+        break
+    fi
+done
+if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
+    echo "failed to create $NETWORK: every candidate subnet is in use" >&2
+    exit 1
+fi
+
+# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
+# .3 host2, .4 host3, .5 host4 — matches the placeholders in build.sh.
+SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
+PREFIX="${SUBNET%/*}"
+PREFIX="${PREFIX%.*}"
+LIGHTHOUSE_IP="$PREFIX.2"
+HOST2_IP="$PREFIX.3"
+HOST3_IP="$PREFIX.4"
+HOST4_IP="$PREFIX.5"
+
+# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
+# build/lighthouse1.yml has no IPs to rewrite so it's skipped.
+for f in build/host2.yml build/host3.yml build/host4.yml; do
+    sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
+    mv "$f.tmp" "$f"
+done
+
 CONTAINER="nebula:${NAME:-smoke}"
 
 docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test
-docker run --name host2 --rm "$CONTAINER" -config host2.yml -test
+docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" "$CONTAINER" -config host2.yml -test
-docker run --name host3 --rm "$CONTAINER" -config host3.yml -test
+docker run --name host3 --rm -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" "$CONTAINER" -config host3.yml -test
-docker run --name host4 --rm "$CONTAINER" -config host4.yml -test
+docker run --name host4 --rm -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" "$CONTAINER" -config host4.yml -test
 
-docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
+docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
 sleep 1
-docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
+docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
 sleep 1
-docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
+docker run --name host3 --network "$NETWORK" --ip "$HOST3_IP" -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
 sleep 1
-docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
+docker run --name host4 --network "$NETWORK" --ip "$HOST4_IP" -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
 sleep 1
 
 # grab tcpdump pcaps for debugging
@@ -124,13 +158,20 @@ set -x
 # host2 speaking to host4 on UDP 4000 should allow it to reply, when firewall rules would normally not permit this
 docker exec host2 sh -c "/usr/bin/echo host2 | ncat -nuv 192.168.100.4 4000"
 docker exec host2 ncat -e '/usr/bin/echo helloagainfromhost2' -nkluv 0.0.0.0 4000 &
+sleep 1
 docker exec host4 sh -c "/usr/bin/echo host4 | ncat -nuv 192.168.100.2 4000"
 
 docker exec host4 sh -c 'kill 1'
 docker exec host3 sh -c 'kill 1'
 docker exec host2 sh -c 'kill 1'
 docker exec lighthouse1 sh -c 'kill 1'
-sleep 5
+
+# Wait up to 30s for all backgrounded jobs to exit rather than relying on a
+# fixed sleep.
+for _ in $(seq 1 30); do
+    [ -z "$(jobs -r)" ] && break
+    sleep 1
+done
 
 if [ "$(jobs -r)" ]
 then

.github/workflows/smoke/vagrant-linux-amd64-ipv6disable/Vagrantfile

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 Vagrant.configure("2") do |config|
-  config.vm.box = "ubuntu/jammy64"
+  config.vm.box = "bento/ubuntu-24.04"
 
   config.vm.synced_folder "../build", "/nebula"
 

.github/workflows/smoke/vagrant-openbsd-amd64/Vagrantfile

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 Vagrant.configure("2") do |config|
-  config.vm.box = "generic/openbsd7"
+  config.vm.box = "DefinedNet/openbsd78"
 
   config.vm.synced_folder "../build", "/nebula", type: "rsync"
 end

CODEOWNERS

@@ -0,0 +1 @@
+#ECCN:Open Source

README.md

@@ -57,7 +57,7 @@ Check the [releases](https://github.com/slackhq/nebula/releases/latest) page for
     docker pull nebulaoss/nebula
     ```
 
-#### Mobile
+#### Mobile ([source code](https://github.com/DefinedNet/mobile_nebula))
 
 - [iOS](https://apps.apple.com/us/app/mobile-nebula/id1509587936?itsct=apps_box&itscg=30200)
 - [Android](https://play.google.com/store/apps/details?id=net.defined.mobile_nebula&pcampaignid=pcampaignidMKT-Other-global-all-co-prtnr-py-PartBadge-Mar2515-1)
@@ -76,6 +76,8 @@ Nebula was created to provide a mechanism for groups of hosts to communicate sec
 
 ## Getting started (quickly)
 
+**Don't want to manage your own PKI and lighthouses?** [Managed Nebula](https://www.defined.net/) from Defined Networking handles all of this for you.
+
 To set up a Nebula network, you'll need:
 
 #### 1. The [Nebula binaries](https://github.com/slackhq/nebula/releases) or [Distribution Packages](https://github.com/slackhq/nebula#distribution-packages) for your specific platform. Specifically you'll need `nebula-cert` and the specific nebula binary for each platform you use.

cert/ca_pool.go

@@ -1,11 +1,14 @@
 package cert
 
 import (
+	"bufio"
+	"bytes"
+	"encoding/pem"
 	"errors"
 	"fmt"
+	"io"
 	"net/netip"
 	"slices"
-	"strings"
 	"time"
 )
 
@@ -29,22 +32,46 @@ func NewCAPool() *CAPool {
 // If the pool contains any expired certificates, an ErrExpired will be
 // returned along with the pool. The caller must handle any such errors.
 func NewCAPoolFromPEM(caPEMs []byte) (*CAPool, error) {
+	return NewCAPoolFromPEMReader(bytes.NewReader(caPEMs))
+}
+
+// NewCAPoolFromPEMReader will create a new CA pool from the provided reader.
+// The reader must contain a PEM-encoded set of nebula certificates.
+func NewCAPoolFromPEMReader(r io.Reader) (*CAPool, error) {
 	pool := NewCAPool()
-	var err error
+
 	var expired bool
-	for {
+
-		caPEMs, err = pool.AddCAFromPEM(caPEMs)
+	scanner := bufio.NewScanner(r)
-		if errors.Is(err, ErrExpired) {
+	scanner.Split(SplitPEM)
-			expired = true
+
-			err = nil
+	for scanner.Scan() {
+		pemBytes := scanner.Bytes()
+
+		block, rest := pem.Decode(pemBytes)
+		if len(bytes.TrimSpace(rest)) > 0 {
+			return nil, ErrInvalidPEMBlock
 		}
+		if block == nil {
+			return nil, ErrInvalidPEMBlock
+		}
+
+		c, err := unmarshalCertificateBlock(block)
 		if err != nil {
 			return nil, err
 		}
-		if len(caPEMs) == 0 || strings.TrimSpace(string(caPEMs)) == "" {
+
-			break
+		err = pool.AddCA(c)
+		if errors.Is(err, ErrExpired) {
+			expired = true
+			continue
+		} else if err != nil {
+			return nil, err
 		}
 	}
+	if err := scanner.Err(); err != nil {
+		return nil, ErrInvalidPEMBlock
+	}
 
 	if expired {
 		return pool, ErrExpired

cert/ca_pool_test.go

@@ -1,7 +1,10 @@
 package cert
 
 import (
+	"bytes"
+	"io"
 	"net/netip"
+	"strings"
 	"testing"
 	"time"
 
@@ -112,6 +115,60 @@ k+coOv04r+zh33ISyhbsafnYduN17p2eD7CmHvHuerguXD9f32gcxo/KsFCKEjMe
 	assert.Len(t, ppppp.CAs, 1)
 }
 
+// oneByteReader wraps a reader to return at most 1 byte per Read call,
+// exercising the streaming accumulation logic in NewCAPoolFromPEMReader.
+type oneByteReader struct {
+	r io.Reader
+}
+
+func (o *oneByteReader) Read(p []byte) (int, error) {
+	if len(p) == 0 {
+		return 0, nil
+	}
+	return o.r.Read(p[:1])
+}
+
+func TestNewCAPoolFromPEMReader_EmptyReader(t *testing.T) {
+	pool, err := NewCAPoolFromPEMReader(bytes.NewReader(nil))
+	require.NoError(t, err)
+	assert.Empty(t, pool.CAs)
+
+	pool, err = NewCAPoolFromPEMReader(strings.NewReader("   \n\t\n  "))
+	require.NoError(t, err)
+	assert.Empty(t, pool.CAs)
+}
+
+func TestNewCAPoolFromPEMReader_OneByteReads(t *testing.T) {
+	ca1, _, _, pem1 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil)
+	ca2, _, _, pem2 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil)
+
+	bundle := append(pem1, pem2...)
+	pool, err := NewCAPoolFromPEMReader(&oneByteReader{r: bytes.NewReader(bundle)})
+	require.NoError(t, err)
+	assert.Len(t, pool.CAs, 2)
+
+	fp1, err := ca1.Fingerprint()
+	require.NoError(t, err)
+	fp2, err := ca2.Fingerprint()
+	require.NoError(t, err)
+
+	assert.Contains(t, pool.CAs, fp1)
+	assert.Contains(t, pool.CAs, fp2)
+}
+
+func TestNewCAPoolFromPEMReader_TruncatedPEM(t *testing.T) {
+	_, err := NewCAPoolFromPEMReader(strings.NewReader("-----BEGIN NEBULA CERTIFICATE-----\npartialdata"))
+	assert.ErrorIs(t, err, ErrInvalidPEMBlock)
+}
+
+func TestNewCAPoolFromPEMReader_TrailingGarbage(t *testing.T) {
+	_, _, _, pem1 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil)
+
+	bundle := append(pem1, []byte("some trailing garbage")...)
+	_, err := NewCAPoolFromPEMReader(bytes.NewReader(bundle))
+	assert.ErrorIs(t, err, ErrInvalidPEMBlock)
+}
+
 func TestCertificateV1_Verify(t *testing.T) {
 	ca, _, caKey, _ := NewTestCaCert(Version1, Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, nil)
 	c, _, _, _ := NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test cert", time.Now(), time.Now().Add(5*time.Minute), nil, nil, nil)

cert/cert_v2.go

@@ -439,7 +439,11 @@ func (c *certificateV2) validate() error {
 				if !hasV6Networks {
 					return NewErrInvalidCertificateProperties("IPv6 unsafe networks require an IPv6 address assignment: %s", network)
 				}
-			} // as long as we have any IP address, IPv4 UnsafeNetworks are allowed
+			} else if network.Addr().Is4() {
+				if !hasV4Networks {
+					return NewErrInvalidCertificateProperties("IPv4 unsafe networks require an IPv4 address assignment: %s", network)
+				}
+			}
 		}
 	}
 

cert/p256/p256.go

@@ -44,7 +44,12 @@ func swap(r, s []byte) ([]byte, []byte, error) {
 	}
 	sNormalized := nMod.Nat().Sub(bigS, nMod)
 
-	return r, sNormalized.Bytes(nMod), nil
+	result := sNormalized.Bytes(nMod)
+	for len(result) > 1 && result[0] == 0 {
+		result = result[1:]
+	}
+
+	return r, result, nil
 }
 
 func Normalize(sig []byte) ([]byte, error) {

cert/pem.go

@@ -1,12 +1,66 @@
 package cert
 
 import (
+	"bytes"
 	"encoding/pem"
+	"errors"
 	"fmt"
 
 	"golang.org/x/crypto/ed25519"
 )
 
+var ErrTruncatedPEMBlock = errors.New("truncated PEM block")
+
+// SplitPEM is a split function for bufio.Scanner that returns each PEM block.
+func SplitPEM(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	// Look for the start of a PEM block
+	start := bytes.Index(data, []byte("-----BEGIN "))
+	if start == -1 {
+		if atEOF && len(bytes.TrimSpace(data)) > 0 {
+			// Non-whitespace content with no PEM block
+			return 0, nil, ErrTruncatedPEMBlock
+		}
+		if atEOF {
+			return len(data), nil, nil
+		}
+		// Request more data
+		return 0, nil, nil
+	}
+
+	// Look for the end marker
+	endMarkerStart := bytes.Index(data[start:], []byte("-----END "))
+	if endMarkerStart == -1 {
+		if atEOF {
+			// Incomplete PEM block at EOF
+			return 0, nil, ErrTruncatedPEMBlock
+		}
+		// Need more data to find the end
+		return 0, nil, nil
+	}
+
+	// Find the actual end of the END line (after the newline)
+	endMarkerStart += start
+	endLineEnd := bytes.IndexByte(data[endMarkerStart:], '\n')
+	var end int
+	if endLineEnd == -1 {
+		if atEOF {
+			// END marker without newline at EOF - take it anyway
+			end = len(data)
+		} else {
+			// Need more data
+			return 0, nil, nil
+		}
+	} else {
+		end = endMarkerStart + endLineEnd + 1
+	}
+
+	// Extract the PEM block
+	pemBlock := data[start:end]
+
+	// Return the valid PEM block
+	return end, pemBlock, nil
+}
+
 const ( //cert banners
 	CertificateBanner   = "NEBULA CERTIFICATE"
 	CertificateV2Banner = "NEBULA CERTIFICATE V2"
@@ -37,19 +91,7 @@ func UnmarshalCertificateFromPEM(b []byte) (Certificate, []byte, error) {
 		return nil, r, ErrInvalidPEMBlock
 	}
 
-	var c Certificate
+	c, err := unmarshalCertificateBlock(p)
-	var err error
-
-	switch p.Type {
-	// Implementations must validate the resulting certificate contains valid information
-	case CertificateBanner:
-		c, err = unmarshalCertificateV1(p.Bytes, nil)
-	case CertificateV2Banner:
-		c, err = unmarshalCertificateV2(p.Bytes, nil, Curve_CURVE25519)
-	default:
-		return nil, r, ErrInvalidPEMCertificateBanner
-	}
-
 	if err != nil {
 		return nil, r, err
 	}
@@ -58,6 +100,20 @@ func UnmarshalCertificateFromPEM(b []byte) (Certificate, []byte, error) {
 
 }
 
+// unmarshalCertificateBlock decodes a single PEM block into a certificate.
+// It expects a Nebula certificate banner and returns ErrInvalidPEMCertificateBanner otherwise.
+func unmarshalCertificateBlock(block *pem.Block) (Certificate, error) {
+	switch block.Type {
+	// Implementations must validate the resulting certificate contains valid information
+	case CertificateBanner:
+		return unmarshalCertificateV1(block.Bytes, nil)
+	case CertificateV2Banner:
+		return unmarshalCertificateV2(block.Bytes, nil, Curve_CURVE25519)
+	default:
+		return nil, ErrInvalidPEMCertificateBanner
+	}
+}
+
 func marshalCertPublicKeyToPEM(c Certificate) []byte {
 	if c.IsCA() {
 		return MarshalSigningPublicKeyToPEM(c.Curve(), c.PublicKey())

cert/pem_test.go

@@ -1,12 +1,88 @@
 package cert
 
 import (
+	"bufio"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+func scanAll(t *testing.T, input string) ([]string, error) {
+	t.Helper()
+	scanner := bufio.NewScanner(strings.NewReader(input))
+	scanner.Split(SplitPEM)
+	var blocks []string
+	for scanner.Scan() {
+		blocks = append(blocks, scanner.Text())
+	}
+	return blocks, scanner.Err()
+}
+
+func TestSplitPEM_Single(t *testing.T) {
+	input := "-----BEGIN TEST-----\ndata\n-----END TEST-----\n"
+	blocks, err := scanAll(t, input)
+	require.NoError(t, err)
+	require.Len(t, blocks, 1)
+	require.Equal(t, input, blocks[0])
+}
+
+func TestSplitPEM_Multiple(t *testing.T) {
+	block1 := "-----BEGIN TEST-----\naaa\n-----END TEST-----\n"
+	block2 := "-----BEGIN TEST-----\nbbb\n-----END TEST-----\n"
+	blocks, err := scanAll(t, block1+block2)
+	require.NoError(t, err)
+	require.Len(t, blocks, 2)
+	require.Equal(t, block1, blocks[0])
+	require.Equal(t, block2, blocks[1])
+}
+
+func TestSplitPEM_CommentsAndWhitespaceBetweenBlocks(t *testing.T) {
+	input := "# comment\n\n-----BEGIN TEST-----\naaa\n-----END TEST-----\n\n# another comment\n\n-----BEGIN TEST-----\nbbb\n-----END TEST-----\n"
+	blocks, err := scanAll(t, input)
+	require.NoError(t, err)
+	require.Len(t, blocks, 2)
+}
+
+func TestSplitPEM_Empty(t *testing.T) {
+	blocks, err := scanAll(t, "")
+	require.NoError(t, err)
+	require.Empty(t, blocks)
+}
+
+func TestSplitPEM_WhitespaceOnly(t *testing.T) {
+	blocks, err := scanAll(t, "  \n\t\n  ")
+	require.NoError(t, err)
+	require.Empty(t, blocks)
+}
+
+func TestSplitPEM_TrailingGarbage(t *testing.T) {
+	input := "-----BEGIN TEST-----\ndata\n-----END TEST-----\ngarbage"
+	blocks, err := scanAll(t, input)
+	require.ErrorIs(t, err, ErrTruncatedPEMBlock)
+	require.Len(t, blocks, 1)
+}
+
+func TestSplitPEM_TruncatedBlock(t *testing.T) {
+	input := "-----BEGIN TEST-----\npartial data with no end"
+	_, err := scanAll(t, input)
+	require.ErrorIs(t, err, ErrTruncatedPEMBlock)
+}
+
+func TestSplitPEM_NoEndNewline(t *testing.T) {
+	input := "-----BEGIN TEST-----\ndata\n-----END TEST-----"
+	blocks, err := scanAll(t, input)
+	require.NoError(t, err)
+	require.Len(t, blocks, 1)
+	require.Equal(t, input, blocks[0])
+}
+
+func TestSplitPEM_GarbageOnly(t *testing.T) {
+	_, err := scanAll(t, "this is not PEM data")
+	require.ErrorIs(t, err, ErrTruncatedPEMBlock)
+}
+
 func TestUnmarshalCertificateFromPEM(t *testing.T) {
 	goodCert := []byte(`
 # A good cert

cmd/nebula-cert/verify.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/slackhq/nebula/cert"
@@ -40,21 +39,15 @@ func verify(args []string, out io.Writer, errOut io.Writer) error {
 		return err
 	}
 
-	rawCACert, err := os.ReadFile(*vf.caPath)
+	caFile, err := os.Open(*vf.caPath)
 	if err != nil {
 		return fmt.Errorf("error while reading ca: %w", err)
 	}
+	defer caFile.Close()
 
-	caPool := cert.NewCAPool()
+	caPool, err := cert.NewCAPoolFromPEMReader(caFile)
-	for {
+	if err != nil && !errors.Is(err, cert.ErrExpired) {
-		rawCACert, err = caPool.AddCAFromPEM(rawCACert)
+		return fmt.Errorf("error while adding ca cert to pool: %w", err)
-		if err != nil {
-			return fmt.Errorf("error while adding ca cert to pool: %w", err)
-		}
-
-		if rawCACert == nil || len(rawCACert) == 0 || strings.TrimSpace(string(rawCACert)) == "" {
-			break
-		}
 	}
 
 	rawCert, err := os.ReadFile(*vf.certPath)

cmd/nebula-cert/verify_test.go

@@ -64,7 +64,7 @@ func Test_verify(t *testing.T) {
 	err = verify([]string{"-ca", caFile.Name(), "-crt", "does_not_exist"}, ob, eb)
 	assert.Empty(t, ob.String())
 	assert.Empty(t, eb.String())
-	require.EqualError(t, err, "error while adding ca cert to pool: input did not contain a valid PEM encoded block")
+	require.ErrorIs(t, err, cert.ErrInvalidPEMBlock)
 
 	// make a ca for later
 	caPub, caPriv, _ := ed25519.GenerateKey(rand.Reader)

cmd/nebula-service/main.go

@@ -50,9 +50,15 @@ func main() {
 		os.Exit(0)
 	}
 
+	l := logrus.New()
+	l.Out = os.Stdout
+
 	if *serviceFlag != "" {
-		doService(configPath, configTest, Build, serviceFlag)
+		if err := doService(configPath, configTest, Build, serviceFlag); err != nil {
-		os.Exit(1)
+			l.WithError(err).Error("Service command failed")
+			os.Exit(1)
+		}
+		return
 	}
 
 	if *configPath == "" {
@@ -61,9 +67,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	l := logrus.New()
-	l.Out = os.Stdout
-
 	c := config.NewC(l)
 	err := c.Load(*configPath)
 	if err != nil {
@@ -78,8 +81,20 @@ func main() {
 	}
 
 	if !*configTest {
-		ctrl.Start()
+		wait, err := ctrl.Start()
-		ctrl.ShutdownBlock()
+		if err != nil {
+			util.LogWithContextIfNeeded("Error while running", err, l)
+			os.Exit(1)
+		}
+
+		go ctrl.ShutdownBlock()
+
+		if err := wait(); err != nil {
+			l.WithError(err).Error("Nebula stopped due to fatal error")
+			os.Exit(2)
+		}
+
+		l.Info("Goodbye")
 	}
 
 	os.Exit(0)

cmd/nebula-service/service.go

@@ -57,11 +57,11 @@ func fileExists(filename string) bool {
 	return true
 }
 
-func doService(configPath *string, configTest *bool, build string, serviceFlag *string) {
+func doService(configPath *string, configTest *bool, build string, serviceFlag *string) error {
 	if *configPath == "" {
 		ex, err := os.Executable()
 		if err != nil {
-			panic(err)
+			return err
 		}
 		*configPath = filepath.Dir(ex) + "/config.yaml"
 		if !fileExists(*configPath) {
@@ -88,13 +88,13 @@ func doService(configPath *string, configTest *bool, build string, serviceFlag *
 	// - above, in `Run` we create a `logrus.Logger` which is what nebula expects to use
 	s, err := service.New(prg, svcConfig)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 
 	errs := make(chan error, 5)
 	logger, err = s.Logger(errs)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 
 	go func() {
@@ -109,18 +109,16 @@ func doService(configPath *string, configTest *bool, build string, serviceFlag *
 
 	switch *serviceFlag {
 	case "run":
-		err = s.Run()
+		if err := s.Run(); err != nil {
-		if err != nil {
 			// Route any errors to the system logger
 			logger.Error(err)
 		}
 	default:
-		err := service.Control(s, *serviceFlag)
+		if err := service.Control(s, *serviceFlag); err != nil {
-		if err != nil {
 			log.Printf("Valid actions: %q\n", service.ControlAction)
-			log.Fatal(err)
+			return err
 		}
-		return
 	}
 
+	return nil
 }

cmd/nebula/main.go

@@ -72,9 +72,21 @@ func main() {
 	}
 
 	if !*configTest {
-		ctrl.Start()
+		wait, err := ctrl.Start()
+		if err != nil {
+			util.LogWithContextIfNeeded("Error while running", err, l)
+			os.Exit(1)
+		}
+
+		go ctrl.ShutdownBlock()
 		notifyReady(l)
-		ctrl.ShutdownBlock()
+
+		if err := wait(); err != nil {
+			l.WithError(err).Error("Nebula stopped due to fatal error")
+			os.Exit(2)
+		}
+
+		l.Info("Goodbye")
 	}
 
 	os.Exit(0)

control.go

@@ -2,17 +2,34 @@ package nebula
 
 import (
 	"context"
+	"errors"
 	"net/netip"
 	"os"
 	"os/signal"
+	"sync"
 	"syscall"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/firewall/events"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/overlay"
 )
 
+type RunState int
+
+const (
+	StateUnknown RunState = iota
+	StateReady
+	StateStarted
+	StateStopping
+	StateStopped
+)
+
+var ErrAlreadyStarted = errors.New("nebula is already started")
+var ErrAlreadyStopped = errors.New("nebula cannot be restarted")
+var ErrUnknownState = errors.New("nebula state is invalid")
+
 // Every interaction here needs to take extra care to copy memory and not return or use arguments "as is" when touching
 // core. This means copying IP objects, slices, de-referencing pointers and taking the actual value, etc
 
@@ -26,6 +43,9 @@ type controlHostLister interface {
 }
 
 type Control struct {
+	stateLock sync.Mutex
+	state     RunState
+
 	f                      *Interface
 	l                      *logrus.Logger
 	ctx                    context.Context
@@ -49,10 +69,31 @@ type ControlHostInfo struct {
 	CurrentRelaysThroughMe []netip.Addr     `json:"currentRelaysThroughMe"`
 }
 
-// Start actually runs nebula, this is a nonblocking call. To block use Control.ShutdownBlock()
+// Start actually runs nebula, this is a nonblocking call.
-func (c *Control) Start() {
+// The returned function blocks until nebula has fully stopped and returns the
+// first fatal reader error (if any). A nil error means nebula shut down
+// gracefully; a non-nil error means a reader hit an unexpected failure that
+// triggered the shutdown.
+func (c *Control) Start() (func() error, error) {
+	c.stateLock.Lock()
+	defer c.stateLock.Unlock()
+	switch c.state {
+	case StateReady:
+		//yay!
+	case StateStopped, StateStopping:
+		return nil, ErrAlreadyStopped
+	case StateStarted:
+		return nil, ErrAlreadyStarted
+	default:
+		return nil, ErrUnknownState
+	}
+
 	// Activate the interface
-	c.f.activate()
+	err := c.f.activate()
+	if err != nil {
+		c.state = StateStopped
+		return nil, err
+	}
 
 	// Call all the delayed funcs that waited patiently for the interface to be created.
 	if c.sshStart != nil {
@@ -71,16 +112,40 @@ func (c *Control) Start() {
 		c.lighthouseStart()
 	}
 
+	c.f.triggerShutdown = c.Stop
+
 	// Start reading packets.
-	c.f.run()
+	out, err := c.f.run()
+	if err != nil {
+		c.state = StateStopped
+		return nil, err
+	}
+	c.state = StateStarted
+	return out, nil
+}
+
+func (c *Control) State() RunState {
+	c.stateLock.Lock()
+	defer c.stateLock.Unlock()
+	return c.state
 }
 
 func (c *Control) Context() context.Context {
 	return c.ctx
 }
 
-// Stop signals nebula to shutdown and close all tunnels, returns after the shutdown is complete
+// Stop is a non-blocking call that signals nebula to close all tunnels and shut down
 func (c *Control) Stop() {
+	c.stateLock.Lock()
+	if c.state != StateStarted {
+		c.stateLock.Unlock()
+		// We are stopping or stopped already
+		return
+	}
+
+	c.state = StateStopping
+	c.stateLock.Unlock()
+
 	// Stop the handshakeManager (and other services), to prevent new tunnels from
 	// being created while we're shutting them all down.
 	c.cancel()
@@ -89,7 +154,9 @@ func (c *Control) Stop() {
 	if err := c.f.Close(); err != nil {
 		c.l.WithError(err).Error("Close interface failed")
 	}
-	c.l.Info("Goodbye")
+	c.stateLock.Lock()
+	c.state = StateStopped
+	c.stateLock.Unlock()
 }
 
 // ShutdownBlock will listen for and block on term and interrupt signals, calling Control.Stop() once signalled
@@ -274,6 +341,36 @@ func (c *Control) Device() overlay.Device {
 	return c.f.inside
 }
 
+// SetFirewallEventReporter installs an event reporter on the current firewall.
+// Passing nil clears any installed reporter. The reporter is carried across
+// firewall rule reloads. Report* methods are invoked while nebula holds
+// internal locks and must be non-blocking; in particular they must not call
+// back into *Control methods that touch the firewall, or deadlock will
+// result.
+//
+// Installation is performed by shallow-copying the current *Firewall,
+// setting the reporter field on the copy, and swapping the pointer under
+// the conntrack lock. Every Firewall the data path sees therefore has an
+// immutable reporter slot, and emit sites can read it without any
+// synchronization of their own.
+func (c *Control) SetFirewallEventReporter(r events.Reporter) {
+	old := c.f.firewall
+	if old == nil {
+		return
+	}
+	old.Conntrack.Lock()
+	defer old.Conntrack.Unlock()
+
+	// Re-read under the lock in case a concurrent reload swapped in a new
+	// Firewall between the unlocked load above and here. Both Firewalls share
+	// the same Conntrack pointer in the normal (non-overflow) reload path,
+	// so the lock we hold is the right one for whichever we see now.
+	current := c.f.firewall
+	fw := *current
+	fw.reporter = r
+	c.f.firewall = &fw
+}
+
 func copyHostInfo(h *HostInfo, preferredRanges []netip.Prefix) ControlHostInfo {
 	chi := ControlHostInfo{
 		VpnAddrs:               make([]netip.Addr, len(h.vpnAddrs)),

control_test.go

@@ -79,6 +79,7 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) {
 	}, &Interface{})
 
 	c := Control{
+		state: StateReady,
 		f: &Interface{
 			hostMap: hm,
 		},

dist/wireshark/nebula.lua

@@ -84,30 +84,24 @@ end
 
 function nebula.prefs_changed()
     if default_settings.all_ports == nebula.prefs.all_ports and default_settings.port == nebula.prefs.port then
-        -- Nothing changed, bail
         return
     end
 
-    -- Remove our old dissector
+    -- Remove all existing registrations
     DissectorTable.get("udp.port"):remove_all(nebula)
 
-    if nebula.prefs.all_ports and default_settings.all_ports ~= nebula.prefs.all_ports then
+    if nebula.prefs.all_ports then
-        default_settings.all_port = nebula.prefs.all_ports
+        -- Register on every port for hole punch capture
-
         for i=0, 65535 do
             DissectorTable.get("udp.port"):add(i, nebula)
         end
-
+    else
-        -- no need to establish again on specific ports
+        -- Register on the configured port only
-        return
+        DissectorTable.get("udp.port"):add(nebula.prefs.port, nebula)
     end
 
-
+    default_settings.all_ports = nebula.prefs.all_ports
-    if default_settings.all_ports ~= nebula.prefs.all_ports then
+    default_settings.port = nebula.prefs.port
-        -- Add our new port dissector
-        default_settings.port = nebula.prefs.port
-        DissectorTable.get("udp.port"):add(default_settings.port, nebula)
-    end
 end
 
 DissectorTable.get("udp.port"):add(default_settings.port, nebula)

dns_server.go

@@ -1,12 +1,14 @@
 package nebula
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/netip"
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 
 	"github.com/gaissmai/bart"
 	"github.com/miekg/dns"
@@ -14,32 +16,207 @@ import (
 	"github.com/slackhq/nebula/config"
 )
 
-// This whole thing should be rewritten to use context
+type dnsServer struct {
-
-var dnsR *dnsRecords
-var dnsServer *dns.Server
-var dnsAddr string
-
-type dnsRecords struct {
 	sync.RWMutex
 	l               *logrus.Logger
+	ctx             context.Context
 	dnsMap4         map[string]netip.Addr
 	dnsMap6         map[string]netip.Addr
 	hostMap         *HostMap
 	myVpnAddrsTable *bart.Lite
+
+	mux *dns.ServeMux
+
+	// enabled mirrors `lighthouse.serve_dns && lighthouse.am_lighthouse`.
+	// Start, Add, and reload consult it so callers don't need to know the
+	// gating rules. When it toggles off via reload, accumulated records are
+	// cleared so a later re-enable starts with a fresh map populated from
+	// new handshakes.
+	enabled atomic.Bool
+
+	serverMu sync.Mutex
+	server   *dns.Server
+	// started is closed once `server` has finished binding (or after
+	// ListenAndServe returns on a bind failure). Stop waits on it before
+	// calling Shutdown to avoid the miekg/dns "server not started" race
+	// where a Shutdown that arrives before bind completes is silently
+	// ignored, leaving the listener running forever.
+	started chan struct{}
+	addr    string
 }
 
-func newDnsRecords(l *logrus.Logger, cs *CertState, hostMap *HostMap) *dnsRecords {
+// newDnsServerFromConfig builds a dnsServer, applies the initial config, and
-	return &dnsRecords{
+// registers a reload callback. The reload callback is registered before the
+// initial config is applied, so a SIGHUP can later enable, fix, or disable
+// DNS even if the initial application failed.
+//
+// The dnsServer internally gates on `lighthouse.serve_dns &&
+// lighthouse.am_lighthouse`. Start and Add are safe to call unconditionally,
+// they no-op when DNS isn't enabled. Each Start invocation owns a ctx-cancel
+// watcher that tears the listener down on nebula shutdown. The returned
+// pointer is always non-nil, even on error.
+func newDnsServerFromConfig(ctx context.Context, l *logrus.Logger, cs *CertState, hostMap *HostMap, c *config.C) (*dnsServer, error) {
+	ds := &dnsServer{
 		l:               l,
+		ctx:             ctx,
 		dnsMap4:         make(map[string]netip.Addr),
 		dnsMap6:         make(map[string]netip.Addr),
 		hostMap:         hostMap,
 		myVpnAddrsTable: cs.myVpnAddrsTable,
 	}
+	ds.mux = dns.NewServeMux()
+	ds.mux.HandleFunc(".", ds.handleDnsRequest)
+
+	c.RegisterReloadCallback(func(c *config.C) {
+		if err := ds.reload(c, false); err != nil {
+			l.WithError(err).Error("Failed to reload DNS responder from config")
+		}
+	})
+
+	if err := ds.reload(c, true); err != nil {
+		return ds, err
+	}
+	return ds, nil
 }
 
-func (d *dnsRecords) Query(q uint16, data string) netip.Addr {
+// reload applies the latest config and reconciles the running state with it:
+//   - enabled toggled on  -> spawn a runner
+//   - enabled toggled off -> stop the runner
+//   - listen address changed (while running) -> restart on the new address
+//   - everything else     -> no-op
+//
+// On the initial call it only records configuration; Control.Start is what
+// launches the first runner via dnsStart.
+func (d *dnsServer) reload(c *config.C, initial bool) error {
+	wantsDns := c.GetBool("lighthouse.serve_dns", false)
+	amLighthouse := c.GetBool("lighthouse.am_lighthouse", false)
+	enabled := wantsDns && amLighthouse
+	newAddr := getDnsServerAddr(c)
+
+	d.serverMu.Lock()
+	running := d.server
+	runningStarted := d.started
+	sameAddr := d.addr == newAddr
+	d.addr = newAddr
+	d.enabled.Store(enabled)
+	d.serverMu.Unlock()
+
+	if initial {
+		if wantsDns && !amLighthouse {
+			d.l.Warn("DNS server refusing to run because this host is not a lighthouse.")
+		}
+		return nil
+	}
+
+	if !enabled {
+		if running != nil {
+			d.Stop()
+		}
+		// Drop any records that accumulated while enabled; a later re-enable
+		// will repopulate from fresh handshakes.
+		d.clearRecords()
+		return nil
+	}
+
+	if running == nil {
+		// Was disabled (or never started); bring it up now.
+		go d.Start()
+		return nil
+	}
+
+	if sameAddr {
+		return nil
+	}
+
+	d.shutdownServer(running, runningStarted, "reload")
+	// Old Start goroutine has now exited; bring up a fresh listener on the
+	// new address.
+	go d.Start()
+	return nil
+}
+
+// shutdownServer waits for the server to finish binding (so Shutdown actually
+// stops it rather than no-oping) and then shuts it down.
+func (d *dnsServer) shutdownServer(srv *dns.Server, started chan struct{}, reason string) {
+	if srv == nil {
+		return
+	}
+	if started != nil {
+		<-started
+	}
+	if err := srv.Shutdown(); err != nil {
+		d.l.WithError(err).WithField("reason", reason).Warn("Failed to shut down the DNS responder")
+	}
+}
+
+// Start binds and serves the DNS responder. Blocks until Stop is called or
+// the listener errors. Safe to call when DNS is disabled (returns
+// immediately). This is what Control.dnsStart points at.
+//
+// Must be invoked after the tun device is active so that lighthouse.dns.host
+// may bind to a nebula IP.
+func (d *dnsServer) Start() {
+	if !d.enabled.Load() {
+		return
+	}
+
+	started := make(chan struct{})
+	d.serverMu.Lock()
+	if d.ctx.Err() != nil {
+		d.serverMu.Unlock()
+		return
+	}
+	addr := d.addr
+	server := &dns.Server{
+		Addr:              addr,
+		Net:               "udp",
+		Handler:           d.mux,
+		NotifyStartedFunc: func() { close(started) },
+	}
+	d.server = server
+	d.started = started
+	d.serverMu.Unlock()
+
+	// Per-invocation ctx watcher. Exits when Start does, so we don't leak a
+	// watcher per reload-driven restart.
+	done := make(chan struct{})
+	go func() {
+		select {
+		case <-d.ctx.Done():
+			d.shutdownServer(server, started, "shutdown")
+		case <-done:
+		}
+	}()
+
+	d.l.WithField("dnsListener", addr).Info("Starting DNS responder")
+	err := server.ListenAndServe()
+	close(done)
+
+	// If the listener never bound (bind error) NotifyStartedFunc never fires,
+	// so close started here to release any Stop caller waiting on it.
+	select {
+	case <-started:
+	default:
+		close(started)
+	}
+
+	if err != nil {
+		d.l.WithError(err).Warn("Failed to run the DNS responder")
+	}
+}
+
+// Stop shuts down the active server, if any. Idempotent.
+func (d *dnsServer) Stop() {
+	d.serverMu.Lock()
+	srv := d.server
+	started := d.started
+	d.server = nil
+	d.started = nil
+	d.serverMu.Unlock()
+	d.shutdownServer(srv, started, "stop")
+}
+
+func (d *dnsServer) Query(q uint16, data string) netip.Addr {
 	data = strings.ToLower(data)
 	d.RLock()
 	defer d.RUnlock()
@@ -57,7 +234,7 @@ func (d *dnsRecords) Query(q uint16, data string) netip.Addr {
 	return netip.Addr{}
 }
 
-func (d *dnsRecords) QueryCert(data string) string {
+func (d *dnsServer) QueryCert(data string) string {
 	ip, err := netip.ParseAddr(data[:len(data)-1])
 	if err != nil {
 		return ""
@@ -80,8 +257,19 @@ func (d *dnsRecords) QueryCert(data string) string {
 	return string(b)
 }
 
+// clearRecords drops all DNS records.
+func (d *dnsServer) clearRecords() {
+	d.Lock()
+	defer d.Unlock()
+	clear(d.dnsMap4)
+	clear(d.dnsMap6)
+}
+
 // Add adds the first IPv4 and IPv6 address that appears in `addresses` as the record for `host`
-func (d *dnsRecords) Add(host string, addresses []netip.Addr) {
+func (d *dnsServer) Add(host string, addresses []netip.Addr) {
+	if !d.enabled.Load() {
+		return
+	}
 	host = strings.ToLower(host)
 	d.Lock()
 	defer d.Unlock()
@@ -101,7 +289,7 @@ func (d *dnsRecords) Add(host string, addresses []netip.Addr) {
 	}
 }
 
-func (d *dnsRecords) isSelfNebulaOrLocalhost(addr string) bool {
+func (d *dnsServer) isSelfNebulaOrLocalhost(addr string) bool {
 	a, _, _ := net.SplitHostPort(addr)
 	b, err := netip.ParseAddr(a)
 	if err != nil {
@@ -116,7 +304,7 @@ func (d *dnsRecords) isSelfNebulaOrLocalhost(addr string) bool {
 	return d.myVpnAddrsTable.Contains(b)
 }
 
-func (d *dnsRecords) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
+func (d *dnsServer) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
 	for _, q := range m.Question {
 		switch q.Qtype {
 		case dns.TypeA, dns.TypeAAAA:
@@ -150,7 +338,7 @@ func (d *dnsRecords) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
 	}
 }
 
-func (d *dnsRecords) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
+func (d *dnsServer) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
 	m := new(dns.Msg)
 	m.SetReply(r)
 	m.Compress = false
@@ -163,21 +351,6 @@ func (d *dnsRecords) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
 	w.WriteMsg(m)
 }
 
-func dnsMain(l *logrus.Logger, cs *CertState, hostMap *HostMap, c *config.C) func() {
-	dnsR = newDnsRecords(l, cs, hostMap)
-
-	// attach request handler func
-	dns.HandleFunc(".", dnsR.handleDnsRequest)
-
-	c.RegisterReloadCallback(func(c *config.C) {
-		reloadDns(l, c)
-	})
-
-	return func() {
-		startDns(l, c)
-	}
-}
-
 func getDnsServerAddr(c *config.C) string {
 	dnsHost := strings.TrimSpace(c.GetString("lighthouse.dns.host", ""))
 	// Old guidance was to provide the literal `[::]` in `lighthouse.dns.host` but that won't resolve.
@@ -186,25 +359,3 @@ func getDnsServerAddr(c *config.C) string {
 	}
 	return net.JoinHostPort(dnsHost, strconv.Itoa(c.GetInt("lighthouse.dns.port", 53)))
 }
-
-func startDns(l *logrus.Logger, c *config.C) {
-	dnsAddr = getDnsServerAddr(c)
-	dnsServer = &dns.Server{Addr: dnsAddr, Net: "udp"}
-	l.WithField("dnsListener", dnsAddr).Info("Starting DNS responder")
-	err := dnsServer.ListenAndServe()
-	defer dnsServer.Shutdown()
-	if err != nil {
-		l.Errorf("Failed to start server: %s\n ", err.Error())
-	}
-}
-
-func reloadDns(l *logrus.Logger, c *config.C) {
-	if dnsAddr == getDnsServerAddr(c) {
-		l.Debug("No DNS server config change detected")
-		return
-	}
-
-	l.Debug("Restarting DNS server")
-	dnsServer.Shutdown()
-	go startDns(l, c)
-}

dns_server_test.go

@@ -1,19 +1,31 @@
 package nebula
 
 import (
+	"context"
+	"io"
+	"net"
 	"net/netip"
+	"strconv"
 	"testing"
+	"time"
 
 	"github.com/miekg/dns"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestParsequery(t *testing.T) {
 	l := logrus.New()
 	hostMap := &HostMap{}
-	ds := newDnsRecords(l, &CertState{}, hostMap)
+	ds := &dnsServer{
+		l:       l,
+		dnsMap4: make(map[string]netip.Addr),
+		dnsMap6: make(map[string]netip.Addr),
+		hostMap: hostMap,
+	}
+	ds.enabled.Store(true)
 	addrs := []netip.Addr{
 		netip.MustParseAddr("1.2.3.4"),
 		netip.MustParseAddr("1.2.3.5"),
@@ -71,3 +83,209 @@ func Test_getDnsServerAddr(t *testing.T) {
 	}
 	assert.Equal(t, "[::]:1", getDnsServerAddr(c))
 }
+
+func newTestDnsServer(t *testing.T) (*dnsServer, *config.C) {
+	t.Helper()
+	l := logrus.New()
+	l.Out = io.Discard
+	ds := &dnsServer{
+		l:       l,
+		ctx:     context.Background(),
+		dnsMap4: make(map[string]netip.Addr),
+		dnsMap6: make(map[string]netip.Addr),
+		hostMap: &HostMap{},
+	}
+	ds.mux = dns.NewServeMux()
+	ds.mux.HandleFunc(".", ds.handleDnsRequest)
+	return ds, config.NewC(l)
+}
+
+func setDnsConfig(c *config.C, host string, port string, amLighthouse, serveDns bool) {
+	c.Settings["lighthouse"] = map[string]any{
+		"am_lighthouse": amLighthouse,
+		"serve_dns":     serveDns,
+		"dns": map[string]any{
+			"host": host,
+			"port": port,
+		},
+	}
+}
+
+func TestDnsServer_reload_initial_disabled(t *testing.T) {
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", "0", true, false)
+
+	require.NoError(t, ds.reload(c, true))
+	assert.False(t, ds.enabled.Load())
+	assert.Equal(t, "127.0.0.1:0", ds.addr)
+	assert.Nil(t, ds.server)
+}
+
+func TestDnsServer_reload_initial_enabled(t *testing.T) {
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", "0", true, true)
+
+	require.NoError(t, ds.reload(c, true))
+	assert.True(t, ds.enabled.Load())
+	assert.Equal(t, "127.0.0.1:0", ds.addr)
+	// initial never starts a runner; that's Control.Start's job
+	assert.Nil(t, ds.server)
+}
+
+func TestDnsServer_reload_initial_serveDnsWithoutLighthouse(t *testing.T) {
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", "0", false, true)
+
+	require.NoError(t, ds.reload(c, true))
+	// Wants DNS but isn't a lighthouse: gated off, no runner.
+	assert.False(t, ds.enabled.Load())
+}
+
+func TestDnsServer_reload_sameAddr_noOp(t *testing.T) {
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", "0", true, true)
+
+	require.NoError(t, ds.reload(c, true))
+	// No server running yet, no addr change. Reload should not spawn anything.
+	require.NoError(t, ds.reload(c, false))
+	assert.True(t, ds.enabled.Load())
+	assert.Nil(t, ds.server)
+}
+
+func TestDnsServer_StartStop_lifecycle(t *testing.T) {
+	// Bind to a real (random) UDP port so we exercise the actual
+	// ListenAndServe + Shutdown plumbing including the started-chan race fix.
+	port := freeUDPPort(t)
+
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", port, true, true)
+	require.NoError(t, ds.reload(c, true))
+
+	done := make(chan struct{})
+	go func() {
+		ds.Start()
+		close(done)
+	}()
+
+	waitFor(t, func() bool {
+		ds.serverMu.Lock()
+		started := ds.started
+		ds.serverMu.Unlock()
+		if started == nil {
+			return false
+		}
+		select {
+		case <-started:
+			return true
+		default:
+			return false
+		}
+	})
+
+	ds.Stop()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not return after Stop")
+	}
+}
+
+func TestDnsServer_Stop_beforeBind_doesNotHang(t *testing.T) {
+	// Stop called immediately after Start should not deadlock even if bind
+	// hasn't completed yet. This exercises the started-chan close-on-bind-fail
+	// path: by binding to an obviously bad port (privileged) we get a fast
+	// bind error before NotifyStartedFunc fires.
+	ds, c := newTestDnsServer(t)
+	// Use a port that should fail to bind (negative would be invalid, use a
+	// host that won't resolve to ensure listenUDP fails quickly).
+	setDnsConfig(c, "256.256.256.256", "53", true, true)
+	require.NoError(t, ds.reload(c, true))
+
+	done := make(chan struct{})
+	go func() {
+		ds.Start()
+		close(done)
+	}()
+
+	// Give Start a moment to attempt the bind and fail.
+	select {
+	case <-done:
+		// Bind failed and Start returned; Stop should be a no-op.
+	case <-time.After(time.Second):
+		t.Fatal("Start did not return after a bad bind")
+	}
+
+	stopped := make(chan struct{})
+	go func() {
+		ds.Stop()
+		close(stopped)
+	}()
+	select {
+	case <-stopped:
+	case <-time.After(time.Second):
+		t.Fatal("Stop hung after a failed bind")
+	}
+}
+
+func TestDnsServer_reload_disable_stopsRunningServer(t *testing.T) {
+	port := freeUDPPort(t)
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", port, true, true)
+	require.NoError(t, ds.reload(c, true))
+
+	startReturned := make(chan struct{})
+	go func() {
+		ds.Start()
+		close(startReturned)
+	}()
+	waitForBind(t, ds)
+
+	// Toggle serve_dns off; reload should shut the running server down.
+	setDnsConfig(c, "127.0.0.1", port, true, false)
+	require.NoError(t, ds.reload(c, false))
+	select {
+	case <-startReturned:
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not return after reload disabled DNS")
+	}
+	assert.False(t, ds.enabled.Load())
+}
+
+func freeUDPPort(t *testing.T) string {
+	t.Helper()
+	conn, err := net.ListenPacket("udp", "127.0.0.1:0")
+	require.NoError(t, err)
+	port := conn.LocalAddr().(*net.UDPAddr).Port
+	require.NoError(t, conn.Close())
+	return strconv.Itoa(port)
+}
+
+func waitForBind(t *testing.T, ds *dnsServer) {
+	t.Helper()
+	waitFor(t, func() bool {
+		ds.serverMu.Lock()
+		started := ds.started
+		ds.serverMu.Unlock()
+		if started == nil {
+			return false
+		}
+		select {
+		case <-started:
+			return true
+		default:
+			return false
+		}
+	})
+}
+
+func waitFor(t *testing.T, cond func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		if cond() {
+			return
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	t.Fatal("timed out waiting for condition")
+}

e2e/handshake_manager_test.go

@@ -0,0 +1,565 @@
+//go:build e2e_testing
+// +build e2e_testing
+
+package e2e
+
+import (
+	"net/netip"
+	"testing"
+	"time"
+
+	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/cert_test"
+	"github.com/slackhq/nebula/e2e/router"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/udp"
+	"github.com/stretchr/testify/assert"
+)
+
+// makeHandshakePacket creates a handshake packet with the given parameters.
+func makeHandshakePacket(from, to netip.AddrPort, subtype header.MessageSubType, remoteIndex uint32, counter uint64) *udp.Packet {
+	data := make([]byte, 200)
+	header.Encode(data, header.Version, header.Handshake, subtype, remoteIndex, counter)
+	for i := header.Len; i < len(data); i++ {
+		data[i] = byte(i)
+	}
+	return &udp.Packet{To: to, From: from, Data: data}
+}
+
+func TestHandshakeRetransmitDuplicate(t *testing.T) {
+	// Verify the responder correctly handles receiving the same msg1 multiple times
+	// (retransmission). The duplicate goes through CheckAndComplete -> ErrAlreadySeen
+	// and the cached response is resent.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	myControl.Start()
+	theirControl.Start()
+
+	r := router.NewR(t, myControl, theirControl)
+	defer r.RenderFlow()
+
+	t.Log("Trigger handshake from me to them")
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
+
+	t.Log("Grab my msg1")
+	msg1 := myControl.GetFromUDP(true)
+
+	t.Log("Inject msg1 into them, first time")
+	theirControl.InjectUDPPacket(msg1)
+	_ = theirControl.GetFromUDP(true)
+
+	t.Log("Inject the SAME msg1 again, tests ErrAlreadySeen path")
+	theirControl.InjectUDPPacket(msg1)
+	resp2 := theirControl.GetFromUDP(true)
+	assert.NotNil(t, resp2, "should get cached response on duplicate msg1")
+
+	t.Log("Complete handshake with cached response")
+	myControl.InjectUDPPacket(resp2)
+	myControl.WaitForType(1, 0, theirControl)
+
+	t.Log("Drain cached packet and verify tunnel works")
+	cachedPacket := theirControl.GetFromTun(true)
+	assertUdpPacket(t, []byte("Hi"), cachedPacket, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	t.Log("Verify only one tunnel exists on each side")
+	assert.Len(t, myControl.ListHostmapHosts(false), 1)
+	assert.Len(t, theirControl.ListHostmapHosts(false), 1)
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestHandshakeTruncatedPacketRecovery(t *testing.T) {
+	// Verify that a truncated handshake packet is ignored and the real
+	// packet can still complete the handshake.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	myControl.Start()
+	theirControl.Start()
+
+	r := router.NewR(t, myControl, theirControl)
+	defer r.RenderFlow()
+
+	t.Log("Trigger handshake")
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
+
+	t.Log("Get msg1 and deliver to responder")
+	msg1 := myControl.GetFromUDP(true)
+	theirControl.InjectUDPPacket(msg1)
+
+	t.Log("Get the real response")
+	realResp := theirControl.GetFromUDP(true)
+
+	t.Log("Truncate the response and inject, should be ignored")
+	truncResp := realResp.Copy()
+	truncResp.Data = truncResp.Data[:header.Len]
+	myControl.InjectUDPPacket(truncResp)
+
+	t.Log("Verify pending handshake survived the truncated packet")
+	assert.NotEmpty(t, myControl.ListHostmapHosts(true), "pending handshake should still exist")
+
+	t.Log("Inject real response, should complete handshake")
+	myControl.InjectUDPPacket(realResp)
+	myControl.WaitForType(1, 0, theirControl)
+
+	t.Log("Drain and verify tunnel")
+	cachedPacket := theirControl.GetFromTun(true)
+	assertUdpPacket(t, []byte("Hi"), cachedPacket, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestHandshakeOrphanedMsg2Dropped(t *testing.T) {
+	// A msg2 arriving with no matching pending index should be silently dropped
+	// with no response sent and no state changes.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	myControl.Start()
+	theirControl.Start()
+
+	r := router.NewR(t, myControl, theirControl)
+	defer r.RenderFlow()
+
+	t.Log("Complete a normal handshake")
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
+	r.RouteForAllUntilTxTun(theirControl)
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	t.Log("Record hostmap state")
+	myIndexes := len(myControl.ListHostmapIndexes(false))
+
+	t.Log("Inject a fake msg2 with unknown RemoteIndex")
+	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0xDEADBEEF, 2))
+
+	t.Log("Verify no new indexes created")
+	assert.Equal(t, myIndexes, len(myControl.ListHostmapIndexes(false)))
+
+	t.Log("Verify no UDP response was sent")
+	time.Sleep(100 * time.Millisecond)
+	assert.Nil(t, myControl.GetFromUDP(false), "should not send a response to orphaned msg2")
+
+	t.Log("Verify existing tunnel still works")
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestHandshakeUnknownMessageCounter(t *testing.T) {
+	// A handshake packet with an unexpected message counter should be silently
+	// dropped with no side effects and no UDP response.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+
+	myControl.Start()
+	theirControl.Start()
+
+	t.Log("Inject handshake with MessageCounter=3")
+	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 3))
+
+	t.Log("Inject handshake with MessageCounter=99")
+	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 99))
+
+	t.Log("Verify no tunnels or pending handshakes")
+	assert.Empty(t, myControl.ListHostmapHosts(false))
+	assert.Empty(t, myControl.ListHostmapHosts(true))
+
+	t.Log("Verify no UDP response was sent")
+	time.Sleep(100 * time.Millisecond)
+	assert.Nil(t, myControl.GetFromUDP(false))
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestHandshakeUnknownSubtype(t *testing.T) {
+	// A handshake packet with an unknown subtype should be silently dropped.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+	theirControl, _, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.Start()
+	theirControl.Start()
+
+	t.Log("Inject handshake with unknown subtype 99")
+	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.MessageSubType(99), 0, 1))
+
+	t.Log("Verify no tunnels or pending handshakes")
+	assert.Empty(t, myControl.ListHostmapHosts(false))
+	assert.Empty(t, myControl.ListHostmapHosts(true))
+
+	t.Log("Verify no UDP response was sent")
+	time.Sleep(100 * time.Millisecond)
+	assert.Nil(t, myControl.GetFromUDP(false))
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestHandshakeLateResponse(t *testing.T) {
+	// After a handshake times out, a late response should be silently ignored
+	// with no new tunnels created.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{
+		"handshakes": m{
+			"try_interval": "200ms",
+			"retries":      2,
+		},
+	})
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+
+	myControl.Start()
+	theirControl.Start()
+
+	t.Log("Trigger handshake from me")
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
+
+	t.Log("Grab msg1 but don't deliver")
+	msg1 := myControl.GetFromUDP(true)
+
+	t.Log("Wait for handshake to time out")
+	for i := 0; i < 5; i++ {
+		time.Sleep(300 * time.Millisecond)
+		myControl.GetFromUDP(false)
+	}
+
+	t.Log("Confirm no pending handshakes remain")
+	assert.Empty(t, myControl.ListHostmapHosts(true))
+
+	t.Log("Deliver old msg1 to them, they create a tunnel")
+	theirControl.InjectUDPPacket(msg1)
+	resp := theirControl.GetFromUDP(true)
+	assert.NotNil(t, resp)
+
+	t.Log("Inject late response into me, should be ignored")
+	myControl.InjectUDPPacket(resp)
+
+	t.Log("No tunnel should exist on my side")
+	assert.Empty(t, myControl.ListHostmapHosts(false))
+	assert.Empty(t, myControl.ListHostmapHosts(true))
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestHandshakeSelfConnectionRejected(t *testing.T) {
+	// Verify that a node rejects a handshake containing its own VPN IP in the
+	// peer cert. We do this by sending the initiator's own msg1 back to itself.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+
+	// Need a lighthouse entry to trigger a handshake
+	myControl.InjectLightHouseAddr(netip.MustParseAddr("10.128.0.2"), netip.MustParseAddrPort("10.0.0.2:4242"))
+
+	myControl.Start()
+
+	t.Log("Trigger handshake from me")
+	myControl.InjectTunUDPPacket(netip.MustParseAddr("10.128.0.2"), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
+	msg1 := myControl.GetFromUDP(true)
+
+	t.Log("Drain any handshake retransmits before injecting")
+	time.Sleep(100 * time.Millisecond)
+	for myControl.GetFromUDP(false) != nil {
+	}
+
+	t.Log("Feed my own msg1 back to me as if it came from someone else")
+	selfMsg := msg1.Copy()
+	selfMsg.From = netip.MustParseAddrPort("10.0.0.99:4242")
+	selfMsg.To = myUdpAddr
+	myControl.InjectUDPPacket(selfMsg)
+
+	t.Log("Verify no response was sent (self-connection rejected)")
+	time.Sleep(100 * time.Millisecond)
+	// Drain any further retransmits from the original handshake, then check
+	// that none of them are a handshake response (MessageCounter=2)
+	h := &header.H{}
+	for {
+		p := myControl.GetFromUDP(false)
+		if p == nil {
+			break
+		}
+		_ = h.Parse(p.Data)
+		assert.NotEqual(t, uint64(2), h.MessageCounter,
+			"should not send a stage 2 response to self-connection")
+	}
+
+	t.Log("Verify no tunnel to myself was created")
+	assert.Nil(t, myControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false))
+
+	myControl.Stop()
+}
+
+func TestHandshakeMessageCounter0Dropped(t *testing.T) {
+	// MessageCounter=0 is not a valid handshake message and should be dropped.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+	_, _, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.Start()
+
+	t.Log("Inject handshake with MessageCounter=0")
+	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 0))
+
+	time.Sleep(100 * time.Millisecond)
+	assert.Empty(t, myControl.ListHostmapHosts(false))
+	assert.Empty(t, myControl.ListHostmapHosts(true))
+	assert.Nil(t, myControl.GetFromUDP(false))
+
+	myControl.Stop()
+}
+
+func TestHandshakeRemoteAllowList(t *testing.T) {
+	// Verify that a handshake from a blocked underlay IP is dropped with no
+	// response and no state changes. Then verify the same packet from an
+	// allowed IP succeeds.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{
+		"lighthouse": m{
+			"remote_allow_list": m{
+				"10.0.0.0/8": true,
+				"0.0.0.0/0":  false,
+			},
+		},
+	})
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	myControl.Start()
+	theirControl.Start()
+
+	r := router.NewR(t, myControl, theirControl)
+	defer r.RenderFlow()
+
+	t.Log("Trigger handshake from them")
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi"))
+	msg1 := theirControl.GetFromUDP(true)
+
+	t.Log("Rewrite the source to a blocked IP and inject")
+	blockedMsg := msg1.Copy()
+	blockedMsg.From = netip.MustParseAddrPort("192.168.1.1:4242")
+	myControl.InjectUDPPacket(blockedMsg)
+
+	t.Log("Verify no tunnel, no pending, no response from blocked source")
+	time.Sleep(100 * time.Millisecond)
+	assert.Empty(t, myControl.ListHostmapHosts(false))
+	assert.Empty(t, myControl.ListHostmapHosts(true))
+	assert.Nil(t, myControl.GetFromUDP(false), "should not respond to blocked source")
+
+	t.Log("Now inject the real packet from the allowed source")
+	myControl.InjectUDPPacket(msg1)
+
+	t.Log("Verify handshake completes from allowed source")
+	resp := myControl.GetFromUDP(true)
+	assert.NotNil(t, resp)
+	theirControl.InjectUDPPacket(resp)
+	theirControl.WaitForType(1, 0, myControl)
+
+	t.Log("Drain cached packet and verify tunnel works")
+	cachedPacket := myControl.GetFromTun(true)
+	assertUdpPacket(t, []byte("Hi"), cachedPacket, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), 80, 80)
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestHandshakeAlreadySeenPreferredRemote(t *testing.T) {
+	// When a duplicate msg1 arrives via ErrAlreadySeen, verify the tunnel
+	// remains functional and hostmap index count is stable.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	myControl.Start()
+	theirControl.Start()
+
+	r := router.NewR(t, myControl, theirControl)
+	defer r.RenderFlow()
+
+	t.Log("Complete a normal handshake via the router")
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
+	r.RouteForAllUntilTxTun(theirControl)
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	t.Log("Record hostmap state")
+	theirIndexes := len(theirControl.ListHostmapIndexes(false))
+	hi := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
+	assert.NotNil(t, hi)
+	originalRemote := hi.CurrentRemote
+
+	t.Log("Re-trigger traffic to cause a new handshake attempt (ErrAlreadySeen)")
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("roam"))
+	r.RouteForAllUntilTxTun(theirControl)
+
+	t.Log("Verify tunnel still works")
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	t.Log("Verify remote is still valid and index count is stable")
+	hi2 := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
+	assert.NotNil(t, hi2)
+	assert.Equal(t, originalRemote, hi2.CurrentRemote)
+	assert.Equal(t, theirIndexes, len(theirControl.ListHostmapIndexes(false)),
+		"no extra indexes should be created from ErrAlreadySeen")
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestHandshakeWrongResponderPacketStore(t *testing.T) {
+	// Verify that when the wrong host responds, the cached packets are
+	// transferred to the new handshake, the evil tunnel is closed, evil's
+	// address is blocked, and the correct tunnel is eventually established.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.100/24", nil)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.99/24", nil)
+	evilControl, evilVpnIpNet, evilUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "evil", "10.128.0.2/24", nil)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), evilUdpAddr)
+
+	r := router.NewR(t, myControl, theirControl, evilControl)
+	defer r.RenderFlow()
+
+	myControl.Start()
+	theirControl.Start()
+	evilControl.Start()
+
+	t.Log("Send multiple packets to them (cached during handshake)")
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet1"))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet2"))
+
+	t.Log("Route until evil tunnel is closed")
+	h := &header.H{}
+	r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType {
+		if err := h.Parse(p.Data); err != nil {
+			panic(err)
+		}
+		if h.Type == header.CloseTunnel && p.To == evilUdpAddr {
+			return router.RouteAndExit
+		}
+		return router.KeepRouting
+	})
+
+	t.Log("Verify evil's address is blocked in the new pending handshake")
+	pendingHI := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), true)
+	if pendingHI != nil {
+		assert.NotContains(t, pendingHI.RemoteAddrs, evilUdpAddr,
+			"evil's address should be blocked")
+	}
+
+	t.Log("Inject correct lighthouse addr for them")
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+
+	t.Log("Route until cached packets arrive at the real them")
+	p := r.RouteForAllUntilTxTun(theirControl)
+	assert.NotNil(t, p, "a cached packet should be delivered to the correct host")
+
+	t.Log("Verify the correct host has a tunnel")
+	assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet, theirVpnIpNet, myControl, theirControl)
+
+	t.Log("Verify no hostinfo artifacts from evil remain")
+	assert.Nil(t, myControl.GetHostInfoByVpnAddr(evilVpnIpNet[0].Addr(), true),
+		"no pending hostinfo for evil")
+	assert.Nil(t, myControl.GetHostInfoByVpnAddr(evilVpnIpNet[0].Addr(), false),
+		"no main hostinfo for evil")
+
+	myControl.Stop()
+	theirControl.Stop()
+	evilControl.Stop()
+}
+
+func TestHandshakeRelayComplete(t *testing.T) {
+	// Verify that a relay handshake completes correctly and relay state is
+	// properly maintained on all three nodes.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
+	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
+
+	myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
+	myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
+	relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+
+	r := router.NewR(t, myControl, relayControl, theirControl)
+	defer r.RenderFlow()
+
+	myControl.Start()
+	relayControl.Start()
+	theirControl.Start()
+
+	t.Log("Trigger handshake via relay")
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi via relay"))
+
+	p := r.RouteForAllUntilTxTun(theirControl)
+	assertUdpPacket(t, []byte("Hi via relay"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
+
+	t.Log("Verify bidirectional tunnel via relay")
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	t.Log("Verify relay state on my side shows relay-to-me")
+	myHI := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
+	assert.NotNil(t, myHI)
+	assert.NotEmpty(t, myHI.CurrentRelaysToMe, "should have relay-to-me for them")
+
+	t.Log("Verify relay state on their side shows relay-to-me")
+	theirHI := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
+	assert.NotNil(t, theirHI)
+	assert.NotEmpty(t, theirHI.CurrentRelaysToMe, "should have relay-to-me for me")
+
+	t.Log("Verify relay node shows through-me relays")
+	relayHI := relayControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
+	assert.NotNil(t, relayHI)
+
+	myControl.Stop()
+	relayControl.Stop()
+	theirControl.Stop()
+}
+
+// NOTE: Relay V1 cert + IPv6 rejection is not tested here because
+// InjectTunUDPPacket from a V4 node to a V6 address panics in the test
+// framework. The check is in handshake_manager.go handleOutbound relay
+// logic (lines ~304-313): if the relay host has a V1 cert and either
+// address is IPv6, the relay is skipped.
+
+// NOTE: Relay reestablishment (Disestablished state transition) is covered
+// by the existing TestReestablishRelays in handshakes_test.go.

e2e/snat_test.go

@@ -1,400 +0,0 @@
-//go:build e2e_testing
-// +build e2e_testing
-
-package e2e
-
-import (
-	"encoding/binary"
-	"net/netip"
-	"testing"
-	"time"
-
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/e2e/router"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// parseIPv4UDPPacket extracts source/dest IPs, ports, and payload from an IPv4 UDP packet.
-func parseIPv4UDPPacket(t testing.TB, pkt []byte) (srcIP, dstIP netip.Addr, srcPort, dstPort uint16, payload []byte) {
-	t.Helper()
-	require.True(t, len(pkt) >= 28, "packet too short for IPv4+UDP header")
-	require.Equal(t, byte(0x45), pkt[0]&0xF0|pkt[0]&0x0F, "not a simple IPv4 packet (IHL!=5)")
-
-	srcIP, _ = netip.AddrFromSlice(pkt[12:16])
-	dstIP, _ = netip.AddrFromSlice(pkt[16:20])
-
-	ihl := int(pkt[0]&0x0F) * 4
-	require.True(t, len(pkt) >= ihl+8, "packet too short for UDP header")
-	srcPort = binary.BigEndian.Uint16(pkt[ihl : ihl+2])
-	dstPort = binary.BigEndian.Uint16(pkt[ihl+2 : ihl+4])
-	udpLen := binary.BigEndian.Uint16(pkt[ihl+4 : ihl+6])
-	payload = pkt[ihl+8 : ihl+int(udpLen)]
-	return
-}
-
-func TestSNAT_IPv6OnlyPeer_IPv4UnsafeTraffic(t *testing.T) {
-	// Scenario: Two IPv6-only VPN nodes. The "router" node has unsafe networks
-	// (192.168.0.0/16) in its cert and a configured SNAT address. The "sender"
-	// node has an unsafe route for 192.168.0.0/16 via the router.
-	//
-	// When sender injects an IPv4 packet destined for the unsafe network, it
-	// gets tunneled to the router. The router's firewall detects this is IPv4
-	// from an IPv6-only peer and applies SNAT, rewriting the source IP to the
-	// SNAT address before delivering it to TUN.
-	//
-	// When a reply comes back from TUN addressed to the SNAT address, the
-	// router un-SNATs it (restoring the original destination) and tunnels it
-	// back to the sender.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	unsafePrefix := "192.168.0.0/16"
-	snatAddr := netip.MustParseAddr("169.254.42.42")
-
-	// Router: IPv6-only with unsafe networks and a manual SNAT address.
-	// Override inbound firewall with local_cidr: "any" so both IPv4 (unsafe)
-	// and IPv6 (VPN) traffic is accepted.
-	routerControl, routerVpnIpNet, routerUdpAddr, _ := newSimpleServerWithUdpAndUnsafeNetworks(
-		cert.Version2, ca, caKey, "router", "ff::1/64",
-		netip.MustParseAddrPort("[beef::1]:4242"),
-		unsafePrefix,
-		m{
-			"firewall": m{
-				"inbound": []m{{
-					"proto":      "any",
-					"port":       "any",
-					"host":       "any",
-					"local_cidr": "any",
-				}},
-			},
-			"tun": m{
-				"snat_address_for_4over6": snatAddr.String(),
-			},
-		},
-	)
-
-	// Sender: IPv6-only with an unsafe route via the router
-	senderControl, _, _, _ := newSimpleServerWithUdp(
-		cert.Version2, ca, caKey, "sender", "ff::2/64",
-		netip.MustParseAddrPort("[beef::2]:4242"),
-		m{
-			"tun": m{
-				"unsafe_routes": []m{
-					{"route": unsafePrefix, "via": routerVpnIpNet[0].Addr().String()},
-				},
-			},
-		},
-	)
-
-	// Tell sender where the router lives
-	senderControl.InjectLightHouseAddr(routerVpnIpNet[0].Addr(), routerUdpAddr)
-
-	// Build the router and start both nodes
-	r := router.NewR(t, routerControl, senderControl)
-	defer r.RenderFlow()
-
-	routerControl.Start()
-	senderControl.Start()
-
-	// --- Outbound: sender -> IPv4 unsafe dest (via router with SNAT) ---
-
-	origSrcIP := netip.MustParseAddr("10.0.0.1")
-	unsafeDest := netip.MustParseAddr("192.168.1.1")
-	var origSrcPort uint16 = 12345
-	var dstPort uint16 = 80
-
-	t.Log("Sender injects an IPv4 packet to the unsafe network")
-	senderControl.InjectTunUDPPacket(unsafeDest, dstPort, origSrcIP, origSrcPort, []byte("snat me"))
-
-	t.Log("Route packets (handshake + data) until the router gets the packet on TUN")
-	snatPkt := r.RouteForAllUntilTxTun(routerControl)
-
-	t.Log("Verify the packet was SNATted")
-	gotSrcIP, gotDstIP, gotSrcPort, gotDstPort, gotPayload := parseIPv4UDPPacket(t, snatPkt)
-	assert.Equal(t, snatAddr, gotSrcIP, "source IP should be rewritten to the SNAT address")
-	assert.Equal(t, unsafeDest, gotDstIP, "destination IP should be unchanged")
-	assert.Equal(t, dstPort, gotDstPort, "destination port should be unchanged")
-	assert.Equal(t, []byte("snat me"), gotPayload, "payload should be unchanged")
-
-	// Capture the SNAT port (may differ from original if port was remapped)
-	snatPort := gotSrcPort
-	t.Logf("SNAT port: %d (original: %d)", snatPort, origSrcPort)
-
-	// --- Return: reply from unsafe dest -> un-SNATted back to sender ---
-
-	t.Log("Router injects a reply packet from the unsafe dest to the SNAT address")
-	routerControl.InjectTunUDPPacket(snatAddr, snatPort, unsafeDest, dstPort, []byte("reply from unsafe"))
-
-	t.Log("Route until sender gets the reply on TUN")
-	replyPkt := r.RouteForAllUntilTxTun(senderControl)
-
-	t.Log("Verify the reply was un-SNATted")
-	replySrcIP, replyDstIP, replySrcPort, replyDstPort, replyPayload := parseIPv4UDPPacket(t, replyPkt)
-	assert.Equal(t, unsafeDest, replySrcIP, "reply source should be the unsafe dest")
-	assert.Equal(t, origSrcIP, replyDstIP, "reply dest should be the original source IP (un-SNATted)")
-	assert.Equal(t, dstPort, replySrcPort, "reply source port should be the unsafe dest port")
-	assert.Equal(t, origSrcPort, replyDstPort, "reply dest port should be the original source port (un-SNATted)")
-	assert.Equal(t, []byte("reply from unsafe"), replyPayload, "payload should be unchanged")
-
-	r.RenderHostmaps("Final hostmaps", routerControl, senderControl)
-
-	// Also verify normal IPv6 VPN traffic still works between the nodes
-	t.Log("Verify normal IPv6 VPN tunnel still works")
-	assertTunnel(t, routerVpnIpNet[0].Addr(), senderControl.GetVpnAddrs()[0], routerControl, senderControl, r)
-
-	routerControl.Stop()
-	senderControl.Stop()
-}
-
-func TestSNAT_MultipleFlows(t *testing.T) {
-	// Test that multiple distinct IPv4 flows from the same IPv6-only peer
-	// are tracked independently through SNAT.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	unsafePrefix := "192.168.0.0/16"
-	snatAddr := netip.MustParseAddr("169.254.42.42")
-
-	routerControl, routerVpnIpNet, routerUdpAddr, _ := newSimpleServerWithUdpAndUnsafeNetworks(
-		cert.Version2, ca, caKey, "router", "ff::1/64",
-		netip.MustParseAddrPort("[beef::1]:4242"),
-		unsafePrefix,
-		m{
-			"firewall": m{
-				"inbound": []m{{
-					"proto":      "any",
-					"port":       "any",
-					"host":       "any",
-					"local_cidr": "any",
-				}},
-			},
-			"tun": m{
-				"snat_address_for_4over6": snatAddr.String(),
-			},
-		},
-	)
-
-	senderControl, _, _, _ := newSimpleServerWithUdp(
-		cert.Version2, ca, caKey, "sender", "ff::2/64",
-		netip.MustParseAddrPort("[beef::2]:4242"),
-		m{
-			"tun": m{
-				"unsafe_routes": []m{
-					{"route": unsafePrefix, "via": routerVpnIpNet[0].Addr().String()},
-				},
-			},
-		},
-	)
-
-	senderControl.InjectLightHouseAddr(routerVpnIpNet[0].Addr(), routerUdpAddr)
-
-	r := router.NewR(t, routerControl, senderControl)
-	defer r.RenderFlow()
-	r.CancelFlowLogs()
-
-	routerControl.Start()
-	senderControl.Start()
-
-	unsafeDest := netip.MustParseAddr("192.168.1.1")
-
-	// Send first flow
-	senderControl.InjectTunUDPPacket(unsafeDest, 80, netip.MustParseAddr("10.0.0.1"), 1111, []byte("flow1"))
-	pkt1 := r.RouteForAllUntilTxTun(routerControl)
-	srcIP1, _, srcPort1, _, payload1 := parseIPv4UDPPacket(t, pkt1)
-	assert.Equal(t, snatAddr, srcIP1)
-	assert.Equal(t, []byte("flow1"), payload1)
-
-	// Send second flow (different source port)
-	senderControl.InjectTunUDPPacket(unsafeDest, 80, netip.MustParseAddr("10.0.0.1"), 2222, []byte("flow2"))
-	pkt2 := r.RouteForAllUntilTxTun(routerControl)
-	srcIP2, _, srcPort2, _, payload2 := parseIPv4UDPPacket(t, pkt2)
-	assert.Equal(t, snatAddr, srcIP2)
-	assert.Equal(t, []byte("flow2"), payload2)
-
-	// The two flows should have different SNAT ports (since they're different conntracks)
-	t.Logf("Flow 1 SNAT port: %d, Flow 2 SNAT port: %d", srcPort1, srcPort2)
-
-	// Reply to flow 2 first (out of order)
-	routerControl.InjectTunUDPPacket(snatAddr, srcPort2, unsafeDest, 80, []byte("reply2"))
-	reply2 := r.RouteForAllUntilTxTun(senderControl)
-	_, replyDst2, _, replyDstPort2, replyPayload2 := parseIPv4UDPPacket(t, reply2)
-	assert.Equal(t, netip.MustParseAddr("10.0.0.1"), replyDst2)
-	assert.Equal(t, uint16(2222), replyDstPort2, "reply to flow 2 should restore original port 2222")
-	assert.Equal(t, []byte("reply2"), replyPayload2)
-
-	// Reply to flow 1
-	routerControl.InjectTunUDPPacket(snatAddr, srcPort1, unsafeDest, 80, []byte("reply1"))
-	reply1 := r.RouteForAllUntilTxTun(senderControl)
-	_, replyDst1, _, replyDstPort1, replyPayload1 := parseIPv4UDPPacket(t, reply1)
-	assert.Equal(t, netip.MustParseAddr("10.0.0.1"), replyDst1)
-	assert.Equal(t, uint16(1111), replyDstPort1, "reply to flow 1 should restore original port 1111")
-	assert.Equal(t, []byte("reply1"), replyPayload1)
-
-	routerControl.Stop()
-	senderControl.Stop()
-}
-
-// --- Adversarial SNAT E2E Tests ---
-
-func TestSNAT_UnsolicitedReplyDropped(t *testing.T) {
-	// Without any outbound SNAT traffic, inject a packet from the router's TUN
-	// addressed to the SNAT address. The sender must never receive it because
-	// there's no conntrack entry to un-SNAT through.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	unsafePrefix := "192.168.0.0/16"
-	snatAddr := netip.MustParseAddr("169.254.42.42")
-
-	routerControl, routerVpnIpNet, routerUdpAddr, _ := newSimpleServerWithUdpAndUnsafeNetworks(
-		cert.Version2, ca, caKey, "router", "ff::1/64",
-		netip.MustParseAddrPort("[beef::1]:4242"),
-		unsafePrefix,
-		m{
-			"firewall": m{
-				"inbound": []m{{
-					"proto":      "any",
-					"port":       "any",
-					"host":       "any",
-					"local_cidr": "any",
-				}},
-			},
-			"tun": m{
-				"snat_address_for_4over6": snatAddr.String(),
-			},
-		},
-	)
-
-	senderControl, _, _, _ := newSimpleServerWithUdp(
-		cert.Version2, ca, caKey, "sender", "ff::2/64",
-		netip.MustParseAddrPort("[beef::2]:4242"),
-		m{
-			"tun": m{
-				"unsafe_routes": []m{
-					{"route": unsafePrefix, "via": routerVpnIpNet[0].Addr().String()},
-				},
-			},
-		},
-	)
-
-	senderControl.InjectLightHouseAddr(routerVpnIpNet[0].Addr(), routerUdpAddr)
-
-	r := router.NewR(t, routerControl, senderControl)
-	defer r.RenderFlow()
-	r.CancelFlowLogs()
-
-	routerControl.Start()
-	senderControl.Start()
-
-	// First establish the tunnel with normal IPv6 traffic so handshake completes
-	assertTunnel(t, routerVpnIpNet[0].Addr(), senderControl.GetVpnAddrs()[0], routerControl, senderControl, r)
-
-	// Inject the unsolicited reply from router's TUN to the SNAT address.
-	// There is NO prior outbound SNAT flow, so no conntrack entry exists.
-	// The router should silently drop this because unSnat finds no matching conntrack.
-	routerControl.InjectTunUDPPacket(snatAddr, 55555, netip.MustParseAddr("192.168.1.1"), 80, []byte("unsolicited"))
-
-	// Send a canary IPv6 VPN packet after the bad one. Since the router processes
-	// TUN packets sequentially, the canary arriving proves the bad packet was processed first.
-	senderVpnAddr := senderControl.GetVpnAddrs()[0]
-	routerControl.InjectTunUDPPacket(senderVpnAddr, 90, routerVpnIpNet[0].Addr(), 80, []byte("canary"))
-	canaryPkt := r.RouteForAllUntilTxTun(senderControl)
-	assertUdpPacket(t, []byte("canary"), canaryPkt, routerVpnIpNet[0].Addr(), senderVpnAddr, 80, 90)
-
-	// The unsolicited packet should have been dropped — nothing else on sender's TUN
-	got := senderControl.GetFromTun(false)
-	assert.Nil(t, got, "sender should not receive unsolicited packet to SNAT address with no conntrack entry")
-
-	routerControl.Stop()
-	senderControl.Stop()
-}
-
-func TestSNAT_NonUnsafeDestDropped(t *testing.T) {
-	// An IPv6-only sender sends IPv4 traffic to a destination outside the router's
-	// unsafe networks (172.16.0.1 when unsafe is 192.168.0.0/16). The router should
-	// reject this because the local address is not routable. This verifies that
-	// willingToHandleLocalAddr enforces boundaries on what SNAT traffic can reach.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	unsafePrefix := "192.168.0.0/16"
-	snatAddr := netip.MustParseAddr("169.254.42.42")
-
-	routerControl, routerVpnIpNet, routerUdpAddr, _ := newSimpleServerWithUdpAndUnsafeNetworks(
-		cert.Version2, ca, caKey, "router", "ff::1/64",
-		netip.MustParseAddrPort("[beef::1]:4242"),
-		unsafePrefix,
-		m{
-			"firewall": m{
-				"inbound": []m{{
-					"proto":      "any",
-					"port":       "any",
-					"host":       "any",
-					"local_cidr": "any",
-				}},
-			},
-			"tun": m{
-				"snat_address_for_4over6": snatAddr.String(),
-			},
-		},
-	)
-
-	// Sender has unsafe routes for BOTH 192.168.0.0/16 AND 172.16.0.0/12 via router.
-	// This means the sender will route 172.16.0.1 through the tunnel to the router.
-	// But the router should reject it because 172.16.0.0/12 is NOT in its unsafe networks.
-	senderControl, _, _, _ := newSimpleServerWithUdp(
-		cert.Version2, ca, caKey, "sender", "ff::2/64",
-		netip.MustParseAddrPort("[beef::2]:4242"),
-		m{
-			"tun": m{
-				"unsafe_routes": []m{
-					{"route": unsafePrefix, "via": routerVpnIpNet[0].Addr().String()},
-					{"route": "172.16.0.0/12", "via": routerVpnIpNet[0].Addr().String()},
-				},
-			},
-		},
-	)
-
-	senderControl.InjectLightHouseAddr(routerVpnIpNet[0].Addr(), routerUdpAddr)
-
-	r := router.NewR(t, routerControl, senderControl)
-	defer r.RenderFlow()
-	r.CancelFlowLogs()
-
-	routerControl.Start()
-	senderControl.Start()
-
-	// Establish the tunnel first
-	assertTunnel(t, routerVpnIpNet[0].Addr(), senderControl.GetVpnAddrs()[0], routerControl, senderControl, r)
-
-	// Send to 172.16.0.1 (NOT in router's unsafe networks 192.168.0.0/16).
-	// The router should reject this at willingToHandleLocalAddr.
-	senderControl.InjectTunUDPPacket(
-		netip.MustParseAddr("172.16.0.1"), 80,
-		netip.MustParseAddr("10.0.0.1"), 12345,
-		[]byte("wrong dest"),
-	)
-
-	// Send a canary to a valid unsafe destination to prove the bad packet was processed
-	senderControl.InjectTunUDPPacket(
-		netip.MustParseAddr("192.168.1.1"), 80,
-		netip.MustParseAddr("10.0.0.1"), 33333,
-		[]byte("canary"),
-	)
-
-	// Route until the canary arrives — the 172.16.0.1 packet should have been
-	// processed and dropped before the canary gets through
-	canaryPkt := r.RouteForAllUntilTxTun(routerControl)
-	_, canaryDst, _, _, canaryPayload := parseIPv4UDPPacket(t, canaryPkt)
-	assert.Equal(t, netip.MustParseAddr("192.168.1.1"), canaryDst, "canary should arrive at the valid unsafe dest")
-	assert.Equal(t, []byte("canary"), canaryPayload)
-
-	// No more packets — the 172.16.0.1 packet was dropped
-	got := routerControl.GetFromTun(false)
-	assert.Nil(t, got, "packet to non-unsafe destination 172.16.0.1 should be dropped by the router")
-
-	routerControl.Stop()
-	senderControl.Stop()
-}

e2e/tunnels_test.go

@@ -12,6 +12,8 @@ import (
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/cert_test"
 	"github.com/slackhq/nebula/e2e/router"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/udp"
 	"github.com/stretchr/testify/assert"
 	"gopkg.in/yaml.v3"
 )
@@ -365,3 +367,106 @@ func TestCrossStackRelaysWork(t *testing.T) {
 	//theirControl.Stop()
 	//relayControl.Stop()
 }
+
+func TestCloseTunnelAuthenticated(t *testing.T) {
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "5s"}})
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "10m"}})
+
+	// Share our underlay information
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	// Start the servers
+	myControl.Start()
+	theirControl.Start()
+
+	r := router.NewR(t, myControl, theirControl)
+
+	r.Log("Assert the tunnel between me and them works")
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	r.Log("Close the tunnel")
+	myControl.CloseTunnel(theirVpnIpNet[0].Addr(), false)
+	r.FlushAll()
+
+	waitStart := time.Now()
+	for {
+		myIndexes := len(myControl.GetHostmap().Indexes)
+		theirIndexes := len(theirControl.GetHostmap().Indexes)
+		if myIndexes == 0 && theirIndexes == 0 {
+			break
+		}
+
+		since := time.Since(waitStart)
+		r.Logf("my tunnels: %v; their tunnels: %v; duration: %v", myIndexes, theirIndexes, since)
+		if since > time.Second*6 {
+			t.Fatal("Tunnel should have been declared inactive after 2 seconds and before 6 seconds")
+		}
+
+		time.Sleep(1 * time.Second)
+		//r.FlushAll()
+	}
+
+	r.Logf("Happy path success, tunnels were dropped within %v", time.Since(waitStart))
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+	r.Log("Assert another tunnel between me and them works")
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+	hi := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
+	if hi == nil {
+		t.Fatal("There is no hostinfo for this tunnel")
+	}
+	myHi := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
+	if myHi == nil {
+		t.Fatal("There is no hostinfo for my tunnel")
+	}
+	r.Log("It does")
+
+	buf := make([]byte, 1024)
+	hdr := header.H{
+		Version:        1,
+		Type:           header.CloseTunnel,
+		Subtype:        0,
+		Reserved:       0,
+		RemoteIndex:    hi.RemoteIndex,
+		MessageCounter: 5,
+	}
+	out, err := hdr.Encode(buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pkt := &udp.Packet{
+		To:   hi.CurrentRemote,
+		From: myHi.CurrentRemote,
+		Data: out,
+	}
+	r.InjectUDPPacket(myControl, theirControl, pkt)
+	r.Log("Injected bogus close tunnel. Let's see!")
+	waitStart = time.Now()
+	for {
+		myIndexes := len(myControl.GetHostmap().Indexes)
+		theirIndexes := len(theirControl.GetHostmap().Indexes)
+		if myIndexes == 0 {
+			t.Fatal("myIndexes should not be 0")
+		}
+		if theirIndexes == 0 {
+			t.Fatal("theirIndexes should not be 0, they should have rejected this bogus packet")
+		}
+
+		since := time.Since(waitStart)
+		r.Logf("my tunnels: %v; their tunnels: %v; duration: %v", myIndexes, theirIndexes, since)
+		if since > time.Second*4 {
+			t.Log("The tunnel would have been gone by now")
+			break
+		}
+
+		time.Sleep(1 * time.Second)
+		r.FlushAll()
+	}
+
+	myControl.Stop()
+	theirControl.Stop()
+}

examples/config.yml

@@ -204,6 +204,12 @@ punchy:
   # Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access.
   #trusted_cas:
     #- "ssh public key string"
+  # sandbox_dir restricts file paths for profiling commands (start-cpu-profile, save-heap-profile,
+  # save-mutex-profile) to the specified directory. Relative paths will be resolved within this directory,
+  # and absolute paths outside of it will be rejected. Default is $TMP/nebula-debug.
+  # The directory is NOT automatically created.
+  # Overriding this to "" is the same as "/" and will allow overwriting any path on the host.
+  #sandbox_dir: /var/tmp/nebula-debug
 
 # EXPERIMENTAL: relay support for networks that can't establish direct connections.
 relay:
@@ -283,15 +289,6 @@ tun:
   # If using massive routes updates, for example BGP, you may need to increase this value to avoid packet loss.
   # SO_RCVBUFFORCE is used to avoid having to raise the system wide max
   #use_system_route_table_buffer_size: 0
-  #
-  # When using the feature to carry IPv4 unsafe_routes through a IPv6-only overlay network (4over6),
-  # the system still needs to use IPv4 addresses to pass that traffic. 
-  # By default, they are randomly chosen out of 169.254.0.0/16 at startup, but if you need them to be static, you can set them here.
-  # Setting these options is not required for use of the 4over6 feature.
-  # this is the address that a client will use as a source IP for packets sent to the router
-  #unsafe_origin_address_for_4over6: 169.254.55.96
-  # this is the IP the router will use internally to hairpin-NAT 4over6 traffic before it hits the operating system.
-  #snat_address_for_4over6: 169.254.12.34
 
 # Configure logging level
 logging:

firewall.go

@@ -2,7 +2,6 @@ package nebula
 
 import (
 	"crypto/sha256"
-	"encoding/binary"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -21,37 +20,13 @@ import (
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/firewall"
+	"github.com/slackhq/nebula/firewall/events"
 )
 
-var ErrCannotSNAT = errors.New("cannot SNAT this packet")
-var ErrSNATIdentityMismatch = errors.New("refusing to SNAT for mismatched host")
-var ErrSNATAddressCollision = errors.New("refusing to accept an incoming packet with my SNAT address")
-
-const ipv4SourcePosition = 12
-const ipv4DestinationPosition = 16
-const sourcePortOffset = 0
-const destinationPortOffset = 2
-
 type FirewallInterface interface {
 	AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr string, caName string, caSha string) error
 }
 
-type snatInfo struct {
-	//Src is the source IP+port to write into unsafe-route-bound packet
-	Src netip.AddrPort
-	//SrcVpnIp is the overlay IP associated with this flow. It's needed to associate reply traffic so we can get it back to the right host.
-	SrcVpnIp netip.Addr
-	//SnatPort is the port to rewrite into an overlay-bound packet
-	SnatPort uint16
-}
-
-func (s *snatInfo) Valid() bool {
-	if s == nil {
-		return false
-	}
-	return s.Src.IsValid()
-}
-
 type conn struct {
 	Expires time.Time // Time when this conntrack entry will expire
 
@@ -60,9 +35,6 @@ type conn struct {
 	// fields pack for free after the uint32 above
 	incoming     bool
 	rulesVersion uint16
-
-	//for SNAT support
-	snat *snatInfo
 }
 
 // TODO: need conntrack max tracked connections handling
@@ -95,8 +67,14 @@ type Firewall struct {
 	defaultLocalCIDRAny bool
 	incomingMetrics     firewallMetrics
 	outgoingMetrics     firewallMetrics
-	unsafeIPv4Origin    netip.Addr
+
-	snatAddr            netip.Addr
+	// reporter is the optional embedder-supplied event sink. Immutable for
+	// the lifetime of this Firewall; Control.SetFirewallEventReporter
+	// installs it by shallow-copying the Firewall under the conntrack lock
+	// and swapping the pointer, and reloadFirewall carries it forward.
+	// Read unsynchronized on the data path: the preceding Firewall-pointer
+	// read pins the field's value for the duration of that call.
+	reporter events.Reporter
 
 	l *logrus.Logger
 }
@@ -224,12 +202,12 @@ func NewFirewall(l *logrus.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.D
 
 func NewFirewallFromConfig(l *logrus.Logger, cs *CertState, c *config.C) (*Firewall, error) {
 	certificate := cs.getCertificate(cert.Version2)
-	if certificate == nil { //todo if config.initiating_version is set to 1, and unsafe_networks differ, things will suck
+	if certificate == nil {
 		certificate = cs.getCertificate(cert.Version1)
 	}
 
 	if certificate == nil {
-		return nil, errors.New("no certificate available to reconfigure the firewall")
+		panic("No certificate available to reconfigure the firewall")
 	}
 
 	fw := NewFirewall(
@@ -238,6 +216,7 @@ func NewFirewallFromConfig(l *logrus.Logger, cs *CertState, c *config.C) (*Firew
 		c.GetDuration("firewall.conntrack.udp_timeout", time.Minute*3),
 		c.GetDuration("firewall.conntrack.default_timeout", time.Minute*10),
 		certificate,
+		//TODO: max_connections
 	)
 
 	fw.defaultLocalCIDRAny = c.GetBool("firewall.default_local_cidr_any", false)
@@ -260,7 +239,7 @@ func NewFirewallFromConfig(l *logrus.Logger, cs *CertState, c *config.C) (*Firew
 	case "drop":
 		fw.OutSendReject = false
 	default:
-		l.WithField("action", inboundAction).Warn("invalid firewall.outbound_action, defaulting to `drop`")
+		l.WithField("action", outboundAction).Warn("invalid firewall.outbound_action, defaulting to `drop`")
 		fw.OutSendReject = false
 	}
 
@@ -344,18 +323,6 @@ func (f *Firewall) GetRuleHashes() string {
 	return "SHA:" + f.GetRuleHash() + ",FNV:" + strconv.FormatUint(uint64(f.GetRuleHashFNV()), 10)
 }
 
-func (f *Firewall) SetSNATAddressFromInterface(i *Interface) {
-	//address-mutation-avoidance is done inside Interface, the firewall doesn't need to care
-	//todo should snatted conntracks get expired out? Probably not needed until if/when we allow reload
-	f.snatAddr = i.inside.SNATAddress().Addr()
-	f.unsafeIPv4Origin = i.inside.UnsafeIPv4OriginAddress().Addr()
-}
-
-func (f *Firewall) ShouldUnSNAT(fp *firewall.Packet) bool {
-	// f.snatAddr is only valid if we're a snat-capable router
-	return f.snatAddr.IsValid() && fp.RemoteAddr == f.snatAddr
-}
-
 func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw FirewallInterface) error {
 	var table string
 	if inbound {
@@ -456,224 +423,70 @@ var ErrInvalidRemoteIP = errors.New("remote address is not in remote certificate
 var ErrInvalidLocalIP = errors.New("local address is not in list of handled local addresses")
 var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
 
-func (f *Firewall) unSnat(data []byte, fp *firewall.Packet) netip.Addr {
+// Drop returns an error if the packet should be dropped, explaining why. It
-	c := f.peek(*fp) //unfortunately this needs to lock. Surely there's a better way.
+// returns nil if the packet should not be dropped.
-	if c == nil {
+func (f *Firewall) Drop(fp firewall.Packet, ctx firewall.PacketContext, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error {
-		return netip.Addr{}
+	// Check if we spoke to this tuple, if we did then allow this packet
-	}
+	if f.inConns(fp, h, caPool, localCache) {
-	if !c.snat.Valid() {
+		return nil
-		return netip.Addr{}
-	}
-	oldIP := netip.AddrPortFrom(f.snatAddr, fp.RemotePort)
-	rewritePacket(data, fp, oldIP, c.snat.Src, ipv4DestinationPosition, destinationPortOffset)
-	return c.snat.SrcVpnIp
-}
-
-func rewritePacket(data []byte, fp *firewall.Packet, oldIP netip.AddrPort, newIP netip.AddrPort, ipOffset int, portOffset int) {
-	//change address
-	copy(data[ipOffset:], newIP.Addr().AsSlice())
-	recalcIPv4Checksum(data, oldIP.Addr(), newIP.Addr())
-	ipHeaderLen := int(data[0]&0x0F) * 4
-
-	switch fp.Protocol {
-	case firewall.ProtoICMP:
-		binary.BigEndian.PutUint16(data[ipHeaderLen+4:ipHeaderLen+6], newIP.Port()) //we use the ID field as a "port" for ICMP
-		icmpCode := uint16(data[ipHeaderLen+1])                                     //todo not snatting on code yet (but Linux would)
-		recalcICMPv4Checksum(data, icmpCode, icmpCode, oldIP.Port(), newIP.Port())
-	case firewall.ProtoUDP:
-		dstport := ipHeaderLen + portOffset
-		binary.BigEndian.PutUint16(data[dstport:dstport+2], newIP.Port())
-		recalcUDPv4Checksum(data, oldIP, newIP)
-	case firewall.ProtoTCP:
-		dstport := ipHeaderLen + portOffset
-		binary.BigEndian.PutUint16(data[dstport:dstport+2], newIP.Port())
-		recalcTCPv4Checksum(data, oldIP, newIP)
-	}
-}
-
-func (f *Firewall) findUsableSNATPort(fp *firewall.Packet, c *conn) error {
-	const halfThePorts = 0x7fff
-	oldPort := fp.RemotePort
-	conntrack := f.Conntrack
-	conntrack.Lock()
-	defer conntrack.Unlock()
-	for numPortsChecked := 0; numPortsChecked < halfThePorts; numPortsChecked++ {
-		_, ok := conntrack.Conns[*fp]
-		if !ok {
-			//yay, we can use this port
-			//track the snatted flow with the same expiration as the unsnatted version
-			c.snat.SnatPort = fp.RemotePort
-			conntrack.Conns[*fp] = c
-			return nil
-		}
-		//increment and retry. There's probably better strategies out there
-		fp.RemotePort++
-		if fp.RemotePort < halfThePorts {
-			fp.RemotePort += halfThePorts // keep it ephemeral for now
-		}
 	}
 
-	//if we made it here, we failed
+	peerCert := h.ConnectionState.peerCert
-	fp.RemotePort = oldPort
-	return ErrCannotSNAT
-}
 
-func (f *Firewall) applySnat(data []byte, fp *firewall.Packet, c *conn, hostinfo *HostInfo) error {
+	// Make sure remote address matches nebula certificate, and determine how to treat it
-	if !f.snatAddr.IsValid() {
-		return ErrCannotSNAT
-	}
-	if f.snatAddr == fp.LocalAddr { //a packet that came from UDP (incoming) should never ever have our snat address on it
-		return ErrSNATAddressCollision
-	}
-	if c.snat.Valid() {
-		//old flow: make sure it came from the right place
-		if !slices.Contains(hostinfo.vpnAddrs, c.snat.SrcVpnIp) {
-			return ErrSNATIdentityMismatch
-		}
-		fp.RemoteAddr = f.snatAddr
-		fp.RemotePort = c.snat.SnatPort
-	} else if hostinfo.vpnAddrs[0].Is6() {
-		//we got a new flow
-		c.snat = &snatInfo{
-			Src:      netip.AddrPortFrom(fp.RemoteAddr, fp.RemotePort),
-			SrcVpnIp: hostinfo.vpnAddrs[0],
-		}
-		fp.RemoteAddr = f.snatAddr
-		//find a new port to use, if needed
-		err := f.findUsableSNATPort(fp, c)
-		if err != nil {
-			c.snat = nil
-			return err
-		}
-	} else {
-		return ErrCannotSNAT
-	}
-
-	newIP := netip.AddrPortFrom(f.snatAddr, c.snat.SnatPort)
-	rewritePacket(data, fp, c.snat.Src, newIP, ipv4SourcePosition, sourcePortOffset)
-	return nil
-}
-
-func (f *Firewall) identifyRemoteNetworkType(h *HostInfo, fp firewall.Packet) NetworkType {
 	if h.networks == nil {
 		// Simple case: Certificate has one address and no unsafe networks
-		if h.vpnAddrs[0] == fp.RemoteAddr {
+		if h.vpnAddrs[0] != fp.RemoteAddr {
-			return NetworkTypeVPN
+			f.metrics(incoming).droppedRemoteAddr.Inc(1)
-		} //else, fallthrough
+			f.reportDrop(incoming, events.DropInvalidRemoteIP, fp, ctx, peerCert)
-	} else if nwType, ok := h.networks.Lookup(fp.RemoteAddr); ok {
-		return nwType //will return NetworkTypeVPN or NetworkTypeUnsafe
-	}
-
-	//RemoteAddr not in our networks table
-	if f.snatAddr.IsValid() && fp.IsIPv4() && h.HasOnlyV6Addresses() {
-		return NetworkTypeUnverifiedSNATPeer
-	} else {
-		return NetworkTypeInvalidPeer
-	}
-}
-
-func (f *Firewall) allowRemoteNetworkType(nwType NetworkType, fp firewall.Packet) error {
-	switch nwType {
-	case NetworkTypeVPN:
-		return nil
-	case NetworkTypeInvalidPeer:
-		return ErrInvalidRemoteIP
-	case NetworkTypeVPNPeer:
-		//one day we might need a specialSnatMode case in here to handle routers with v4 addresses when we don't also have a v4 address?
-		return ErrPeerRejected // reject for now, one day this may have different FW rules
-	case NetworkTypeUnsafe:
-		return nil // nothing special, one day this may have different FW rules
-	case NetworkTypeUnverifiedSNATPeer:
-		if f.unsafeIPv4Origin.IsValid() && fp.LocalAddr == f.unsafeIPv4Origin {
-			return nil //the client case
-		}
-		if f.snatAddr.IsValid() {
-			if fp.RemoteAddr == f.snatAddr {
-				return ErrInvalidRemoteIP //we should never get a packet with our SNAT addr as the destination, or "from" our SNAT addr
-			}
-			return nil
-		} else {
 			return ErrInvalidRemoteIP
 		}
-	default:
+	} else {
-		return ErrUnknownNetworkType //should never happen
+		nwType, ok := h.networks.Lookup(fp.RemoteAddr)
+		if !ok {
+			f.metrics(incoming).droppedRemoteAddr.Inc(1)
+			f.reportDrop(incoming, events.DropInvalidRemoteIP, fp, ctx, peerCert)
+			return ErrInvalidRemoteIP
+		}
+		switch nwType {
+		case NetworkTypeVPN:
+			break // nothing special
+		case NetworkTypeVPNPeer:
+			f.metrics(incoming).droppedRemoteAddr.Inc(1)
+			f.reportDrop(incoming, events.DropPeerRejected, fp, ctx, peerCert)
+			return ErrPeerRejected // reject for now, one day this may have different FW rules
+		case NetworkTypeUnsafe:
+			break // nothing special, one day this may have different FW rules
+		default:
+			f.metrics(incoming).droppedRemoteAddr.Inc(1)
+			f.reportDrop(incoming, events.DropUnknownNetwork, fp, ctx, peerCert)
+			return ErrUnknownNetworkType //should never happen
+		}
 	}
-}
 
-func (f *Firewall) willingToHandleLocalAddr(incoming bool, fp firewall.Packet, remoteNwType NetworkType) error {
+	// Make sure we are supposed to be handling this local ip address
-	if f.routableNetworks.Contains(fp.LocalAddr) {
+	if !f.routableNetworks.Contains(fp.LocalAddr) {
-		return nil //easy, this should handle NetworkTypeVPN in all cases, and NetworkTypeUnsafe on the router side
+		f.metrics(incoming).droppedLocalAddr.Inc(1)
-	}
+		f.reportDrop(incoming, events.DropInvalidLocalIP, fp, ctx, peerCert)
-	if incoming { //at least for now, reject all traffic other than what we've already decided is locally routable
 		return ErrInvalidLocalIP
 	}
 
-	//below this line, all traffic is outgoing. Outgoing traffic to NetworkTypeUnsafe is not required to be considered inbound-routable
-	if remoteNwType == NetworkTypeUnsafe {
-		return nil
-	}
-
-	return ErrInvalidLocalIP
-}
-
-// Drop returns an error if the packet should be dropped, explaining why. It
-// returns nil if the packet should not be dropped.
-func (f *Firewall) Drop(fp firewall.Packet, pkt []byte, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error {
 	table := f.OutRules
 	if incoming {
 		table = f.InRules
 	}
 
-	snatmode := fp.IsIPv4() && h.HasOnlyV6Addresses() && f.snatAddr.IsValid()
-	if snatmode {
-		//if this is an IPv4 packet from a V6 only host, and we're configured to snat that kind of traffic, it must be snatted,
-		//so it can never be in the localcache, which lacks SNAT data
-		//nil out the pointer to avoid ever using it
-		localCache = nil
-	}
-
-	// Check if we spoke to this tuple, if we did then allow this packet
-	if localCache != nil {
-		if _, ok := localCache[fp]; ok {
-			return nil //packet matched the cache, we're not snatting, we can return early
-		}
-	}
-	c := f.inConns(fp, h, caPool, localCache)
-	if c != nil {
-		if incoming && snatmode {
-			return f.applySnat(pkt, &fp, c, h)
-		}
-		return nil
-	}
-
-	// Make sure remote address matches nebula certificate, and determine how to treat it
-	remoteNetworkType := f.identifyRemoteNetworkType(h, fp)
-	if err := f.allowRemoteNetworkType(remoteNetworkType, fp); err != nil {
-		f.metrics(incoming).droppedRemoteAddr.Inc(1)
-		return err
-	}
-
-	// Make sure we are supposed to be handling this local ip address
-	if err := f.willingToHandleLocalAddr(incoming, fp, remoteNetworkType); err != nil {
-		f.metrics(incoming).droppedLocalAddr.Inc(1)
-		return err
-	}
-
 	// We now know which firewall table to check against
-	if !table.match(fp, incoming, h.ConnectionState.peerCert, caPool) {
+	if !table.match(fp, incoming, peerCert, caPool) {
 		f.metrics(incoming).droppedNoRule.Inc(1)
+		f.reportDrop(incoming, events.DropNoMatchingRule, fp, ctx, peerCert)
 		return ErrNoMatchingRule
 	}
 
 	// We always want to conntrack since it is a faster operation
-	c = f.addConn(fp, incoming)
+	f.addConn(fp, ctx, incoming, peerCert)
 
-	if incoming && remoteNetworkType == NetworkTypeUnverifiedSNATPeer {
+	return nil
-		return f.applySnat(pkt, &fp, c, h)
-	} else {
-		//outgoing snat is handled before this function is called
-		return nil
-	}
 }
 
 func (f *Firewall) metrics(incoming bool) firewallMetrics {
@@ -690,6 +503,59 @@ func (f *Firewall) Destroy() {
 	//TODO: clean references if/when needed
 }
 
+func (f *Firewall) reportDrop(incoming bool, reason events.DropReason, fp firewall.Packet, ctx firewall.PacketContext, peerCert *cert.CachedCertificate) {
+	r := f.reporter
+	if r == nil {
+		return
+	}
+	r.ReportDrop(events.DropEvent{
+		Incoming:     incoming,
+		Reason:       reason,
+		Packet:       fp,
+		Context:      ctx,
+		PeerCert:     peerCert,
+		RulesVersion: f.rulesVersion,
+	})
+}
+
+func (f *Firewall) reportFlowCreate(incoming bool, fp firewall.Packet, ctx firewall.PacketContext, peerCert *cert.CachedCertificate) {
+	r := f.reporter
+	if r == nil {
+		return
+	}
+	r.ReportFlowCreate(events.FlowCreateEvent{
+		Incoming:     incoming,
+		Packet:       fp,
+		Context:      ctx,
+		PeerCert:     peerCert,
+		RulesVersion: f.rulesVersion,
+	})
+}
+
+func (f *Firewall) reportFlowEvict(incoming bool, fp firewall.Packet, rulesVersion uint16, expired bool) {
+	r := f.reporter
+	if r == nil {
+		return
+	}
+	r.ReportFlowEvict(events.FlowEvictEvent{
+		Incoming:     incoming,
+		Packet:       fp,
+		RulesVersion: rulesVersion,
+		Expired:      expired,
+	})
+}
+
+func (f *Firewall) reportRulesReload(oldVersion, newVersion uint16) {
+	r := f.reporter
+	if r == nil {
+		return
+	}
+	r.ReportRulesReload(events.RulesReloadEvent{
+		OldVersion: oldVersion,
+		NewVersion: newVersion,
+	})
+}
+
 func (f *Firewall) EmitStats() {
 	conntrack := f.Conntrack
 	conntrack.Lock()
@@ -700,14 +566,12 @@ func (f *Firewall) EmitStats() {
 	metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV()))
 }
 
-func (f *Firewall) peek(fp firewall.Packet) *conn {
+func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool {
-	f.Conntrack.Lock()
+	if localCache != nil {
-	c := f.Conntrack.Conns[fp]
+		if _, ok := localCache[fp]; ok {
-	f.Conntrack.Unlock()
+			return true
-	return c
+		}
-}
+	}
-
-func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) *conn {
 	conntrack := f.Conntrack
 	conntrack.Lock()
 
@@ -721,7 +585,7 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 
 	if !ok {
 		conntrack.Unlock()
-		return nil
+		return false
 	}
 
 	if c.rulesVersion != f.rulesVersion {
@@ -742,9 +606,11 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 					WithField("oldRulesVersion", c.rulesVersion).
 					Debugln("dropping old conntrack entry, does not match new ruleset")
 			}
+			oldRulesVersion := c.rulesVersion
 			delete(conntrack.Conns, fp)
+			f.reportFlowEvict(c.incoming, fp, oldRulesVersion, false)
 			conntrack.Unlock()
-			return nil
+			return false
 		}
 
 		if f.l.Level >= logrus.DebugLevel {
@@ -774,11 +640,12 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 		localCache[fp] = struct{}{}
 	}
 
-	return c
+	return true
 }
 
-func (f *Firewall) packetTimeout(fp firewall.Packet) time.Duration {
+func (f *Firewall) addConn(fp firewall.Packet, ctx firewall.PacketContext, incoming bool, peerCert *cert.CachedCertificate) {
 	var timeout time.Duration
+	c := &conn{}
 
 	switch fp.Protocol {
 	case firewall.ProtoTCP:
@@ -788,16 +655,11 @@ func (f *Firewall) packetTimeout(fp firewall.Packet) time.Duration {
 	default:
 		timeout = f.DefaultTimeout
 	}
-	return timeout
-}
 
-func (f *Firewall) addConn(fp firewall.Packet, incoming bool) *conn {
-	c := &conn{}
-
-	timeout := f.packetTimeout(fp)
 	conntrack := f.Conntrack
 	conntrack.Lock()
-	if _, ok := conntrack.Conns[fp]; !ok {
+	_, existing := conntrack.Conns[fp]
+	if !existing {
 		conntrack.TimerWheel.Advance(time.Now())
 		conntrack.TimerWheel.Add(fp, timeout)
 	}
@@ -809,8 +671,13 @@ func (f *Firewall) addConn(fp firewall.Packet, incoming bool) *conn {
 	c.Expires = time.Now().Add(timeout)
 	conntrack.Conns[fp] = c
 
+	// Report only when this represents a genuinely new flow. Fires under the
+	// conntrack lock so FlowCreate/FlowEvict events stay ordered relative to
+	// RulesReloadEvent, which also fires under this lock.
+	if !existing {
+		f.reportFlowCreate(incoming, fp, ctx, peerCert)
+	}
 	conntrack.Unlock()
-	return c
 }
 
 // Evict checks if a conntrack entry has expired, if so it is removed, if not it is re-added to the wheel
@@ -833,7 +700,10 @@ func (f *Firewall) evict(p firewall.Packet) {
 	}
 
 	// This conn is done
+	rulesVersion := t.rulesVersion
+	incoming := t.incoming
 	delete(conntrack.Conns, p)
+	f.reportFlowEvict(incoming, p, rulesVersion, true)
 }
 
 func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {

firewall/cache.go

@@ -1,6 +1,7 @@
 package firewall
 
 import (
+	"context"
 	"sync/atomic"
 	"time"
 
@@ -18,7 +19,7 @@ type ConntrackCacheTicker struct {
 	cache ConntrackCache
 }
 
-func NewConntrackCacheTicker(d time.Duration) *ConntrackCacheTicker {
+func NewConntrackCacheTicker(ctx context.Context, d time.Duration) *ConntrackCacheTicker {
 	if d == 0 {
 		return nil
 	}
@@ -27,15 +28,21 @@ func NewConntrackCacheTicker(d time.Duration) *ConntrackCacheTicker {
 		cache: ConntrackCache{},
 	}
 
-	go c.tick(d)
+	go c.tick(ctx, d)
 
 	return c
 }
 
-func (c *ConntrackCacheTicker) tick(d time.Duration) {
+func (c *ConntrackCacheTicker) tick(ctx context.Context, d time.Duration) {
+	t := time.NewTicker(d)
+	defer t.Stop()
 	for {
-		time.Sleep(d)
+		select {
-		c.cacheTick.Add(1)
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			c.cacheTick.Add(1)
+		}
 	}
 }
 

firewall/events/events.go

@@ -0,0 +1,102 @@
+// Package events defines the opt-in firewall event reporting interface.
+//
+// Nebula emits raw packet-level events (drops, flow creations, flow evictions,
+// rule reloads) and does no aggregation, counting, batching, rule-description,
+// transport, or timestamping. Embedders correlate events back to yaml rules
+// out of band and capture whatever clock they need themselves. All Report*
+// methods are invoked while nebula holds internal locks and must be
+// non-blocking.
+//
+// Events are passed to Report* methods by value. Implementations must not
+// take the address of a received event: doing so forces Go's escape
+// analysis to move the event to the heap and costs one allocation per call.
+// To forward an event, either copy its fields into the reporter's own
+// pooled record or send it through a value-typed channel (chan DropEvent,
+// not chan *DropEvent).
+package events
+
+import (
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/firewall"
+)
+
+type DropReason uint8
+
+const (
+	DropInvalidLocalIP DropReason = iota
+	DropInvalidRemoteIP
+	DropPeerRejected
+	DropUnknownNetwork
+	DropNoMatchingRule
+)
+
+func (r DropReason) String() string {
+	switch r {
+	case DropInvalidLocalIP:
+		return "invalid_local_ip"
+	case DropInvalidRemoteIP:
+		return "invalid_remote_ip"
+	case DropPeerRejected:
+		return "peer_rejected"
+	case DropUnknownNetwork:
+		return "unknown_network"
+	case DropNoMatchingRule:
+		return "no_matching_rule"
+	default:
+		return "unknown"
+	}
+}
+
+// DropEvent is emitted for every packet that fails the firewall check. Drops
+// are not aggregated; every drop produces one event.
+type DropEvent struct {
+	Incoming     bool
+	Reason       DropReason
+	Packet       firewall.Packet
+	Context      firewall.PacketContext
+	PeerCert     *cert.CachedCertificate
+	RulesVersion uint16
+}
+
+// FlowCreateEvent is emitted when a packet is allowed and a new conntrack
+// entry is created. Subsequent packets in the same flow do not re-emit.
+type FlowCreateEvent struct {
+	Incoming     bool
+	Packet       firewall.Packet
+	Context      firewall.PacketContext
+	PeerCert     *cert.CachedCertificate
+	RulesVersion uint16
+}
+
+// FlowEvictEvent is emitted when a conntrack entry is removed. Context is
+// not carried: timer-wheel eviction has no packet in hand, and reload
+// revalidation evicts the OLD flow rather than the triggering packet.
+// RulesVersion is the version under which the flow was originally allowed,
+// which may differ from the current firewall version.
+type FlowEvictEvent struct {
+	Incoming     bool
+	Packet       firewall.Packet
+	RulesVersion uint16
+	// Expired is true when eviction was due to conntrack timeout; false when
+	// the entry was removed because it failed re-validation after a reload.
+	Expired bool
+}
+
+// RulesReloadEvent is emitted once after each successful firewall reload.
+// Reporters that bucket state by RulesVersion should close the old bucket
+// and open a new one on receipt.
+type RulesReloadEvent struct {
+	OldVersion uint16
+	NewVersion uint16
+}
+
+// Reporter is the embedder-supplied sink for firewall events. Implementations
+// that want a timestamp should call time.Now() themselves at the top of the
+// method; nebula does not provide one. See the package doc for the
+// do-not-take-address rule.
+type Reporter interface {
+	ReportDrop(DropEvent)
+	ReportFlowCreate(FlowCreateEvent)
+	ReportFlowEvict(FlowEvictEvent)
+	ReportRulesReload(RulesReloadEvent)
+}

firewall/packet.go

@@ -31,8 +31,25 @@ type Packet struct {
 	Fragment   bool
 }
 
-func (fp *Packet) IsIPv4() bool {
+// PacketContext carries additional parsed details about a packet that are
-	return fp.LocalAddr.Is4() && fp.RemoteAddr.Is4()
+// useful for event reporting but deliberately kept out of Packet so Packet
+// can keep being used as a conntrack map key. Populated alongside Packet by
+// newPacket.
+//
+// Fields are interpreted based on Packet.Protocol:
+//   - ProtoTCP: TCPFlags is meaningful; ICMPType / ICMPCode are zero
+//   - ProtoICMP, ProtoICMPv6: ICMPType / ICMPCode are meaningful; TCPFlags is zero
+//   - ProtoUDP and others: only Length is meaningful
+type PacketContext struct {
+	// Length is the total IP packet length in bytes, including headers.
+	Length uint16
+	// TCPFlags is the flag byte from the TCP header (bits for FIN, SYN, RST,
+	// PSH, ACK, URG, ECE, CWR).
+	TCPFlags uint8
+	// ICMPType is the type field of the ICMP / ICMPv6 header.
+	ICMPType uint8
+	// ICMPCode is the code field of the ICMP / ICMPv6 header.
+	ICMPCode uint8
 }
 
 func (fp *Packet) Copy() *Packet {

firewall_events_test.go

@@ -0,0 +1,731 @@
+package nebula
+
+import (
+	"net"
+	"net/netip"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/gaissmai/bart"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/firewall"
+	"github.com/slackhq/nebula/firewall/events"
+	"github.com/slackhq/nebula/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// recordingReporter captures every event fired against it. Its methods take
+// the conntrack lock implicitly (via the firewall code path that invokes
+// them), so we synchronize accumulator mutations with a small mutex to keep
+// the race detector happy across goroutines in case a test introduces any.
+type recordingReporter struct {
+	mu      sync.Mutex
+	drops   []recordedDrop
+	creates []recordedCreate
+	evicts  []recordedEvict
+	reloads []recordedReload
+}
+
+type recordedDrop struct {
+	incoming     bool
+	reason       events.DropReason
+	remote       netip.Addr
+	local        netip.Addr
+	peerName     string
+	rulesVersion uint16
+	ctx          firewall.PacketContext
+}
+
+type recordedCreate struct {
+	incoming     bool
+	remote       netip.Addr
+	local        netip.Addr
+	peerName     string
+	rulesVersion uint16
+	ctx          firewall.PacketContext
+}
+
+type recordedEvict struct {
+	incoming     bool
+	remote       netip.Addr
+	local        netip.Addr
+	rulesVersion uint16
+	expired      bool
+}
+
+type recordedReload struct {
+	oldVersion uint16
+	newVersion uint16
+}
+
+func (r *recordingReporter) ReportDrop(e events.DropEvent) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	name := ""
+	if e.PeerCert != nil && e.PeerCert.Certificate != nil {
+		name = e.PeerCert.Certificate.Name()
+	}
+	r.drops = append(r.drops, recordedDrop{
+		incoming:     e.Incoming,
+		reason:       e.Reason,
+		remote:       e.Packet.RemoteAddr,
+		local:        e.Packet.LocalAddr,
+		peerName:     name,
+		rulesVersion: e.RulesVersion,
+		ctx:          e.Context,
+	})
+}
+
+func (r *recordingReporter) ReportFlowCreate(e events.FlowCreateEvent) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	name := ""
+	if e.PeerCert != nil && e.PeerCert.Certificate != nil {
+		name = e.PeerCert.Certificate.Name()
+	}
+	r.creates = append(r.creates, recordedCreate{
+		incoming:     e.Incoming,
+		remote:       e.Packet.RemoteAddr,
+		local:        e.Packet.LocalAddr,
+		peerName:     name,
+		rulesVersion: e.RulesVersion,
+		ctx:          e.Context,
+	})
+}
+
+func (r *recordingReporter) ReportFlowEvict(e events.FlowEvictEvent) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.evicts = append(r.evicts, recordedEvict{
+		incoming:     e.Incoming,
+		remote:       e.Packet.RemoteAddr,
+		local:        e.Packet.LocalAddr,
+		rulesVersion: e.RulesVersion,
+		expired:      e.Expired,
+	})
+}
+
+func (r *recordingReporter) ReportRulesReload(e events.RulesReloadEvent) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.reloads = append(r.reloads, recordedReload{
+		oldVersion: e.OldVersion,
+		newVersion: e.NewVersion,
+	})
+}
+
+// eventFixture builds a Firewall wired to a Control plus a packet/hostinfo
+// pair that a test can reuse. By default the ruleset allows the packet;
+// callers mutate fw / p / h as needed before invoking Drop.
+type eventFixture struct {
+	ctl *Control
+	fw  *Firewall
+	p   firewall.Packet
+	h   *HostInfo
+	cp  *cert.CAPool
+}
+
+func newEventFixture(t *testing.T) *eventFixture {
+	t.Helper()
+	l := test.NewLogger()
+
+	// myVpnNetworksTable covers our single peer address so buildNetworks takes
+	// the "simple case" path (h.networks stays nil); tests that want a populated
+	// BART table overwrite h.networks directly.
+	vpnNetworks := new(bart.Lite)
+	vpnNetworks.Insert(netip.MustParsePrefix("1.2.3.0/24"))
+
+	// Use the same cert for "peer" and "local" endpoints, matching the
+	// TestFirewall_Drop fixture style: LocalAddr == RemoteAddr == peer vpn addr.
+	c := &dummyCert{
+		name:     "host1",
+		networks: []netip.Prefix{netip.MustParsePrefix("1.2.3.4/24")},
+		groups:   []string{"default-group"},
+		issuer:   "signer-shasum",
+	}
+	h := &HostInfo{
+		ConnectionState: &ConnectionState{
+			peerCert: &cert.CachedCertificate{
+				Certificate:    c,
+				InvertedGroups: map[string]struct{}{"default-group": {}},
+			},
+		},
+		vpnAddrs: []netip.Addr{netip.MustParseAddr("1.2.3.4")},
+	}
+	h.buildNetworks(vpnNetworks, c)
+
+	fw := NewFirewall(l, time.Minute, time.Minute, time.Minute, c)
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
+	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
+
+	ctl := &Control{
+		f: &Interface{firewall: fw},
+		l: l,
+	}
+
+	return &eventFixture{
+		ctl: ctl,
+		fw:  fw,
+		p: firewall.Packet{
+			LocalAddr:  netip.MustParseAddr("1.2.3.4"),
+			RemoteAddr: netip.MustParseAddr("1.2.3.4"),
+			LocalPort:  10,
+			RemotePort: 90,
+			Protocol:   firewall.ProtoUDP,
+		},
+		h:  h,
+		cp: cert.NewCAPool(),
+	}
+}
+
+// firewall() returns the currently-installed firewall. Needed because
+// SetFirewallEventReporter replaces it via shallow-copy swap.
+func (f *eventFixture) firewall() *Firewall {
+	return f.ctl.f.firewall
+}
+
+func TestEvents_ReportDrop_InvalidRemoteIP(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// Packet to an address not in the cert's networks.
+	f.p.RemoteAddr = netip.MustParseAddr("9.9.9.9")
+	assert.Equal(t, ErrInvalidRemoteIP, f.firewall().Drop(f.p, firewall.PacketContext{}, false, f.h, f.cp, nil))
+
+	require.Len(t, r.drops, 1)
+	assert.Equal(t, events.DropInvalidRemoteIP, r.drops[0].reason)
+	assert.False(t, r.drops[0].incoming)
+	assert.Equal(t, "host1", r.drops[0].peerName)
+	assert.Empty(t, r.creates)
+	assert.Empty(t, r.evicts)
+}
+
+func TestEvents_ReportDrop_InvalidLocalIP(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// LocalAddr outside our routable networks.
+	f.p.LocalAddr = netip.MustParseAddr("9.9.9.9")
+	assert.Equal(t, ErrInvalidLocalIP, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+
+	require.Len(t, r.drops, 1)
+	assert.Equal(t, events.DropInvalidLocalIP, r.drops[0].reason)
+	assert.True(t, r.drops[0].incoming)
+}
+
+func TestEvents_ReportDrop_NoMatchingRule(t *testing.T) {
+	f := newEventFixture(t)
+	// Reset to a firewall with no matching rule.
+	l := test.NewLogger()
+	fw := NewFirewall(l, time.Minute, time.Minute, time.Minute, f.h.ConnectionState.peerCert.Certificate)
+	// Rule that won't match (group not in peer's groups).
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", ""))
+	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", ""))
+	f.ctl.f.firewall = fw
+
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	assert.Equal(t, ErrNoMatchingRule, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.drops, 1)
+	assert.Equal(t, events.DropNoMatchingRule, r.drops[0].reason)
+}
+
+func TestEvents_ReportDrop_PeerRejected(t *testing.T) {
+	f := newEventFixture(t)
+	// Re-classify the remote as VPNPeer so it triggers DropPeerRejected.
+	f.h.networks = new(bart.Table[NetworkType])
+	f.h.networks.Insert(netip.MustParsePrefix("1.2.3.0/24"), NetworkTypeVPNPeer)
+
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	assert.Equal(t, ErrPeerRejected, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.drops, 1)
+	assert.Equal(t, events.DropPeerRejected, r.drops[0].reason)
+}
+
+func TestEvents_ReportDrop_UnknownNetwork(t *testing.T) {
+	f := newEventFixture(t)
+	// Insert an unrecognized NetworkType value to hit the default branch.
+	f.h.networks = new(bart.Table[NetworkType])
+	f.h.networks.Insert(netip.MustParsePrefix("1.2.3.0/24"), NetworkTypeUnknown)
+
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	assert.Equal(t, ErrUnknownNetworkType, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.drops, 1)
+	assert.Equal(t, events.DropUnknownNetwork, r.drops[0].reason)
+}
+
+func TestEvents_ReportFlowCreate_OnceOnly(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// First allowed packet creates the conntrack entry.
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	// Second matching packet on the same tuple is short-circuited by conntrack
+	// and must not fire another FlowCreate.
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+
+	require.Len(t, r.creates, 1)
+	assert.True(t, r.creates[0].incoming)
+	assert.Equal(t, f.p.RemoteAddr, r.creates[0].remote)
+	assert.Empty(t, r.drops)
+}
+
+func TestEvents_ReportFlowEvict_OnReloadPurge(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// Create a flow under the current rules.
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.creates, 1)
+
+	// Simulate a reload that produces rules the existing flow no longer
+	// matches. Bump rulesVersion and replace InRules with an empty table so
+	// revalidation fails.
+	fw := f.firewall()
+	fw.Conntrack.Lock()
+	fw.rulesVersion++
+	fw.InRules = newFirewallTable()
+	fw.Conntrack.Unlock()
+
+	// Next packet triggers re-validation, which fails and evicts the entry.
+	err := fw.Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
+	assert.Equal(t, ErrNoMatchingRule, err)
+
+	require.Len(t, r.evicts, 1)
+	assert.False(t, r.evicts[0].expired, "evict from reload purge is not expiration")
+	assert.True(t, r.evicts[0].incoming)
+}
+
+func TestEvents_ReportFlowEvict_OnTimeout(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.creates, 1)
+
+	// Force expiration by rewinding the entry's deadline.
+	fw := f.firewall()
+	fw.Conntrack.Lock()
+	c := fw.Conntrack.Conns[f.p]
+	require.NotNil(t, c)
+	c.Expires = time.Now().Add(-time.Hour)
+	fw.evict(f.p)
+	fw.Conntrack.Unlock()
+
+	require.Len(t, r.evicts, 1)
+	assert.True(t, r.evicts[0].expired)
+}
+
+func TestEvents_SetNil_Clears(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.creates, 1)
+
+	f.ctl.SetFirewallEventReporter(nil)
+	resetConntrack(f.firewall())
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	// No second create should be recorded.
+	assert.Len(t, r.creates, 1)
+}
+
+func TestEvents_ReporterSurvivesSwap(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// Simulate a reload by swapping in a fresh Firewall that carries the
+	// reporter forward. Mirrors what reloadFirewall does with the shared
+	// conntrack pointer.
+	l := test.NewLogger()
+	oldFw := f.firewall()
+	newFw := NewFirewall(l, time.Minute, time.Minute, time.Minute, f.h.ConnectionState.peerCert.Certificate)
+	require.NoError(t, newFw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
+	require.NoError(t, newFw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
+	newFw.Conntrack = oldFw.Conntrack
+	newFw.rulesVersion = oldFw.rulesVersion + 1
+	newFw.reporter = oldFw.reporter
+	f.ctl.f.firewall = newFw
+	newFw.reportRulesReload(oldFw.rulesVersion, newFw.rulesVersion)
+
+	require.Len(t, r.reloads, 1)
+	assert.Equal(t, oldFw.rulesVersion, r.reloads[0].oldVersion)
+	assert.Equal(t, newFw.rulesVersion, r.reloads[0].newVersion)
+
+	// Events on the new firewall should still reach the same reporter.
+	require.NoError(t, newFw.Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.creates, 1)
+	assert.Equal(t, newFw.rulesVersion, r.creates[0].rulesVersion)
+}
+
+func TestEvents_InstallDoesNotMutateOldFirewall(t *testing.T) {
+	f := newEventFixture(t)
+	before := f.firewall()
+
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	after := f.firewall()
+	assert.NotSame(t, before, after, "SetFirewallEventReporter must replace the Firewall pointer")
+	assert.Nil(t, before.reporter, "the pre-install Firewall must remain untouched")
+	assert.NotNil(t, after.reporter)
+}
+
+// --- PacketContext parse tests --------------------------------------------
+
+func mustSerialize(t *testing.T, lrs ...gopacket.SerializableLayer) []byte {
+	t.Helper()
+	buf := gopacket.NewSerializeBuffer()
+	opt := gopacket.SerializeOptions{ComputeChecksums: false, FixLengths: true}
+	require.NoError(t, gopacket.SerializeLayers(buf, opt, lrs...))
+	return buf.Bytes()
+}
+
+func TestPacketContext_IPv4_TCPFlags(t *testing.T) {
+	ip := &layers.IPv4{
+		Version: 4, TTL: 64, Protocol: layers.IPProtocolTCP,
+		SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
+	}
+	tcp := &layers.TCP{SrcPort: 1234, DstPort: 80, SYN: true, ACK: true}
+	require.NoError(t, tcp.SetNetworkLayerForChecksum(ip))
+	data := mustSerialize(t, ip, tcp, gopacket.Payload([]byte("hello")))
+
+	var fp firewall.Packet
+	var ctx firewall.PacketContext
+	require.NoError(t, newPacket(data, true, &fp, &ctx))
+
+	assert.Equal(t, uint8(firewall.ProtoTCP), fp.Protocol)
+	// SYN (0x02) + ACK (0x10) = 0x12
+	assert.Equal(t, uint8(0x12), ctx.TCPFlags)
+	assert.Equal(t, uint16(len(data)), ctx.Length)
+	assert.Equal(t, uint8(0), ctx.ICMPType)
+	assert.Equal(t, uint8(0), ctx.ICMPCode)
+}
+
+func TestPacketContext_IPv4_ICMPTypeCode(t *testing.T) {
+	ip := &layers.IPv4{
+		Version: 4, TTL: 64, Protocol: layers.IPProtocolICMPv4,
+		SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
+	}
+	// Destination Unreachable, code 3 (port unreachable)
+	icmp := &layers.ICMPv4{
+		TypeCode: layers.CreateICMPv4TypeCode(layers.ICMPv4TypeDestinationUnreachable, layers.ICMPv4CodePort),
+	}
+	data := mustSerialize(t, ip, icmp, gopacket.Payload([]byte{0, 0, 0, 0}))
+
+	var fp firewall.Packet
+	var ctx firewall.PacketContext
+	require.NoError(t, newPacket(data, true, &fp, &ctx))
+
+	assert.Equal(t, uint8(firewall.ProtoICMP), fp.Protocol)
+	assert.Equal(t, uint8(layers.ICMPv4TypeDestinationUnreachable), ctx.ICMPType)
+	assert.Equal(t, uint8(layers.ICMPv4CodePort), ctx.ICMPCode)
+	assert.Equal(t, uint16(len(data)), ctx.Length)
+	assert.Equal(t, uint8(0), ctx.TCPFlags)
+}
+
+func TestPacketContext_IPv4_UDPLengthOnly(t *testing.T) {
+	ip := &layers.IPv4{
+		Version: 4, TTL: 64, Protocol: layers.IPProtocolUDP,
+		SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
+	}
+	udp := &layers.UDP{SrcPort: 1234, DstPort: 53}
+	require.NoError(t, udp.SetNetworkLayerForChecksum(ip))
+	data := mustSerialize(t, ip, udp, gopacket.Payload([]byte("query")))
+
+	var fp firewall.Packet
+	var ctx firewall.PacketContext
+	require.NoError(t, newPacket(data, true, &fp, &ctx))
+
+	assert.Equal(t, uint8(firewall.ProtoUDP), fp.Protocol)
+	assert.Equal(t, uint16(len(data)), ctx.Length)
+	assert.Zero(t, ctx.TCPFlags)
+	assert.Zero(t, ctx.ICMPType)
+	assert.Zero(t, ctx.ICMPCode)
+}
+
+func TestPacketContext_IPv6_TCPFlags(t *testing.T) {
+	ip := &layers.IPv6{
+		Version: 6, HopLimit: 64, NextHeader: layers.IPProtocolTCP,
+		SrcIP: net.ParseIP("fd00::1"), DstIP: net.ParseIP("fd00::2"),
+	}
+	tcp := &layers.TCP{SrcPort: 1234, DstPort: 443, FIN: true, ACK: true}
+	require.NoError(t, tcp.SetNetworkLayerForChecksum(ip))
+	data := mustSerialize(t, ip, tcp, gopacket.Payload([]byte("bye")))
+
+	var fp firewall.Packet
+	var ctx firewall.PacketContext
+	require.NoError(t, newPacket(data, true, &fp, &ctx))
+
+	assert.Equal(t, uint8(firewall.ProtoTCP), fp.Protocol)
+	// FIN (0x01) + ACK (0x10) = 0x11
+	assert.Equal(t, uint8(0x11), ctx.TCPFlags)
+	assert.Equal(t, uint16(len(data)), ctx.Length)
+}
+
+func TestPacketContext_IPv6_ICMPv6TypeCode(t *testing.T) {
+	ip := &layers.IPv6{
+		Version: 6, HopLimit: 64, NextHeader: layers.IPProtocolICMPv6,
+		SrcIP: net.ParseIP("fd00::1"), DstIP: net.ParseIP("fd00::2"),
+	}
+	icmp := &layers.ICMPv6{
+		TypeCode: layers.CreateICMPv6TypeCode(layers.ICMPv6TypeDestinationUnreachable, layers.ICMPv6CodePortUnreachable),
+	}
+	require.NoError(t, icmp.SetNetworkLayerForChecksum(ip))
+	data := mustSerialize(t, ip, icmp, gopacket.Payload([]byte{0, 0, 0, 0, 0, 0, 0, 0}))
+
+	var fp firewall.Packet
+	var ctx firewall.PacketContext
+	require.NoError(t, newPacket(data, true, &fp, &ctx))
+
+	assert.Equal(t, uint8(firewall.ProtoICMPv6), fp.Protocol)
+	assert.Equal(t, uint8(layers.ICMPv6TypeDestinationUnreachable), ctx.ICMPType)
+	assert.Equal(t, uint8(layers.ICMPv6CodePortUnreachable), ctx.ICMPCode)
+	assert.Equal(t, uint16(len(data)), ctx.Length)
+}
+
+// TestPacketContext_NilOK confirms a nil context pointer is accepted by
+// newPacket (the hot path may elect not to pass one).
+func TestPacketContext_NilOK(t *testing.T) {
+	ip := &layers.IPv4{
+		Version: 4, TTL: 64, Protocol: layers.IPProtocolUDP,
+		SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
+	}
+	udp := &layers.UDP{SrcPort: 1, DstPort: 2}
+	require.NoError(t, udp.SetNetworkLayerForChecksum(ip))
+	data := mustSerialize(t, ip, udp)
+
+	var fp firewall.Packet
+	require.NoError(t, newPacket(data, true, &fp, nil))
+}
+
+// TestPacketContext_FlowCreateCarriesContext exercises the full Drop -> addConn
+// -> ReportFlowCreate path with a realistic TCP packet and confirms the
+// context makes it into the reporter.
+func TestPacketContext_FlowCreateCarriesContext(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// Hand-construct a matching TCP packet.
+	ctx := firewall.PacketContext{Length: 1500, TCPFlags: 0x12}
+	p := f.p
+	p.Protocol = firewall.ProtoTCP
+	require.NoError(t, f.firewall().Drop(p, ctx, true, f.h, f.cp, nil))
+
+	require.Len(t, r.creates, 1)
+	assert.Equal(t, uint16(1500), r.creates[0].ctx.Length)
+	assert.Equal(t, uint8(0x12), r.creates[0].ctx.TCPFlags)
+}
+
+// --- benchmarks ------------------------------------------------------------
+
+// noopReporter is the cheapest possible reporter. Methods discard the event.
+type noopReporter struct{}
+
+func (noopReporter) ReportDrop(events.DropEvent)             {}
+func (noopReporter) ReportFlowCreate(events.FlowCreateEvent) {}
+func (noopReporter) ReportFlowEvict(events.FlowEvictEvent)   {}
+func (noopReporter) ReportRulesReload(events.RulesReloadEvent) {
+}
+
+// bufferedReporter demonstrates a realistic zero-alloc reporter: each event
+// is forwarded to a value-typed channel. The channel send is a memcpy into
+// the channel's pre-allocated ring buffer -- no heap traffic. A background
+// goroutine would drain these; the bench skips draining to keep the report
+// path pure.
+type bufferedReporter struct {
+	drops  chan events.DropEvent
+	flows  chan events.FlowCreateEvent
+	evicts chan events.FlowEvictEvent
+}
+
+func newBufferedReporter(cap int) *bufferedReporter {
+	return &bufferedReporter{
+		drops:  make(chan events.DropEvent, cap),
+		flows:  make(chan events.FlowCreateEvent, cap),
+		evicts: make(chan events.FlowEvictEvent, cap),
+	}
+}
+
+func (r *bufferedReporter) ReportDrop(e events.DropEvent) {
+	select {
+	case r.drops <- e:
+	default:
+	}
+}
+
+func (r *bufferedReporter) ReportFlowCreate(e events.FlowCreateEvent) {
+	select {
+	case r.flows <- e:
+	default:
+	}
+}
+
+func (r *bufferedReporter) ReportFlowEvict(e events.FlowEvictEvent) {
+	select {
+	case r.evicts <- e:
+	default:
+	}
+}
+
+func (r *bufferedReporter) ReportRulesReload(events.RulesReloadEvent) {}
+
+// pointerReporter is the anti-pattern: it takes the address of the incoming
+// event struct, which forces the callee-side copy onto the heap. Kept for
+// comparison so we can see the alloc cost an unwary reporter would incur.
+type pointerReporter struct {
+	last *events.DropEvent
+}
+
+func (r *pointerReporter) ReportDrop(e events.DropEvent) {
+	r.last = &e
+}
+
+func (r *pointerReporter) ReportFlowCreate(events.FlowCreateEvent) {}
+func (r *pointerReporter) ReportFlowEvict(events.FlowEvictEvent)   {}
+func (r *pointerReporter) ReportRulesReload(events.RulesReloadEvent) {
+}
+
+func newBenchFixture(b *testing.B) *eventFixture {
+	b.Helper()
+	l := test.NewLogger()
+
+	vpnNetworks := new(bart.Lite)
+	vpnNetworks.Insert(netip.MustParsePrefix("1.2.3.0/24"))
+
+	c := &dummyCert{
+		name:     "host1",
+		networks: []netip.Prefix{netip.MustParsePrefix("1.2.3.4/24")},
+		groups:   []string{"default-group"},
+		issuer:   "signer-shasum",
+	}
+	h := &HostInfo{
+		ConnectionState: &ConnectionState{
+			peerCert: &cert.CachedCertificate{
+				Certificate:    c,
+				InvertedGroups: map[string]struct{}{"default-group": {}},
+			},
+		},
+		vpnAddrs: []netip.Addr{netip.MustParseAddr("1.2.3.4")},
+	}
+	h.buildNetworks(vpnNetworks, c)
+
+	fw := NewFirewall(l, time.Minute, time.Minute, time.Minute, c)
+	// Inbound rule that matches our packet; outbound has no match so we can
+	// also benchmark the no-rule drop path.
+	if err := fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""); err != nil {
+		b.Fatal(err)
+	}
+
+	ctl := &Control{f: &Interface{firewall: fw}, l: l}
+	return &eventFixture{
+		ctl: ctl,
+		fw:  fw,
+		p: firewall.Packet{
+			LocalAddr:  netip.MustParseAddr("1.2.3.4"),
+			RemoteAddr: netip.MustParseAddr("1.2.3.4"),
+			LocalPort:  10,
+			RemotePort: 90,
+			Protocol:   firewall.ProtoUDP,
+		},
+		h:  h,
+		cp: cert.NewCAPool(),
+	}
+}
+
+// BenchmarkFirewallDropPath measures the cost of Firewall.Drop on a packet
+// that reaches the no-matching-rule branch (the longest drop path). Compare
+// reporter shapes:
+//
+//	nilReporter      -- no reporter installed (feature cost when off)
+//	noopReporter     -- reporter installed, methods discard args (minimum on-cost)
+//	bufferedReporter -- realistic zero-alloc reporter: value-typed channels
+//	pointerReporter  -- anti-pattern that takes &composite-literal (allocates)
+func BenchmarkFirewallDropPath(b *testing.B) {
+	run := func(b *testing.B, install func(*Control)) {
+		f := newBenchFixture(b)
+		install(f.ctl)
+		b.ReportAllocs()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			_ = f.firewall().Drop(f.p, firewall.PacketContext{}, false, f.h, f.cp, nil)
+		}
+	}
+
+	b.Run("nilReporter", func(b *testing.B) { run(b, func(*Control) {}) })
+	b.Run("noopReporter", func(b *testing.B) {
+		run(b, func(c *Control) { c.SetFirewallEventReporter(noopReporter{}) })
+	})
+	b.Run("bufferedReporter", func(b *testing.B) {
+		run(b, func(c *Control) { c.SetFirewallEventReporter(newBufferedReporter(1024)) })
+	})
+	b.Run("pointerReporter", func(b *testing.B) {
+		run(b, func(c *Control) { c.SetFirewallEventReporter(&pointerReporter{}) })
+	})
+}
+
+// BenchmarkConntrackCreate measures Firewall.Drop for an allowed inbound
+// packet on a fresh conntrack (so addConn fires each iteration).
+func BenchmarkConntrackCreate(b *testing.B) {
+	run := func(b *testing.B, install func(*Control)) {
+		f := newBenchFixture(b)
+		install(f.ctl)
+		b.ReportAllocs()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			resetConntrack(f.firewall())
+			_ = f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
+		}
+	}
+	b.Run("nilReporter", func(b *testing.B) { run(b, func(*Control) {}) })
+	b.Run("noopReporter", func(b *testing.B) {
+		run(b, func(c *Control) { c.SetFirewallEventReporter(noopReporter{}) })
+	})
+	b.Run("bufferedReporter", func(b *testing.B) {
+		run(b, func(c *Control) { c.SetFirewallEventReporter(newBufferedReporter(1024)) })
+	})
+}
+
+// BenchmarkConntrackHit measures the hot path where a flow is already in
+// conntrack and short-circuits rule evaluation. The reporter slot is checked
+// only on create/evict, so this bench should show the reporter having zero
+// impact regardless of install state.
+func BenchmarkConntrackHit(b *testing.B) {
+	b.Run("nilReporter", func(b *testing.B) {
+		f := newBenchFixture(b)
+		// Prime conntrack.
+		require.NoError(b, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+		b.ReportAllocs()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			_ = f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
+		}
+	})
+
+	b.Run("noopReporter", func(b *testing.B) {
+		f := newBenchFixture(b)
+		f.ctl.SetFirewallEventReporter(noopReporter{})
+		require.NoError(b, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+		b.ReportAllocs()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			_ = f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
+		}
+	})
+}

firewall_test.go

@@ -213,44 +213,44 @@ func TestFirewall_Drop(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// Drop outbound
-	assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, nil, false, &h, cp, nil))
+	assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 	// Allow inbound
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, nil, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 	// Allow outbound because conntrack
-	require.NoError(t, fw.Drop(p, nil, false, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 
 	// test remote mismatch
 	oldRemote := p.RemoteAddr
 	p.RemoteAddr = netip.MustParseAddr("1.2.3.10")
-	assert.Equal(t, fw.Drop(p, nil, false, &h, cp, nil), ErrInvalidRemoteIP)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil), ErrInvalidRemoteIP)
 	p.RemoteAddr = oldRemote
 
 	// ensure signer doesn't get in the way of group checks
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum"))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad"))
-	assert.Equal(t, fw.Drop(p, nil, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caSha doesn't drop on match
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad"))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum"))
-	require.NoError(t, fw.Drop(p, nil, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 
 	// ensure ca name doesn't get in the way of group checks
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", ""))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", ""))
-	assert.Equal(t, fw.Drop(p, nil, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caName doesn't drop on match
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", ""))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", ""))
-	require.NoError(t, fw.Drop(p, nil, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 }
 
 func TestFirewall_DropV6(t *testing.T) {
@@ -292,44 +292,44 @@ func TestFirewall_DropV6(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// Drop outbound
-	assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, nil, false, &h, cp, nil))
+	assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 	// Allow inbound
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, nil, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 	// Allow outbound because conntrack
-	require.NoError(t, fw.Drop(p, nil, false, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 
 	// test remote mismatch
 	oldRemote := p.RemoteAddr
 	p.RemoteAddr = netip.MustParseAddr("fd12::56")
-	assert.Equal(t, fw.Drop(p, nil, false, &h, cp, nil), ErrInvalidRemoteIP)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil), ErrInvalidRemoteIP)
 	p.RemoteAddr = oldRemote
 
 	// ensure signer doesn't get in the way of group checks
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum"))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad"))
-	assert.Equal(t, fw.Drop(p, nil, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caSha doesn't drop on match
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad"))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum"))
-	require.NoError(t, fw.Drop(p, nil, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 
 	// ensure ca name doesn't get in the way of group checks
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", ""))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", ""))
-	assert.Equal(t, fw.Drop(p, nil, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caName doesn't drop on match
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", ""))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", ""))
-	require.NoError(t, fw.Drop(p, nil, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 }
 
 func BenchmarkFirewallTable_match(b *testing.B) {
@@ -537,10 +537,10 @@ func TestFirewall_Drop2(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// h1/c1 lacks the proper groups
-	require.ErrorIs(t, fw.Drop(p, nil, true, &h1, cp, nil), ErrNoMatchingRule)
+	require.ErrorIs(t, fw.Drop(p, firewall.PacketContext{}, true, &h1, cp, nil), ErrNoMatchingRule)
 	// c has the proper groups
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, nil, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 }
 
 func TestFirewall_Drop3(t *testing.T) {
@@ -618,18 +618,18 @@ func TestFirewall_Drop3(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// c1 should pass because host match
-	require.NoError(t, fw.Drop(p, nil, true, &h1, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h1, cp, nil))
 	// c2 should pass because ca sha match
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, nil, true, &h2, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h2, cp, nil))
 	// c3 should fail because no match
 	resetConntrack(fw)
-	assert.Equal(t, fw.Drop(p, nil, true, &h3, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h3, cp, nil), ErrNoMatchingRule)
 
 	// Test a remote address match
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "1.2.3.4/24", "", "", ""))
-	require.NoError(t, fw.Drop(p, nil, true, &h1, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h1, cp, nil))
 }
 
 func TestFirewall_Drop3V6(t *testing.T) {
@@ -667,7 +667,7 @@ func TestFirewall_Drop3V6(t *testing.T) {
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
 	cp := cert.NewCAPool()
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "fd12::34/120", "", "", ""))
-	require.NoError(t, fw.Drop(p, nil, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 }
 
 func TestFirewall_DropConntrackReload(t *testing.T) {
@@ -709,12 +709,12 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// Drop outbound
-	assert.Equal(t, fw.Drop(p, nil, false, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 	// Allow inbound
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, nil, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 	// Allow outbound because conntrack
-	require.NoError(t, fw.Drop(p, nil, false, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 
 	oldFw := fw
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
@@ -723,7 +723,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	fw.rulesVersion = oldFw.rulesVersion + 1
 
 	// Allow outbound because conntrack and new rules allow port 10
-	require.NoError(t, fw.Drop(p, nil, false, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 
 	oldFw = fw
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
@@ -732,7 +732,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	fw.rulesVersion = oldFw.rulesVersion + 1
 
 	// Drop outbound because conntrack doesn't match new ruleset
-	assert.Equal(t, fw.Drop(p, nil, false, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 }
 
 func TestFirewall_ICMPPortBehavior(t *testing.T) {
@@ -778,12 +778,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0
 			p.RemotePort = 0
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, nil, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			require.NoError(t, fw.Drop(*p, nil, true, &h, cp, nil))
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil))
 			//now also allow outbound
-			require.NoError(t, fw.Drop(*p, nil, false, &h, cp, nil))
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil))
 		})
 
 		t.Run("nonzero ports", func(t *testing.T) {
@@ -791,12 +791,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0xabcd
 			p.RemotePort = 0x1234
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, nil, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			require.NoError(t, fw.Drop(*p, nil, true, &h, cp, nil))
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil))
 			//now also allow outbound
-			require.NoError(t, fw.Drop(*p, nil, false, &h, cp, nil))
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil))
 		})
 	})
 
@@ -808,12 +808,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0
 			p.RemotePort = 0
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, nil, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			assert.Equal(t, fw.Drop(*p, nil, true, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 			//now also allow outbound
-			assert.Equal(t, fw.Drop(*p, nil, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 		})
 
 		t.Run("nonzero ports, still blocked", func(t *testing.T) {
@@ -821,12 +821,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0xabcd
 			p.RemotePort = 0x1234
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, nil, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			assert.Equal(t, fw.Drop(*p, nil, true, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 			//now also allow outbound
-			assert.Equal(t, fw.Drop(*p, nil, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 		})
 
 		t.Run("nonzero, matching ports, still blocked", func(t *testing.T) {
@@ -834,12 +834,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 80
 			p.RemotePort = 80
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, nil, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			assert.Equal(t, fw.Drop(*p, nil, true, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 			//now also allow outbound
-			assert.Equal(t, fw.Drop(*p, nil, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 		})
 	})
 	t.Run("Any proto, any port", func(t *testing.T) {
@@ -851,12 +851,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0
 			p.RemotePort = 0
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, nil, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			require.NoError(t, fw.Drop(*p, nil, true, &h, cp, nil))
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil))
 			//now also allow outbound
-			require.NoError(t, fw.Drop(*p, nil, false, &h, cp, nil))
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil))
 		})
 
 		t.Run("nonzero ports, allowed", func(t *testing.T) {
@@ -865,15 +865,15 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0xabcd
 			p.RemotePort = 0x1234
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, nil, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			require.NoError(t, fw.Drop(*p, nil, true, &h, cp, nil))
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil))
 			//now also allow outbound
-			require.NoError(t, fw.Drop(*p, nil, false, &h, cp, nil))
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil))
 			//different ID is blocked
 			p.RemotePort++
-			require.Equal(t, fw.Drop(*p, nil, false, &h, cp, nil), ErrNoMatchingRule)
+			require.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 		})
 	})
 
@@ -922,7 +922,7 @@ func TestFirewall_DropIPSpoofing(t *testing.T) {
 		Protocol:   firewall.ProtoUDP,
 		Fragment:   false,
 	}
-	assert.Equal(t, fw.Drop(p, nil, true, &h1, cp, nil), ErrInvalidRemoteIP)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h1, cp, nil), ErrInvalidRemoteIP)
 }
 
 func BenchmarkLookup(b *testing.B) {
@@ -1042,7 +1042,7 @@ func TestNewFirewallFromConfig(t *testing.T) {
 	l := test.NewLogger()
 	// Test a bad rule definition
 	c := &dummyCert{}
-	cs, err := newCertState(l, cert.Version2, nil, c, false, cert.Curve_CURVE25519, nil)
+	cs, err := newCertState(cert.Version2, nil, c, false, cert.Curve_CURVE25519, nil)
 	require.NoError(t, err)
 
 	conf := config.NewC(l)
@@ -1336,7 +1336,7 @@ func (c *testcase) Test(t *testing.T, fw *Firewall) {
 	t.Helper()
 	cp := cert.NewCAPool()
 	resetConntrack(fw)
-	err := fw.Drop(c.p, nil, true, c.h, cp, nil)
+	err := fw.Drop(c.p, firewall.PacketContext{}, true, c.h, cp, nil)
 	if c.err == nil {
 		require.NoError(t, err, "failed to not drop remote address %s", c.p.RemoteAddr)
 	} else {
@@ -1344,7 +1344,7 @@ func (c *testcase) Test(t *testing.T, fw *Firewall) {
 	}
 }
 
-func buildHostinfo(setup testsetup, theirPrefixes ...netip.Prefix) *HostInfo {
+func buildTestCase(setup testsetup, err error, theirPrefixes ...netip.Prefix) testcase {
 	c1 := dummyCert{
 		name:     "host1",
 		networks: theirPrefixes,
@@ -1364,11 +1364,6 @@ func buildHostinfo(setup testsetup, theirPrefixes ...netip.Prefix) *HostInfo {
 		h.vpnAddrs[i] = theirPrefixes[i].Addr()
 	}
 	h.buildNetworks(setup.myVpnNetworksTable, &c1)
-	return &h
-}
-
-func buildTestCase(setup testsetup, err error, theirPrefixes ...netip.Prefix) testcase {
-	h := buildHostinfo(setup, theirPrefixes...)
 	p := firewall.Packet{
 		LocalAddr:  setup.c.Networks()[0].Addr(), //todo?
 		RemoteAddr: theirPrefixes[0].Addr(),
@@ -1378,9 +1373,9 @@ func buildTestCase(setup testsetup, err error, theirPrefixes ...netip.Prefix) te
 		Fragment:   false,
 	}
 	return testcase{
-		h:   h,
+		h:   &h,
 		p:   p,
-		c:   h.ConnectionState.peerCert.Certificate,
+		c:   &c1,
 		err: err,
 	}
 }
@@ -1402,19 +1397,6 @@ func newSetup(t *testing.T, l *logrus.Logger, myPrefixes ...netip.Prefix) testse
 	return newSetupFromCert(t, l, c)
 }
 
-func newSnatSetup(t *testing.T, l *logrus.Logger, myPrefix netip.Prefix, snatAddr netip.Addr) testsetup {
-	c := dummyCert{
-		name:     "me",
-		networks: []netip.Prefix{myPrefix},
-		groups:   []string{"default-group"},
-		issuer:   "signer-shasum",
-	}
-
-	out := newSetupFromCert(t, l, c)
-	out.fw.snatAddr = snatAddr
-	return out
-}
-
 func newSetupFromCert(t *testing.T, l *logrus.Logger, c dummyCert) testsetup {
 	myVpnNetworksTable := new(bart.Lite)
 	for _, prefix := range c.Networks() {
@@ -1550,59 +1532,3 @@ func resetConntrack(fw *Firewall) {
 	fw.Conntrack.Conns = map[firewall.Packet]*conn{}
 	fw.Conntrack.Unlock()
 }
-
-func TestFirewall_SNAT(t *testing.T) {
-	t.Parallel()
-	l := test.NewLogger()
-	ob := &bytes.Buffer{}
-	l.SetOutput(ob)
-	cp := cert.NewCAPool()
-	myPrefix := netip.MustParsePrefix("1.1.1.1/8")
-
-	MyCert := dummyCert{
-		name:     "me",
-		networks: []netip.Prefix{myPrefix},
-		groups:   []string{"default-group"},
-		issuer:   "signer-shasum",
-	}
-
-	theirPrefix := netip.MustParsePrefix("1.2.2.2/8")
-	snatAddr := netip.MustParseAddr("169.254.55.96")
-	t.Run("allow inbound all matching", func(t *testing.T) {
-		t.Parallel()
-		myCert := MyCert.Copy()
-		setup := newSnatSetup(t, l, myPrefix, snatAddr)
-		fw := NewFirewall(l, time.Second, time.Minute, time.Hour, myCert)
-		require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
-		resetConntrack(setup.fw)
-		h := buildHostinfo(setup, theirPrefix)
-		p := firewall.Packet{
-			LocalAddr:  setup.c.Networks()[0].Addr(), //todo?
-			RemoteAddr: h.vpnAddrs[0],
-			LocalPort:  10,
-			RemotePort: 90,
-			Protocol:   firewall.ProtoUDP,
-			Fragment:   false,
-		}
-		require.NoError(t, setup.fw.Drop(p, nil, true, h, cp, nil))
-	})
-	//t.Run("allow inbound unsafe route", func(t *testing.T) {
-	//	t.Parallel()
-	//	unsafePrefix := netip.MustParsePrefix("192.168.0.0/24")
-	//	c := dummyCert{
-	//		name:           "me",
-	//		networks:       []netip.Prefix{myPrefix},
-	//		unsafeNetworks: []netip.Prefix{unsafePrefix},
-	//		groups:         []string{"default-group"},
-	//		issuer:         "signer-shasum",
-	//	}
-	//	unsafeSetup := newSetupFromCert(t, l, c)
-	//	tc := buildTestCase(unsafeSetup, nil, twoPrefixes...)
-	//	tc.p.LocalAddr = netip.MustParseAddr("192.168.0.3")
-	//	tc.err = ErrNoMatchingRule
-	//	tc.Test(t, unsafeSetup.fw) //should hit firewall and bounce off
-	//	require.NoError(t, unsafeSetup.fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", unsafePrefix.String(), "", ""))
-	//	tc.err = nil
-	//	tc.Test(t, unsafeSetup.fw) //should pass
-	//})
-}

go.mod

@@ -1,6 +1,6 @@
 module github.com/slackhq/nebula
 
-go 1.25
+go 1.25.0
 
 require (
 	dario.cat/mergo v1.0.2
@@ -13,8 +13,8 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/gopacket v1.1.19
 	github.com/kardianos/service v1.2.4
-	github.com/miekg/dns v1.1.70
+	github.com/miekg/dns v1.1.72
-	github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b
+	github.com/miekg/pkcs11 v1.1.2
 	github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f
 	github.com/prometheus/client_golang v1.23.2
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
@@ -24,15 +24,15 @@ require (
 	github.com/stretchr/testify v1.11.1
 	github.com/vishvananda/netlink v1.3.1
 	go.yaml.in/yaml/v3 v3.0.4
-	golang.org/x/crypto v0.47.0
+	golang.org/x/crypto v0.50.0
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
-	golang.org/x/net v0.49.0
+	golang.org/x/net v0.52.0
-	golang.org/x/sync v0.19.0
+	golang.org/x/sync v0.20.0
-	golang.org/x/sys v0.40.0
+	golang.org/x/sys v0.43.0
-	golang.org/x/term v0.39.0
+	golang.org/x/term v0.42.0
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
 	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
-	golang.zx2c4.com/wireguard/windows v0.5.3
+	golang.zx2c4.com/wireguard/windows v0.6.1
 	google.golang.org/protobuf v1.36.11
 	gopkg.in/yaml.v3 v3.0.1
 	gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
@@ -50,7 +50,7 @@ require (
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/vishvananda/netns v0.0.5 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
-	golang.org/x/mod v0.31.0 // indirect
+	golang.org/x/mod v0.34.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.40.0 // indirect
+	golang.org/x/tools v0.43.0 // indirect
 )

go.sum

@@ -85,10 +85,10 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/miekg/dns v1.1.70 h1:DZ4u2AV35VJxdD9Fo9fIWm119BsQL5cZU1cQ9s0LkqA=
+github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
-github.com/miekg/dns v1.1.70/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
+github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
-github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b h1:J/AzCvg5z0Hn1rqZUJjpbzALUmkKX0Zwbc/i4fw7Sfk=
+github.com/miekg/pkcs11 v1.1.2 h1:/VxmeAX5qU6Q3EwafypogwWbYryHFmF2RpkJmw3m4MQ=
-github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/miekg/pkcs11 v1.1.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -164,16 +164,16 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.47.0 h1:V6e3FRj+n4dbpw86FJ8Fv7XVOql7TEwpHapKoMJ/GO8=
+golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
-golang.org/x/crypto v0.47.0/go.mod h1:ff3Y9VzzKbwSSEzWqJsJVBnWmRwRSHt/6Op5n9bQc4A=
+golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.31.0 h1:HaW9xtz0+kOcWKwli0ZXy79Ix+UW/vOfmWI5QVd2tgI=
+golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
-golang.org/x/mod v0.31.0/go.mod h1:43JraMp9cGx1Rx3AqioxrbrhNsLl2l/iNAvuBkrezpg=
+golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -184,8 +184,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.49.0 h1:eeHFmOGUTtaaPSGNmjBKpbng9MulQsJURQUAfUwY++o=
+golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
-golang.org/x/net v0.49.0/go.mod h1:/ysNB2EvaqvesRkuLAyjI1ycPZlQHM3q01F02UY/MV8=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -193,8 +193,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -210,11 +210,11 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ=
+golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
-golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.39.0 h1:RclSuaJf32jOqZz74CkPA9qFuVTX7vhLlpfj/IGWlqY=
+golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
-golang.org/x/term v0.39.0/go.mod h1:yxzUCTP/U+FzoxfdKmLaA0RV1WgE0VY7hXBwKtY/4ww=
+golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -225,8 +225,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.40.0 h1:yLkxfA+Qnul4cs9QA3KnlFu0lVmd8JJfoq+E41uSutA=
+golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
-golang.org/x/tools v0.40.0/go.mod h1:Ik/tzLRlbscWpqqMRjyWYDisX8bG13FrdXp3o4Sr9lc=
+golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -235,8 +235,8 @@ golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeu
 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
 golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo=
 golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
-golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE=
+golang.zx2c4.com/wireguard/windows v0.6.1 h1:XMaKojH1Hs/raMrmnir4n35nTvzvWj7NmSYzHn2F4qU=
-golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI=
+golang.zx2c4.com/wireguard/windows v0.6.1/go.mod h1:04aqInu5GYuTFvMuDw/rKBAF7mHrltW/3rekpfbbZDM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

hostmap.go

@@ -224,9 +224,6 @@ const (
 	NetworkTypeVPNPeer
 	// NetworkTypeUnsafe is a network from Certificate.UnsafeNetworks()
 	NetworkTypeUnsafe
-	// NetworkTypeUnverifiedSNATPeer is used to indicate traffic we're willing to route, but never deliver to a NetworkTypeVPN
-	NetworkTypeUnverifiedSNATPeer
-	NetworkTypeInvalidPeer
 )
 
 type HostInfo struct {
@@ -280,15 +277,6 @@ type HostInfo struct {
 	lastUsed time.Time
 }
 
-func (i *HostInfo) HasOnlyV6Addresses() bool {
-	for _, vpnIp := range i.vpnAddrs {
-		if !vpnIp.Is6() {
-			return false
-		}
-	}
-	return true
-}
-
 type ViaSender struct {
 	UdpAddr   netip.AddrPort
 	relayHI   *HostInfo // relayHI is the host info object of the relay
@@ -616,9 +604,9 @@ func (hm *HostMap) queryVpnAddr(vpnIp netip.Addr, promoteIfce *Interface) *HostI
 // unlockedAddHostInfo assumes you have a write-lock and will add a hostinfo object to the hostmap Indexes and RemoteIndexes maps.
 // If an entry exists for the Hosts table (vpnIp -> hostinfo) then the provided hostinfo will be made primary
 func (hm *HostMap) unlockedAddHostInfo(hostinfo *HostInfo, f *Interface) {
-	if f.serveDns {
+	if f.dnsServer != nil {
 		remoteCert := hostinfo.ConnectionState.peerCert
-		dnsR.Add(remoteCert.Certificate.Name()+".", hostinfo.vpnAddrs)
+		f.dnsServer.Add(remoteCert.Certificate.Name()+".", hostinfo.vpnAddrs)
 	}
 	for _, addr := range hostinfo.vpnAddrs {
 		hm.unlockedInnerAddHostInfo(addr, hostinfo, f)

inside.go

@@ -11,8 +11,8 @@ import (
 	"github.com/slackhq/nebula/routing"
 )
 
-func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache) {
+func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, fwCtx *firewall.PacketContext, nb, out []byte, q int, localCache firewall.ConntrackCache) {
-	err := newPacket(packet, false, fwPacket)
+	err := newPacket(packet, false, fwPacket, fwCtx)
 	if err != nil {
 		if f.l.Level >= logrus.DebugLevel {
 			f.l.WithField("packet", packet).Debugf("Error while validating outbound packet: %s", err)
@@ -48,7 +48,9 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 		return
 	}
 
-	hostinfo, ready := f.getHostinfo(packet, fwPacket)
+	hostinfo, ready := f.getOrHandshakeConsiderRouting(fwPacket, func(hh *HandshakeHostInfo) {
+		hh.cachePacket(f.l, header.Message, 0, packet, f.sendMessageNow, f.cachedPacketMetrics)
+	})
 
 	if hostinfo == nil {
 		f.rejectInside(packet, out, q)
@@ -64,9 +66,10 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 		return
 	}
 
-	dropReason := f.firewall.Drop(*fwPacket, packet, false, hostinfo, f.pki.GetCAPool(), localCache)
+	dropReason := f.firewall.Drop(*fwPacket, *fwCtx, false, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason == nil {
 		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
+
 	} else {
 		f.rejectInside(packet, out, q)
 		if f.l.Level >= logrus.DebugLevel {
@@ -78,26 +81,6 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 	}
 }
 
-func (f *Interface) getHostinfo(packet []byte, fwPacket *firewall.Packet) (*HostInfo, bool) {
-	if f.firewall.ShouldUnSNAT(fwPacket) {
-		//unsnat packet re-writing also happens here, would be nice to not,
-		//but we need to do the unsnat lookup to find the hostinfo so we can run the firewall checks
-		destVpnAddr := f.firewall.unSnat(packet, fwPacket)
-		if destVpnAddr.IsValid() {
-			//because this was a snatted packet, we know it has an on-overlay destination, so no routing should be required.
-			return f.getOrHandshakeNoRouting(destVpnAddr, func(hh *HandshakeHostInfo) {
-				hh.cachePacket(f.l, header.Message, 0, packet, f.sendMessageNow, f.cachedPacketMetrics)
-			})
-		} else {
-			return nil, false
-		}
-	} else { //if we didn't need to unsnat
-		return f.getOrHandshakeConsiderRouting(fwPacket, func(hh *HandshakeHostInfo) {
-			hh.cachePacket(f.l, header.Message, 0, packet, f.sendMessageNow, f.cachedPacketMetrics)
-		})
-	}
-}
-
 func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
 	if !f.firewall.InSendReject {
 		return
@@ -228,14 +211,15 @@ func (f *Interface) getOrHandshakeConsiderRouting(fwPacket *firewall.Packet, cac
 
 func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte) {
 	fp := &firewall.Packet{}
-	err := newPacket(p, false, fp)
+	ctx := &firewall.PacketContext{}
+	err := newPacket(p, false, fp, ctx)
 	if err != nil {
 		f.l.Warnf("error while parsing outgoing packet for firewall check; %v", err)
 		return
 	}
 
 	// check if packet is in outbound fw rules
-	dropReason := f.firewall.Drop(*fp, p, false, hostinfo, f.pki.GetCAPool(), nil)
+	dropReason := f.firewall.Drop(*fp, *ctx, false, hostinfo, f.pki.GetCAPool(), nil)
 	if dropReason != nil {
 		if f.l.Level >= logrus.DebugLevel {
 			f.l.WithField("fwPacket", fp).

interface.go

@@ -6,8 +6,7 @@ import (
 	"fmt"
 	"io"
 	"net/netip"
-	"os"
+	"sync"
-	"runtime"
 	"sync/atomic"
 	"time"
 
@@ -30,7 +29,7 @@ type InterfaceConfig struct {
 	pki                *PKI
 	Cipher             string
 	Firewall           *Firewall
-	ServeDns           bool
+	DnsServer          *dnsServer
 	HandshakeManager   *HandshakeManager
 	lightHouse         *LightHouse
 	connectionManager  *connectionManager
@@ -58,7 +57,7 @@ type Interface struct {
 	firewall              *Firewall
 	connectionManager     *connectionManager
 	handshakeManager      *HandshakeManager
-	serveDns              bool
+	dnsServer             *dnsServer
 	createTime            time.Time
 	lightHouse            *LightHouse
 	myBroadcastAddrsTable *bart.Lite
@@ -86,8 +85,16 @@ type Interface struct {
 
 	conntrackCacheTimeout time.Duration
 
+	ctx     context.Context
 	writers []udp.Conn
 	readers []io.ReadWriteCloser
+	wg      sync.WaitGroup
+
+	// fatalErr holds the first unexpected reader error that caused shutdown.
+	// nil means "no fatal error" (yet)
+	fatalErr atomic.Pointer[error]
+	// triggerShutdown is a function that will be run exactly once, when onFatal swaps something non-nil into fatalErr
+	triggerShutdown func()
 
 	metricHandshakes    metrics.Histogram
 	messageMetrics      *MessageMetrics
@@ -164,12 +171,13 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 
 	cs := c.pki.getCertState()
 	ifce := &Interface{
+		ctx:                   ctx,
 		pki:                   c.pki,
 		hostMap:               c.HostMap,
 		outside:               c.Outside,
 		inside:                c.Inside,
 		firewall:              c.Firewall,
-		serveDns:              c.ServeDns,
+		dnsServer:             c.DnsServer,
 		handshakeManager:      c.HandshakeManager,
 		createTime:            time.Now(),
 		lightHouse:            c.lightHouse,
@@ -210,7 +218,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 // activate creates the interface on the host. After the interface is created, any
 // other services that want to bind listeners to its IP may do so successfully. However,
 // the interface isn't going to process anything until run() is called.
-func (f *Interface) activate() {
+func (f *Interface) activate() error {
 	// actually turn on tun dev
 
 	addr, err := f.outside.LocalAddr()
@@ -238,34 +246,58 @@ func (f *Interface) activate() {
 		if i > 0 {
 			reader, err = f.inside.NewMultiQueueReader()
 			if err != nil {
-				f.l.Fatal(err)
+				return err
 			}
 		}
 		f.readers[i] = reader
 	}
 
-	if err := f.inside.Activate(); err != nil {
+	f.wg.Add(1) // for us to wait on Close() to return
+	if err = f.inside.Activate(); err != nil {
+		f.wg.Done()
 		f.inside.Close()
-		f.l.Fatal(err)
+		return err
 	}
-	f.firewall.SetSNATAddressFromInterface(f)
+
+	return nil
 }
 
-func (f *Interface) run() {
+func (f *Interface) run() (func() error, error) {
 	// Launch n queues to read packets from udp
 	for i := 0; i < f.routines; i++ {
-		go f.listenOut(i)
+		f.wg.Go(func() {
+			f.listenOut(i)
+		})
 	}
 
 	// Launch n queues to read packets from tun dev
 	for i := 0; i < f.routines; i++ {
-		go f.listenIn(f.readers[i], i)
+		f.wg.Go(func() {
+			f.listenIn(f.readers[i], i)
+		})
+	}
+
+	return func() error {
+		f.wg.Wait()
+		if e := f.fatalErr.Load(); e != nil {
+			return *e
+		}
+		return nil
+	}, nil
+}
+
+// onFatal stores the first fatal reader error, and calls triggerShutdown if it was the first one
+func (f *Interface) onFatal(err error) {
+	swapped := f.fatalErr.CompareAndSwap(nil, &err)
+	if !swapped {
+		return
+	}
+	if f.triggerShutdown != nil {
+		f.triggerShutdown()
 	}
 }
 
 func (f *Interface) listenOut(i int) {
-	runtime.LockOSThread()
-
 	var li udp.Conn
 	if i > 0 {
 		li = f.writers[i]
@@ -273,42 +305,49 @@ func (f *Interface) listenOut(i int) {
 		li = f.outside
 	}
 
-	ctCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
+	ctCache := firewall.NewConntrackCacheTicker(f.ctx, f.conntrackCacheTimeout)
 	lhh := f.lightHouse.NewRequestHandler()
 	plaintext := make([]byte, udp.MTU)
 	h := &header.H{}
 	fwPacket := &firewall.Packet{}
+	fwCtx := &firewall.PacketContext{}
 	nb := make([]byte, 12, 12)
 
-	li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) {
+	err := li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) {
-		f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, plaintext[:0], payload, h, fwPacket, lhh, nb, i, ctCache.Get(f.l))
+		f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, plaintext[:0], payload, h, fwPacket, fwCtx, lhh, nb, i, ctCache.Get(f.l))
 	})
+
+	if err != nil && !f.closed.Load() {
+		f.l.WithError(err).Error("Error while reading inbound packet, closing")
+		f.onFatal(err)
+	}
+
+	f.l.Debugf("underlay reader %v is done", i)
 }
 
 func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
-	runtime.LockOSThread()
-
 	packet := make([]byte, mtu)
 	out := make([]byte, mtu)
 	fwPacket := &firewall.Packet{}
+	fwCtx := &firewall.PacketContext{}
 	nb := make([]byte, 12, 12)
 
-	conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
+	conntrackCache := firewall.NewConntrackCacheTicker(f.ctx, f.conntrackCacheTimeout)
 
 	for {
 		n, err := reader.Read(packet)
 		if err != nil {
-			if errors.Is(err, os.ErrClosed) && f.closed.Load() {
+			if !f.closed.Load() {
-				return
+				f.l.WithError(err).WithField("reader", i).Error("Error while reading outbound packet, closing")
+				f.onFatal(err)
 			}
-
+			break
-			f.l.WithError(err).Error("Error while reading outbound packet")
-			// This only seems to happen when something fatal happens to the fd, so exit.
-			os.Exit(2)
 		}
 
-		f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get(f.l))
+		f.consumeInsidePacket(packet[:n], fwPacket, fwCtx, nb, out, i, conntrackCache.Get(f.l))
 	}
+
+	f.l.Debugf("overlay reader %v is done", i)
 }
 
 func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
@@ -345,7 +384,6 @@ func (f *Interface) reloadFirewall(c *config.C) {
 		f.l.WithError(err).Error("Error while creating firewall during reload")
 		return
 	}
-	fw.SetSNATAddressFromInterface(f)
 
 	oldFw := f.firewall
 	conntrack := oldFw.Conntrack
@@ -364,8 +402,15 @@ func (f *Interface) reloadFirewall(c *config.C) {
 		fw.Conntrack = conntrack
 	}
 
+	fw.reporter = oldFw.reporter
+
 	f.firewall = fw
 
+	// Fire ReportRulesReload under the conntrack lock so the reporter cannot
+	// observe a FlowCreate/FlowEvict for the new rulesVersion before it
+	// observes the reload marker. Report* must be non-blocking.
+	fw.reportRulesReload(oldFw.rulesVersion, fw.rulesVersion)
+
 	oldFw.Destroy()
 	f.l.WithField("firewallHashes", fw.GetRuleHashes()).
 		WithField("oldFirewallHashes", oldFw.GetRuleHashes()).
@@ -484,23 +529,23 @@ func (f *Interface) GetCertState() *CertState {
 }
 
 func (f *Interface) Close() error {
+	var errs []error
 	f.closed.Store(true)
 
-	for _, u := range f.writers {
+	// Release the udp readers
+	for i, u := range f.writers {
 		err := u.Close()
 		if err != nil {
-			f.l.WithError(err).Error("Error while closing udp socket")
+			f.l.WithError(err).WithField("writer", i).Error("Error while closing udp socket")
-		}
+			errs = append(errs, err)
-	}
-	for i, r := range f.readers {
-		if i == 0 {
-			continue // f.readers[0] is f.inside, which we want to save for last
-		}
-		if err := r.Close(); err != nil {
-			f.l.WithError(err).Error("Error while closing tun reader")
 		}
 	}
 
-	// Release the tun device
+	// Release the tun device (closing the tun also closes all readers)
-	return f.inside.Close()
+	closeErr := f.inside.Close()
+	if closeErr != nil {
+		errs = append(errs, closeErr)
+	}
+	f.wg.Done()
+	return errors.Join(errs...)
 }

main.go

@@ -131,8 +131,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 			deviceFactory = overlay.NewDeviceFromConfig
 		}
 
-		cs := pki.getCertState()
+		tun, err = deviceFactory(c, l, pki.getCertState().myVpnNetworks, routines)
-		tun, err = deviceFactory(c, l, cs.myVpnNetworks, cs.GetDefaultCertificate().UnsafeNetworks(), routines)
 		if err != nil {
 			return nil, util.ContextualizeIfNeeded("Failed to get a tun/tap device", err)
 		}
@@ -216,13 +215,9 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	handshakeManager := NewHandshakeManager(l, hostMap, lightHouse, udpConns[0], handshakeConfig)
 	lightHouse.handshakeTrigger = handshakeManager.trigger
 
-	serveDns := false
+	ds, err := newDnsServerFromConfig(ctx, l, pki.getCertState(), hostMap, c)
-	if c.GetBool("lighthouse.serve_dns", false) {
+	if err != nil {
-		if c.GetBool("lighthouse.am_lighthouse", false) {
+		l.WithError(err).Warn("Failed to start DNS responder")
-			serveDns = true
-		} else {
-			l.Warn("DNS server refusing to run because this host is not a lighthouse.")
-		}
 	}
 
 	ifConfig := &InterfaceConfig{
@@ -231,7 +226,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		Outside:               udpConns[0],
 		pki:                   pki,
 		Firewall:              fw,
-		ServeDns:              serveDns,
+		DnsServer:             ds,
 		HandshakeManager:      handshakeManager,
 		connectionManager:     connManager,
 		lightHouse:            lightHouse,
@@ -281,23 +276,17 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 
 	attachCommands(l, c, ssh, ifce)
 
-	// Start DNS server last to allow using the nebula IP as lighthouse.dns.host
-	var dnsStart func()
-	if lightHouse.amLighthouse && serveDns {
-		l.Debugln("Starting dns server")
-		dnsStart = dnsMain(l, pki.getCertState(), hostMap, c)
-	}
-
 	return &Control{
-		ifce,
+		state:                  StateReady,
-		l,
+		f:                      ifce,
-		ctx,
+		l:                      l,
-		cancel,
+		ctx:                    ctx,
-		sshStart,
+		cancel:                 cancel,
-		statsStart,
+		sshStart:               sshStart,
-		dnsStart,
+		statsStart:             statsStart,
-		lightHouse.StartUpdateWorker,
+		dnsStart:               ds.Start,
-		connManager.Start,
+		lighthouseStart:        lightHouse.StartUpdateWorker,
+		connectionManagerStart: connManager.Start,
 	}, nil
 }
 

outside.go

@@ -19,7 +19,7 @@ const (
 	minFwPacketLen = 4
 )
 
-func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
+func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, fwCtx *firewall.PacketContext, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
 	err := h.Parse(packet)
 	if err != nil {
 		// Hole punch packets are 0 or 1 byte big, so lets ignore printing those errors
@@ -60,7 +60,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 
 		switch h.Subtype {
 		case header.MessageNone:
-			if !f.decryptToTun(hostinfo, h.MessageCounter, out, packet, fwPacket, nb, q, localCache) {
+			if !f.decryptToTun(hostinfo, h.MessageCounter, out, packet, fwPacket, fwCtx, nb, q, localCache) {
 				return
 			}
 		case header.MessageRelay:
@@ -102,7 +102,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 					relay:     relay,
 					IsRelayed: true,
 				}
-				f.readOutsidePackets(via, out[:0], signedPayload, h, fwPacket, lhf, nb, q, localCache)
+				f.readOutsidePackets(via, out[:0], signedPayload, h, fwPacket, fwCtx, lhf, nb, q, localCache)
 				return
 			case ForwardingType:
 				// Find the target HostInfo relay object
@@ -190,6 +190,13 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 		if !f.handleEncrypted(ci, via, h) {
 			return
 		}
+		_, err = f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
+		if err != nil {
+			hostinfo.logger(f.l).WithError(err).WithField("from", via).
+				WithField("packet", packet).
+				Error("Failed to decrypt CloseTunnel packet")
+			return
+		}
 
 		hostinfo.logger(f.l).WithField("from", via).
 			Info("Close tunnel received, tearing down.")
@@ -288,7 +295,10 @@ var (
 )
 
 // newPacket validates and parses the interesting bits for the firewall out of the ip and sub protocol headers
-func newPacket(data []byte, incoming bool, fp *firewall.Packet) error {
+func newPacket(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.PacketContext) error {
+	if ctx != nil {
+		*ctx = firewall.PacketContext{}
+	}
 	if len(data) < 1 {
 		return ErrPacketTooShort
 	}
@@ -296,14 +306,14 @@ func newPacket(data []byte, incoming bool, fp *firewall.Packet) error {
 	version := int((data[0] >> 4) & 0x0f)
 	switch version {
 	case ipv4.Version:
-		return parseV4(data, incoming, fp)
+		return parseV4(data, incoming, fp, ctx)
 	case ipv6.Version:
-		return parseV6(data, incoming, fp)
+		return parseV6(data, incoming, fp, ctx)
 	}
 	return ErrUnknownIPVersion
 }
 
-func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
+func parseV6(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.PacketContext) error {
 	dataLen := len(data)
 	if dataLen < ipv6.HeaderLen {
 		return ErrIPv6PacketTooShort
@@ -348,6 +358,11 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 				fp.RemotePort = 0
 			}
 			fp.Fragment = false
+			if ctx != nil {
+				ctx.Length = binary.BigEndian.Uint16(data[4:6]) + uint16(ipv6.HeaderLen)
+				ctx.ICMPType = data[offset]
+				ctx.ICMPCode = data[offset+1]
+			}
 			return nil
 
 		case layers.IPProtocolTCP, layers.IPProtocolUDP:
@@ -365,6 +380,12 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 			}
 
 			fp.Fragment = false
+			if ctx != nil {
+				ctx.Length = binary.BigEndian.Uint16(data[4:6]) + uint16(ipv6.HeaderLen)
+				if proto == layers.IPProtocolTCP && dataLen >= offset+14 {
+					ctx.TCPFlags = data[offset+13]
+				}
+			}
 			return nil
 
 		case layers.IPProtocolIPv6Fragment:
@@ -416,7 +437,7 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 	return ErrIPv6CouldNotFindPayload
 }
 
-func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
+func parseV4(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.PacketContext) error {
 	// Do we at least have an ipv4 header worth of data?
 	if len(data) < ipv4.HeaderLen {
 		return ErrIPv4PacketTooShort
@@ -473,6 +494,21 @@ func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
 		fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
 	}
 
+	if ctx != nil {
+		ctx.Length = binary.BigEndian.Uint16(data[2:4])
+		if !fp.Fragment {
+			switch fp.Protocol {
+			case firewall.ProtoICMP:
+				ctx.ICMPType = data[ihl]
+				ctx.ICMPCode = data[ihl+1]
+			case firewall.ProtoTCP:
+				if len(data) >= ihl+14 {
+					ctx.TCPFlags = data[ihl+13]
+				}
+			}
+		}
+	}
+
 	return nil
 }
 
@@ -492,7 +528,7 @@ func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, out []byte, packet []
 	return out, nil
 }
 
-func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache) bool {
+func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out []byte, packet []byte, fwPacket *firewall.Packet, fwCtx *firewall.PacketContext, nb []byte, q int, localCache firewall.ConntrackCache) bool {
 	var err error
 
 	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
@@ -501,7 +537,7 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
 		return false
 	}
 
-	err = newPacket(out, true, fwPacket)
+	err = newPacket(out, true, fwPacket, fwCtx)
 	if err != nil {
 		hostinfo.logger(f.l).WithError(err).WithField("packet", out).
 			Warnf("Error while validating inbound packet")
@@ -514,7 +550,7 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
 		return false
 	}
 
-	dropReason := f.firewall.Drop(*fwPacket, out, true, hostinfo, f.pki.GetCAPool(), localCache)
+	dropReason := f.firewall.Drop(*fwPacket, *fwCtx, true, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason != nil {
 		// NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore
 		// This gives us a buffer to build the reject packet in

outside_test.go

@@ -20,13 +20,13 @@ func Test_newPacket(t *testing.T) {
 	p := &firewall.Packet{}
 
 	// length fails
-	err := newPacket([]byte{}, true, p)
+	err := newPacket([]byte{}, true, p, nil)
 	require.ErrorIs(t, err, ErrPacketTooShort)
 
-	err = newPacket([]byte{0x40}, true, p)
+	err = newPacket([]byte{0x40}, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv4PacketTooShort)
 
-	err = newPacket([]byte{0x60}, true, p)
+	err = newPacket([]byte{0x60}, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 
 	// length fail with ip options
@@ -39,15 +39,15 @@ func Test_newPacket(t *testing.T) {
 	}
 
 	b, _ := h.Marshal()
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv4InvalidHeaderLength)
 
 	// not an ipv4 packet
-	err = newPacket([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p)
+	err = newPacket([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p, nil)
 	require.ErrorIs(t, err, ErrUnknownIPVersion)
 
 	// invalid ihl
-	err = newPacket([]byte{4<<4 | (8 >> 2 & 0x0f), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p)
+	err = newPacket([]byte{4<<4 | (8 >> 2 & 0x0f), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv4InvalidHeaderLength)
 
 	// account for variable ip header length - incoming
@@ -62,7 +62,7 @@ func Test_newPacket(t *testing.T) {
 
 	b, _ = h.Marshal()
 	b = append(b, []byte{0, 3, 0, 4}...)
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol)
@@ -84,7 +84,7 @@ func Test_newPacket(t *testing.T) {
 
 	b, _ = h.Marshal()
 	b = append(b, []byte{0, 5, 0, 6}...)
-	err = newPacket(b, false, p)
+	err = newPacket(b, false, p, nil)
 
 	require.NoError(t, err)
 	assert.Equal(t, uint8(2), p.Protocol)
@@ -114,7 +114,7 @@ func Test_newPacket_v6(t *testing.T) {
 	err := gopacket.SerializeLayers(buffer, opt, &ip)
 	require.NoError(t, err)
 
-	err = newPacket(buffer.Bytes(), true, p)
+	err = newPacket(buffer.Bytes(), true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 
 	// A v6 packet with a hop-by-hop extension
@@ -148,12 +148,12 @@ func Test_newPacket_v6(t *testing.T) {
 
 	// A full IPv6 header and 1 byte in the first extension, but missing
 	// the length byte.
-	err = newPacket(buffer.Bytes()[:41], true, p)
+	err = newPacket(buffer.Bytes()[:41], true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 
 	// A full IPv6 header plus 1 full extension, but only 1 byte of the
 	// next layer, missing length byte
-	err = newPacket(buffer.Bytes()[:49], true, p)
+	err = newPacket(buffer.Bytes()[:49], true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 	err = nil
 
@@ -173,7 +173,7 @@ func Test_newPacket_v6(t *testing.T) {
 
 	buffer.Clear()
 	require.NoError(t, gopacket.SerializeLayers(buffer, opt, &ip, &icmp))
-	require.Error(t, newPacket(buffer.Bytes(), true, p))
+	require.Error(t, newPacket(buffer.Bytes(), true, p, nil))
 
 	buffer.Clear()
 	echo := layers.ICMPv6Echo{
@@ -181,7 +181,7 @@ func Test_newPacket_v6(t *testing.T) {
 		SeqNumber:  1234,
 	}
 	require.NoError(t, gopacket.SerializeLayers(buffer, opt, &ip, &icmp, &echo))
-	require.NoError(t, newPacket(buffer.Bytes(), true, p))
+	require.NoError(t, newPacket(buffer.Bytes(), true, p, nil))
 	assert.Equal(t, uint8(layers.IPProtocolICMPv6), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr)
@@ -192,7 +192,7 @@ func Test_newPacket_v6(t *testing.T) {
 	// A good ESP packet
 	b := buffer.Bytes()
 	b[6] = byte(layers.IPProtocolESP)
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(layers.IPProtocolESP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -204,7 +204,7 @@ func Test_newPacket_v6(t *testing.T) {
 	// A good None packet
 	b = buffer.Bytes()
 	b[6] = byte(layers.IPProtocolNoNextHeader)
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(layers.IPProtocolNoNextHeader), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -216,7 +216,7 @@ func Test_newPacket_v6(t *testing.T) {
 	// An unknown protocol packet
 	b = buffer.Bytes()
 	b[6] = 255 // 255 is a reserved protocol number
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 
 	// A good UDP packet
@@ -243,7 +243,7 @@ func Test_newPacket_v6(t *testing.T) {
 	b = buffer.Bytes()
 
 	// incoming
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -253,7 +253,7 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// outgoing
-	err = newPacket(b, false, p)
+	err = newPacket(b, false, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
@@ -263,14 +263,14 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// Too short UDP packet
-	err = newPacket(b[:len(b)-10], false, p) // pull off the last 10 bytes
+	err = newPacket(b[:len(b)-10], false, p, nil) // pull off the last 10 bytes
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 
 	// A good TCP packet
 	b[6] = byte(layers.IPProtocolTCP)
 
 	// incoming
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -280,7 +280,7 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// outgoing
-	err = newPacket(b, false, p)
+	err = newPacket(b, false, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
@@ -290,7 +290,7 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// Too short TCP packet
-	err = newPacket(b[:len(b)-10], false, p) // pull off the last 10 bytes
+	err = newPacket(b[:len(b)-10], false, p, nil) // pull off the last 10 bytes
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 
 	// A good UDP packet with an AH header
@@ -325,7 +325,7 @@ func Test_newPacket_v6(t *testing.T) {
 	b = append(b, ahb...)
 	b = append(b, udpHeader...)
 
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -335,12 +335,12 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// Ensure buffer bounds checking during processing
-	err = newPacket(b[:41], true, p)
+	err = newPacket(b[:41], true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 
 	// Invalid AH header
 	b = buffer.Bytes()
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 }
 
@@ -388,7 +388,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	firstFrag = append(firstFrag, []byte{0xde, 0xad, 0xbe, 0xef}...)
 
 	// Test first fragment incoming
-	err = newPacket(firstFrag, true, p)
+	err = newPacket(firstFrag, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr)
@@ -398,7 +398,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// Test first fragment outgoing
-	err = newPacket(firstFrag, false, p)
+	err = newPacket(firstFrag, false, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.RemoteAddr)
@@ -427,7 +427,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	secondFrag = append(secondFrag, []byte{0xde, 0xad, 0xbe, 0xef}...)
 
 	// Test second fragment incoming
-	err = newPacket(secondFrag, true, p)
+	err = newPacket(secondFrag, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr)
@@ -437,7 +437,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	assert.True(t, p.Fragment)
 
 	// Test second fragment outgoing
-	err = newPacket(secondFrag, false, p)
+	err = newPacket(secondFrag, false, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.RemoteAddr)
@@ -447,7 +447,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	assert.True(t, p.Fragment)
 
 	// Too short of a fragment packet
-	err = newPacket(secondFrag[:len(secondFrag)-10], false, p)
+	err = newPacket(secondFrag[:len(secondFrag)-10], false, p, nil)
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 }
 
@@ -529,7 +529,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("Normal", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(normalPacket, true, fp); err != nil {
+			if err = parseV6(normalPacket, true, fp, nil); err != nil {
 				b.Fatal(err)
 			}
 		}
@@ -537,7 +537,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("FirstFragment", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(firstFrag, true, fp); err != nil {
+			if err = parseV6(firstFrag, true, fp, nil); err != nil {
 				b.Fatal(err)
 			}
 		}
@@ -545,7 +545,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("SecondFragment", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(secondFrag, true, fp); err != nil {
+			if err = parseV6(secondFrag, true, fp, nil); err != nil {
 				b.Fatal(err)
 			}
 		}
@@ -590,7 +590,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("200 HopByHop headers", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(evilBytes, false, fp); err != nil {
+			if err = parseV6(evilBytes, false, fp, nil); err != nil {
 				b.Fatal(err)
 			}
 		}

overlay/device.go

@@ -11,9 +11,6 @@ type Device interface {
 	io.ReadWriteCloser
 	Activate() error
 	Networks() []netip.Prefix
-	UnsafeNetworks() []netip.Prefix
-	UnsafeIPv4OriginAddress() netip.Prefix
-	SNATAddress() netip.Prefix
 	Name() string
 	RoutesFor(netip.Addr) routing.Gateways
 	SupportsMultiqueue() bool

overlay/tun.go

@@ -1,9 +1,7 @@
 package overlay
 
 import (
-	"crypto/rand"
 	"fmt"
-	"io"
 	"net"
 	"net/netip"
 
@@ -24,22 +22,22 @@ func (e *NameError) Error() string {
 }
 
 // TODO: We may be able to remove routines
-type DeviceFactory func(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNetworks []netip.Prefix, routines int) (Device, error)
+type DeviceFactory func(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error)
 
-func NewDeviceFromConfig(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNetworks []netip.Prefix, routines int) (Device, error) {
+func NewDeviceFromConfig(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
 	switch {
 	case c.GetBool("tun.disabled", false):
 		tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l)
 		return tun, nil
 
 	default:
-		return newTun(c, l, vpnNetworks, unsafeNetworks, routines > 1)
+		return newTun(c, l, vpnNetworks, routines > 1)
 	}
 }
 
 func NewFdDeviceFromConfig(fd *int) DeviceFactory {
-	return func(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNetworks []netip.Prefix, routines int) (Device, error) {
+	return func(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
-		return newTunFromFd(c, l, *fd, vpnNetworks, unsafeNetworks)
+		return newTunFromFd(c, l, *fd, vpnNetworks)
 	}
 }
 
@@ -131,85 +129,3 @@ func selectGateway(dest netip.Prefix, gateways []netip.Prefix) (netip.Prefix, er
 
 	return netip.Prefix{}, fmt.Errorf("no gateway found for %v in the list of vpn networks", dest)
 }
-
-// genLinkLocal generates a random IPv4 link-local address.
-// If randomizer is nil, it uses rand.Reader to find two random bytes
-func genLinkLocal(randomizer io.Reader) netip.Prefix {
-	if randomizer == nil {
-		randomizer = rand.Reader
-	}
-	octets := []byte{169, 254, 0, 0}
-	_, _ = randomizer.Read(octets[2:4])
-	return coerceLinkLocal(octets)
-}
-
-func coerceLinkLocal(octets []byte) netip.Prefix {
-	if octets[3] == 0 {
-		octets[3] = 1 //please no .0 addresses
-	} else if octets[2] == 255 && octets[3] == 255 {
-		octets[3] = 254 //please no broadcast addresses
-	}
-	out, _ := netip.AddrFromSlice(octets)
-	return netip.PrefixFrom(out, 32)
-}
-
-// prepareUnsafeOriginAddr provides the IPv4 address used on IPv6-only clients that need to access IPv4 unsafe routes
-func prepareUnsafeOriginAddr(d Device, l *logrus.Logger, c *config.C, routes []Route) netip.Prefix {
-	if !d.Networks()[0].Addr().Is6() {
-		return netip.Prefix{} //if we have an IPv4 assignment within the overlay, we don't need an unsafe origin address
-	}
-
-	needed := false
-	for _, route := range routes { //or if we have a route defined into an IPv4 range
-		if route.Cidr.Addr().Is4() {
-			needed = true //todo should this only apply to unsafe routes? almost certainly
-			break
-		}
-	}
-	if !needed {
-		return netip.Prefix{}
-	}
-
-	//todo better config name for sure
-	if a := c.GetString("tun.unsafe_origin_address_for_4over6", ""); a != "" {
-		out, err := netip.ParseAddr(a)
-		if err != nil {
-			l.WithField("value", a).WithError(err).Warn("failed to parse tun.unsafe_origin_address_for_4over6, will use a random value")
-		} else if !out.Is4() || !out.IsLinkLocalUnicast() {
-			l.WithField("value", out).Warn("tun.unsafe_origin_address_for_4over6 must be an IPv4 address")
-		} else if out.IsValid() {
-			return netip.PrefixFrom(out, 32)
-		}
-	}
-	return genLinkLocal(nil)
-}
-
-// prepareSnatAddr provides the address that an IPv6-only unsafe router should use to SNAT traffic before handing it to the operating system
-func prepareSnatAddr(d Device, l *logrus.Logger, c *config.C) netip.Prefix {
-	if !d.Networks()[0].Addr().Is6() {
-		return netip.Prefix{} //if we have an IPv4 assignment within the overlay, we don't need a snat address
-	}
-
-	needed := false
-	for _, un := range d.UnsafeNetworks() { //if we are an unsafe router for an IPv4 range
-		if un.Addr().Is4() {
-			needed = true
-			break
-		}
-	}
-	if !needed {
-		return netip.Prefix{}
-	}
-
-	if a := c.GetString("tun.snat_address_for_4over6", ""); a != "" {
-		out, err := netip.ParseAddr(a)
-		if err != nil {
-			l.WithField("value", a).WithError(err).Warn("failed to parse tun.snat_address_for_4over6, will use a random value")
-		} else if !out.Is4() || !out.IsLinkLocalUnicast() {
-			l.WithField("value", out).Warn("tun.snat_address_for_4over6 must be an IPv4 address")
-		} else if out.IsValid() {
-			return netip.PrefixFrom(out, 32)
-		}
-	}
-	return genLinkLocal(nil)
-}

overlay/tun_android.go

@@ -19,16 +19,14 @@ import (
 
 type tun struct {
 	io.ReadWriteCloser
-	fd               int
+	fd          int
-	vpnNetworks      []netip.Prefix
+	vpnNetworks []netip.Prefix
-	unsafeNetworks   []netip.Prefix
+	Routes      atomic.Pointer[[]Route]
-	unsafeIPv4Origin netip.Prefix
+	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
-	Routes           atomic.Pointer[[]Route]
+	l           *logrus.Logger
-	routeTree        atomic.Pointer[bart.Table[routing.Gateways]]
-	l                *logrus.Logger
 }
 
-func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix, unsafeNetworks []netip.Prefix) (*tun, error) {
+func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
 	// XXX Android returns an fd in non-blocking mode which is necessary for shutdown to work properly.
 	// Be sure not to call file.Fd() as it will set the fd to blocking mode.
 	file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
@@ -37,7 +35,6 @@ func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []net
 		ReadWriteCloser: file,
 		fd:              deviceFd,
 		vpnNetworks:     vpnNetworks,
-		unsafeNetworks:  unsafeNetworks,
 		l:               l,
 	}
 
@@ -56,7 +53,7 @@ func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []net
 	return t, nil
 }
 
-func newTun(_ *config.C, _ *logrus.Logger, _ []netip.Prefix, _ []netip.Prefix, _ bool) (*tun, error) {
+func newTun(_ *config.C, _ *logrus.Logger, _ []netip.Prefix, _ bool) (*tun, error) {
 	return nil, fmt.Errorf("newTun not supported in Android")
 }
 
@@ -79,8 +76,6 @@ func (t *tun) reload(c *config.C, initial bool) error {
 		return nil
 	}
 
-	t.unsafeIPv4Origin = prepareUnsafeOriginAddr(t, t.l, c, routes)
-
 	routeTree, err := makeRouteTree(t.l, routes, false)
 	if err != nil {
 		return err
@@ -96,18 +91,6 @@ func (t *tun) Networks() []netip.Prefix {
 	return t.vpnNetworks
 }
 
-func (t *tun) UnsafeNetworks() []netip.Prefix {
-	return t.unsafeNetworks
-}
-
-func (t *tun) UnsafeIPv4OriginAddress() netip.Prefix {
-	return t.unsafeIPv4Origin
-}
-
-func (t *tun) SNATAddress() netip.Prefix {
-	return netip.Prefix{}
-}
-
 func (t *tun) Name() string {
 	return "android"
 }

overlay/tun_darwin.go

@@ -24,15 +24,13 @@ import (
 
 type tun struct {
 	io.ReadWriteCloser
-	Device           string
+	Device      string
-	vpnNetworks      []netip.Prefix
+	vpnNetworks []netip.Prefix
-	unsafeNetworks   []netip.Prefix
+	DefaultMTU  int
-	unsafeIPv4Origin netip.Prefix
+	Routes      atomic.Pointer[[]Route]
-	DefaultMTU       int
+	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
-	Routes           atomic.Pointer[[]Route]
+	linkAddr    *netroute.LinkAddr
-	routeTree        atomic.Pointer[bart.Table[routing.Gateways]]
+	l           *logrus.Logger
-	linkAddr         *netroute.LinkAddr
-	l                *logrus.Logger
 
 	// cache out buffer since we need to prepend 4 bytes for tun metadata
 	out []byte
@@ -81,7 +79,7 @@ type ifreqAlias6 struct {
 	Lifetime   addrLifetime
 }
 
-func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNetworks []netip.Prefix, _ bool) (*tun, error) {
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
 	name := c.GetString("tun.dev", "")
 	ifIndex := -1
 	if name != "" && name != "utun" {
@@ -129,7 +127,6 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNet
 		ReadWriteCloser: os.NewFile(uintptr(fd), ""),
 		Device:          name,
 		vpnNetworks:     vpnNetworks,
-		unsafeNetworks:  unsafeNetworks,
 		DefaultMTU:      c.GetInt("tun.mtu", DefaultMTU),
 		l:               l,
 	}
@@ -156,7 +153,7 @@ func (t *tun) deviceBytes() (o [16]byte) {
 	return
 }
 
-func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix, _ []netip.Prefix) (*tun, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in Darwin")
 }
 
@@ -216,11 +213,6 @@ func (t *tun) Activate() error {
 			}
 		}
 	}
-	if t.unsafeIPv4Origin.IsValid() && t.unsafeIPv4Origin.Addr().Is4() {
-		if err = t.activate4(t.unsafeIPv4Origin); err != nil {
-			return err
-		}
-	}
 
 	// Run the interface
 	ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING
@@ -322,10 +314,6 @@ func (t *tun) reload(c *config.C, initial bool) error {
 		return nil
 	}
 
-	if initial {
-		t.unsafeIPv4Origin = prepareUnsafeOriginAddr(t, t.l, c, routes)
-	}
-
 	routeTree, err := makeRouteTree(t.l, routes, false)
 	if err != nil {
 		return err
@@ -557,18 +545,6 @@ func (t *tun) Networks() []netip.Prefix {
 	return t.vpnNetworks
 }
 
-func (t *tun) UnsafeNetworks() []netip.Prefix {
-	return t.unsafeNetworks
-}
-
-func (t *tun) UnsafeIPv4OriginAddress() netip.Prefix {
-	return t.unsafeIPv4Origin
-}
-
-func (t *tun) SNATAddress() netip.Prefix {
-	return netip.Prefix{}
-}
-
 func (t *tun) Name() string {
 	return t.Device
 }

overlay/tun_disabled.go

@@ -52,17 +52,6 @@ func (t *disabledTun) Networks() []netip.Prefix {
 	return t.vpnNetworks
 }
 
-func (*disabledTun) UnsafeNetworks() []netip.Prefix {
-	return nil
-}
-func (*disabledTun) SNATAddress() netip.Prefix {
-	return netip.Prefix{}
-}
-
-func (*disabledTun) UnsafeIPv4OriginAddress() netip.Prefix {
-	return netip.Prefix{}
-}
-
 func (*disabledTun) Name() string {
 	return "disabled"
 }

overlay/tun_file_linux_test.go

@@ -0,0 +1,120 @@
+//go:build linux && !android && !e2e_testing
+// +build linux,!android,!e2e_testing
+
+package overlay
+
+import (
+	"errors"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// newReadPipe returns a read fd. The matching write fd is registered for cleanup.
+// The caller takes ownership of the read fd (pass it to newTunFd / newFriend).
+func newReadPipe(t *testing.T) int {
+	t.Helper()
+	var fds [2]int
+	if err := unix.Pipe2(fds[:], unix.O_CLOEXEC); err != nil {
+		t.Fatalf("pipe2: %v", err)
+	}
+	t.Cleanup(func() { _ = unix.Close(fds[1]) })
+	return fds[0]
+}
+
+func TestTunFile_WakeForShutdown_UnblocksRead(t *testing.T) {
+	tf, err := newTunFd(newReadPipe(t))
+	if err != nil {
+		t.Fatalf("newTunFd: %v", err)
+	}
+	t.Cleanup(func() { _ = tf.Close() })
+
+	done := make(chan error, 1)
+	go func() {
+		_, err := tf.Read(make([]byte, 64))
+		done <- err
+	}()
+
+	// Verify Read is actually blocked in poll.
+	select {
+	case err := <-done:
+		t.Fatalf("Read returned before shutdown signal: %v", err)
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	if err := tf.wakeForShutdown(); err != nil {
+		t.Fatalf("wakeForShutdown: %v", err)
+	}
+
+	select {
+	case err := <-done:
+		if !errors.Is(err, os.ErrClosed) {
+			t.Fatalf("expected os.ErrClosed, got %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("Read did not wake on shutdown")
+	}
+}
+
+func TestTunFile_WakeForShutdown_WakesFriends(t *testing.T) {
+	parent, err := newTunFd(newReadPipe(t))
+	if err != nil {
+		t.Fatalf("newTunFd: %v", err)
+	}
+	friend, err := parent.newFriend(newReadPipe(t))
+	if err != nil {
+		_ = parent.Close()
+		t.Fatalf("newFriend: %v", err)
+	}
+	t.Cleanup(func() {
+		_ = friend.Close()
+		_ = parent.Close()
+	})
+
+	readers := []*tunFile{parent, friend}
+	errs := make([]error, len(readers))
+	var wg sync.WaitGroup
+	for i, r := range readers {
+		wg.Add(1)
+		go func(i int, r *tunFile) {
+			defer wg.Done()
+			_, errs[i] = r.Read(make([]byte, 64))
+		}(i, r)
+	}
+
+	time.Sleep(50 * time.Millisecond)
+
+	if err := parent.wakeForShutdown(); err != nil {
+		t.Fatalf("wakeForShutdown: %v", err)
+	}
+
+	done := make(chan struct{})
+	go func() { wg.Wait(); close(done) }()
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("readers did not wake")
+	}
+
+	for i, err := range errs {
+		if !errors.Is(err, os.ErrClosed) {
+			t.Errorf("reader %d: expected os.ErrClosed, got %v", i, err)
+		}
+	}
+}
+
+func TestTunFile_Close_Idempotent(t *testing.T) {
+	tf, err := newTunFd(newReadPipe(t))
+	if err != nil {
+		t.Fatalf("newTunFd: %v", err)
+	}
+	if err := tf.Close(); err != nil {
+		t.Fatalf("first Close: %v", err)
+	}
+	if err := tf.Close(); err != nil {
+		t.Fatalf("second Close should be a no-op, got %v", err)
+	}
+}

overlay/tun_freebsd.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"io/fs"
 	"net/netip"
+	"os"
 	"sync/atomic"
 	"syscall"
 	"time"
@@ -86,141 +87,238 @@ type ifreqAlias6 struct {
 }
 
 type tun struct {
-	Device           string
+	Device      string
-	vpnNetworks      []netip.Prefix
+	vpnNetworks []netip.Prefix
-	unsafeNetworks   []netip.Prefix
+	MTU         int
-	unsafeIPv4Origin netip.Prefix
+	Routes      atomic.Pointer[[]Route]
-	MTU              int
+	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
-	Routes           atomic.Pointer[[]Route]
+	linkAddr    *netroute.LinkAddr
-	routeTree        atomic.Pointer[bart.Table[routing.Gateways]]
+	l           *logrus.Logger
-	linkAddr         *netroute.LinkAddr
+
-	l                *logrus.Logger
+	fd        int
-	devFd            int
+	shutdownR int // read end of the shutdown pipe; closing the write end wakes blocked polls
+	shutdownW int // write end of the shutdown pipe; closing this signals shutdown to any blocked reader/writer
+	readPoll  [2]unix.PollFd
+	writePoll [2]unix.PollFd
+	closed    atomic.Bool
+}
+
+// blockOnRead waits until the tun fd is readable or shutdown has been signaled.
+// Returns os.ErrClosed if Close was called.
+func (t *tun) blockOnRead() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(t.readPoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	tunEvents := t.readPoll[0].Revents
+	shutdownEvents := t.readPoll[1].Revents
+	t.readPoll[0].Revents = 0
+	t.readPoll[1].Revents = 0
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	}
+	if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
+}
+
+func (t *tun) blockOnWrite() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(t.writePoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	tunEvents := t.writePoll[0].Revents
+	shutdownEvents := t.writePoll[1].Revents
+	t.writePoll[0].Revents = 0
+	t.writePoll[1].Revents = 0
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	}
+	if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
 }
 
 func (t *tun) Read(to []byte) (int, error) {
-	// use readv() to read from the tunnel device, to eliminate the need for copying the buffer
-	if t.devFd < 0 {
-		return -1, syscall.EINVAL
-	}
-
 	// first 4 bytes is protocol family, in network byte order
-	head := make([]byte, 4)
+	var head [4]byte
-
+	iovecs := [2]syscall.Iovec{
-	iovecs := []syscall.Iovec{
 		{&head[0], 4},
 		{&to[0], uint64(len(to))},
 	}
-
+	for {
-	n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
+		n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
-
+		if errno == 0 {
-	var err error
+			bytesRead := int(n)
-	if errno != 0 {
+			if bytesRead < 4 {
-		err = syscall.Errno(errno)
+				return 0, nil
-	} else {
+			}
-		err = nil
+			return bytesRead - 4, nil
-	}
+		}
-	// fix bytes read number to exclude header
+		switch errno {
-	bytesRead := int(n)
+		case unix.EAGAIN:
-	if bytesRead < 0 {
+			if err := t.blockOnRead(); err != nil {
-		return bytesRead, err
+				return 0, err
-	} else if bytesRead < 4 {
+			}
-		return 0, err
+		case unix.EINTR:
-	} else {
+			// retry
-		return bytesRead - 4, err
+		case unix.EBADF:
+			return 0, os.ErrClosed
+		default:
+			return 0, errno
+		}
 	}
 }
 
 // Write is only valid for single threaded use
 func (t *tun) Write(from []byte) (int, error) {
-	// use writev() to write to the tunnel device, to eliminate the need for copying the buffer
-	if t.devFd < 0 {
-		return -1, syscall.EINVAL
-	}
-
 	if len(from) <= 1 {
 		return 0, syscall.EIO
 	}
+
 	ipVer := from[0] >> 4
-	var head []byte
+	var head [4]byte
 	// first 4 bytes is protocol family, in network byte order
-	if ipVer == 4 {
+	switch ipVer {
-		head = []byte{0, 0, 0, syscall.AF_INET}
+	case 4:
-	} else if ipVer == 6 {
+		head[3] = syscall.AF_INET
-		head = []byte{0, 0, 0, syscall.AF_INET6}
+	case 6:
-	} else {
+		head[3] = syscall.AF_INET6
+	default:
 		return 0, fmt.Errorf("unable to determine IP version from packet")
 	}
-	iovecs := []syscall.Iovec{
+
+	iovecs := [2]syscall.Iovec{
 		{&head[0], 4},
 		{&from[0], uint64(len(from))},
 	}
-
+	for {
-	n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
+		n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
-
+		if errno == 0 {
-	var err error
+			return int(n) - 4, nil
-	if errno != 0 {
+		}
-		err = syscall.Errno(errno)
+		switch errno {
-	} else {
+		case unix.EAGAIN:
-		err = nil
+			if err := t.blockOnWrite(); err != nil {
+				return 0, err
+			}
+		case unix.EINTR:
+			// retry
+		case unix.EBADF:
+			return 0, os.ErrClosed
+		default:
+			return 0, errno
+		}
 	}
-
-	return int(n) - 4, err
 }
 
 func (t *tun) Close() error {
-	if t.devFd >= 0 {
+	if t.closed.Swap(true) {
-		err := syscall.Close(t.devFd)
+		return nil
-		if err != nil {
+	}
+
+	// Closing the write end of the shutdown pipe causes any blocked Poll to
+	// return with POLLHUP on the shutdown fd, so readers/writers wake up and
+	// exit with os.ErrClosed.
+	if t.shutdownW >= 0 {
+		_ = unix.Close(t.shutdownW)
+		t.shutdownW = -1
+	}
+
+	if t.fd >= 0 {
+		if err := unix.Close(t.fd); err != nil {
 			t.l.WithError(err).Error("Error closing device")
 		}
-		t.devFd = -1
+		t.fd = -1
+	}
 
-		c := make(chan struct{})
+	if t.shutdownR >= 0 {
-		go func() {
+		_ = unix.Close(t.shutdownR)
-			// destroying the interface can block if a read() is still pending. Do this asynchronously.
+		t.shutdownR = -1
-			defer close(c)
+	}
-			s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
-			if err == nil {
-				defer syscall.Close(s)
-				ifreq := ifreqDestroy{Name: t.deviceBytes()}
-				err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
-			}
-			if err != nil {
-				t.l.WithError(err).Error("Error destroying tunnel")
-			}
-		}()
 
-		// wait up to 1 second so we start blocking at the ioctl
+	c := make(chan struct{})
-		select {
+	go func() {
-		case <-c:
+		// destroying the interface can block if a read() is still pending. Do this asynchronously.
-		case <-time.After(1 * time.Second):
+		defer close(c)
+		s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
+		if err == nil {
+			defer syscall.Close(s)
+			ifreq := ifreqDestroy{Name: t.deviceBytes()}
+			err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
 		}
+		if err != nil {
+			t.l.WithError(err).Error("Error destroying tunnel")
+		}
+	}()
+
+	// wait up to 1 second so we start blocking at the ioctl
+	select {
+	case <-c:
+	case <-time.After(1 * time.Second):
 	}
 
 	return nil
 }
 
-func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix, _ []netip.Prefix) (*tun, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in FreeBSD")
 }
 
-func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNetworks []netip.Prefix, _ bool) (*tun, error) {
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
 	// Try to open existing tun device
 	var fd int
 	var err error
 	deviceName := c.GetString("tun.dev", "")
 	if deviceName != "" {
-		fd, err = syscall.Open("/dev/"+deviceName, syscall.O_RDWR, 0)
+		fd, err = unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
 	}
 	if errors.Is(err, fs.ErrNotExist) || deviceName == "" {
 		// If the device doesn't already exist, request a new one and rename it
-		fd, err = syscall.Open("/dev/tun", syscall.O_RDWR, 0)
+		fd, err = unix.Open("/dev/tun", os.O_RDWR, 0)
 	}
 	if err != nil {
 		return nil, err
 	}
 
+	if err = unix.SetNonblock(fd, true); err != nil {
+		_ = unix.Close(fd)
+		return nil, fmt.Errorf("failed to set tun device as nonblocking: %w", err)
+	}
+
+	// Shutdown pipe lets Close wake any reader/writer blocked in Poll.
+	var pipeFds [2]int
+	if err = unix.Pipe2(pipeFds[:], unix.O_CLOEXEC|unix.O_NONBLOCK); err != nil {
+		_ = unix.Close(fd)
+		return nil, fmt.Errorf("failed to create shutdown pipe: %w", err)
+	}
+	shutdownR, shutdownW := pipeFds[0], pipeFds[1]
+
+	closeOnErr := true
+	defer func() {
+		if closeOnErr {
+			_ = unix.Close(fd)
+			_ = unix.Close(shutdownR)
+			_ = unix.Close(shutdownW)
+		}
+	}()
+
 	// Read the name of the interface
 	var name [16]byte
 	arg := fiodgnameArg{length: 16, buf: unsafe.Pointer(&name)}
@@ -239,7 +337,7 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNet
 	}
 
 	if ctrlErr != nil {
-		return nil, err
+		return nil, ctrlErr
 	}
 
 	ifName := string(bytes.TrimRight(name[:], "\x00"))
@@ -255,8 +353,6 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNet
 		}
 		defer syscall.Close(s)
 
-		fd := uintptr(s)
-
 		var fromName [16]byte
 		var toName [16]byte
 		copy(fromName[:], ifName)
@@ -268,22 +364,32 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNet
 		}
 
 		// Set the device name
-		_ = ioctl(fd, syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr)))
+		_ = ioctl(uintptr(s), syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr)))
 	}
 
 	t := &tun{
-		Device:         deviceName,
+		Device:      deviceName,
-		vpnNetworks:    vpnNetworks,
+		vpnNetworks: vpnNetworks,
-		unsafeNetworks: unsafeNetworks,
+		MTU:         c.GetInt("tun.mtu", DefaultMTU),
-		MTU:            c.GetInt("tun.mtu", DefaultMTU),
+		l:           l,
-		l:              l,
+		fd:          fd,
-		devFd:          fd,
+		shutdownR:   shutdownR,
+		shutdownW:   shutdownW,
+		readPoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLIN},
+			{Fd: int32(shutdownR), Events: unix.POLLIN},
+		},
+		writePoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLOUT},
+			{Fd: int32(shutdownR), Events: unix.POLLIN},
+		},
 	}
 
 	err = t.reload(c, true)
 	if err != nil {
 		return nil, err
 	}
+	closeOnErr = false
 
 	c.RegisterReloadCallback(func(c *config.C) {
 		err := t.reload(c, false)
@@ -413,10 +519,6 @@ func (t *tun) reload(c *config.C, initial bool) error {
 		return nil
 	}
 
-	if initial {
-		t.unsafeIPv4Origin = prepareUnsafeOriginAddr(t, t.l, c, routes)
-	}
-
 	routeTree, err := makeRouteTree(t.l, routes, false)
 	if err != nil {
 		return err
@@ -453,18 +555,6 @@ func (t *tun) Networks() []netip.Prefix {
 	return t.vpnNetworks
 }
 
-func (t *tun) UnsafeNetworks() []netip.Prefix {
-	return t.unsafeNetworks
-}
-
-func (t *tun) UnsafeIPv4OriginAddress() netip.Prefix {
-	return t.unsafeIPv4Origin
-}
-
-func (t *tun) SNATAddress() netip.Prefix {
-	return netip.Prefix{}
-}
-
 func (t *tun) Name() string {
 	return t.Device
 }

overlay/tun_ios.go

@@ -22,23 +22,20 @@ import (
 
 type tun struct {
 	io.ReadWriteCloser
-	vpnNetworks      []netip.Prefix
+	vpnNetworks []netip.Prefix
-	unsafeNetworks   []netip.Prefix
+	Routes      atomic.Pointer[[]Route]
-	unsafeIPv4Origin netip.Prefix
+	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
-	Routes           atomic.Pointer[[]Route]
+	l           *logrus.Logger
-	routeTree        atomic.Pointer[bart.Table[routing.Gateways]]
-	l                *logrus.Logger
 }
 
-func newTun(_ *config.C, _ *logrus.Logger, _ []netip.Prefix, _ []netip.Prefix, _ bool) (*tun, error) {
+func newTun(_ *config.C, _ *logrus.Logger, _ []netip.Prefix, _ bool) (*tun, error) {
 	return nil, fmt.Errorf("newTun not supported in iOS")
 }
 
-func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix, unsafeNetworks []netip.Prefix) (*tun, error) {
+func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
 	file := os.NewFile(uintptr(deviceFd), "/dev/tun")
 	t := &tun{
 		vpnNetworks:     vpnNetworks,
-		unsafeNetworks:  unsafeNetworks,
 		ReadWriteCloser: &tunReadCloser{f: file},
 		l:               l,
 	}
@@ -72,8 +69,6 @@ func (t *tun) reload(c *config.C, initial bool) error {
 		return nil
 	}
 
-	t.unsafeIPv4Origin = prepareUnsafeOriginAddr(t, t.l, c, routes)
-
 	routeTree, err := makeRouteTree(t.l, routes, false)
 	if err != nil {
 		return err
@@ -152,18 +147,6 @@ func (t *tun) Networks() []netip.Prefix {
 	return t.vpnNetworks
 }
 
-func (t *tun) UnsafeNetworks() []netip.Prefix {
-	return t.unsafeNetworks
-}
-
-func (t *tun) UnsafeIPv4OriginAddress() netip.Prefix {
-	return t.unsafeIPv4Origin
-}
-
-func (t *tun) SNATAddress() netip.Prefix {
-	return netip.Prefix{}
-}
-
 func (t *tun) Name() string {
 	return "iOS"
 }

overlay/tun_linux.go

@@ -4,6 +4,7 @@
 package overlay
 
 import (
+	"encoding/binary"
 	"fmt"
 	"io"
 	"net"
@@ -24,17 +25,182 @@ import (
 	"golang.org/x/sys/unix"
 )
 
+// tunFile wraps a TUN file descriptor with poll-based reads. The FD provided will be changed to non-blocking.
+// A shared eventfd allows Close to wake all readers blocked in poll.
+type tunFile struct {
+	fd         int
+	shutdownFd int
+	lastOne    bool
+	readPoll   [2]unix.PollFd
+	writePoll  [2]unix.PollFd
+	closed     bool
+}
+
+// newFriend makes a tunFile for a MultiQueueReader that copies the shutdown eventfd from the parent tun
+func (r *tunFile) newFriend(fd int) (*tunFile, error) {
+	if err := unix.SetNonblock(fd, true); err != nil {
+		return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
+	}
+	return &tunFile{
+		fd:         fd,
+		shutdownFd: r.shutdownFd,
+		readPoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLIN},
+			{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
+		},
+		writePoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLOUT},
+			{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
+		},
+	}, nil
+}
+
+func newTunFd(fd int) (*tunFile, error) {
+	if err := unix.SetNonblock(fd, true); err != nil {
+		return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
+	}
+
+	shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create eventfd: %w", err)
+	}
+
+	out := &tunFile{
+		fd:         fd,
+		shutdownFd: shutdownFd,
+		lastOne:    true,
+		readPoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLIN},
+			{Fd: int32(shutdownFd), Events: unix.POLLIN},
+		},
+		writePoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLOUT},
+			{Fd: int32(shutdownFd), Events: unix.POLLIN},
+		},
+	}
+
+	return out, nil
+}
+
+func (r *tunFile) blockOnRead() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(r.readPoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	//always reset these!
+	tunEvents := r.readPoll[0].Revents
+	shutdownEvents := r.readPoll[1].Revents
+	r.readPoll[0].Revents = 0
+	r.readPoll[1].Revents = 0
+	//do the err check before trusting the potentially bogus bits we just got
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	} else if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
+}
+
+func (r *tunFile) blockOnWrite() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(r.writePoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	//always reset these!
+	tunEvents := r.writePoll[0].Revents
+	shutdownEvents := r.writePoll[1].Revents
+	r.writePoll[0].Revents = 0
+	r.writePoll[1].Revents = 0
+	//do the err check before trusting the potentially bogus bits we just got
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	} else if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
+}
+
+func (r *tunFile) Read(buf []byte) (int, error) {
+	for {
+		if n, err := unix.Read(r.fd, buf); err == nil {
+			return n, nil
+		} else if err == unix.EAGAIN {
+			if err = r.blockOnRead(); err != nil {
+				return 0, err
+			}
+			continue
+		} else if err == unix.EINTR {
+			continue
+		} else if err == unix.EBADF {
+			return 0, os.ErrClosed
+		} else {
+			return 0, err
+		}
+	}
+}
+
+func (r *tunFile) Write(buf []byte) (int, error) {
+	for {
+		if n, err := unix.Write(r.fd, buf); err == nil {
+			return n, nil
+		} else if err == unix.EAGAIN {
+			if err = r.blockOnWrite(); err != nil {
+				return 0, err
+			}
+			continue
+		} else if err == unix.EINTR {
+			continue
+		} else if err == unix.EBADF {
+			return 0, os.ErrClosed
+		} else {
+			return 0, err
+		}
+	}
+}
+
+func (r *tunFile) wakeForShutdown() error {
+	var buf [8]byte
+	binary.NativeEndian.PutUint64(buf[:], 1)
+	_, err := unix.Write(int(r.readPoll[1].Fd), buf[:])
+	return err
+}
+
+func (r *tunFile) Close() error {
+	if r.closed { // avoid closing more than once. Technically a fd could get re-used, which would be a problem
+		return nil
+	}
+	r.closed = true
+	if r.lastOne {
+		_ = unix.Close(r.shutdownFd)
+	}
+	return unix.Close(r.fd)
+}
+
 type tun struct {
-	io.ReadWriteCloser
+	*tunFile
-	fd             int
+	readers     []*tunFile
-	Device         string
+	closeLock   sync.Mutex
-	vpnNetworks    []netip.Prefix
+	Device      string
-	unsafeNetworks []netip.Prefix
+	vpnNetworks []netip.Prefix
-	MaxMTU         int
+	MaxMTU      int
-	DefaultMTU     int
+	DefaultMTU  int
-	TXQueueLen     int
+	TXQueueLen  int
-	deviceIndex    int
+	deviceIndex int
-	ioctlFd        uintptr
+	ioctlFd     uintptr
 
 	Routes                    atomic.Pointer[[]Route]
 	routeTree                 atomic.Pointer[bart.Table[routing.Gateways]]
@@ -47,9 +213,6 @@ type tun struct {
 	routesFromSystem     map[netip.Prefix]routing.Gateways
 	routesFromSystemLock sync.Mutex
 
-	snatAddr         netip.Prefix
-	unsafeIPv4Origin netip.Prefix
-
 	l *logrus.Logger
 }
 
@@ -57,18 +220,6 @@ func (t *tun) Networks() []netip.Prefix {
 	return t.vpnNetworks
 }
 
-func (t *tun) UnsafeNetworks() []netip.Prefix {
-	return t.unsafeNetworks
-}
-
-func (t *tun) UnsafeIPv4OriginAddress() netip.Prefix {
-	return t.unsafeIPv4Origin
-}
-
-func (t *tun) SNATAddress() netip.Prefix {
-	return t.snatAddr
-}
-
 type ifReq struct {
 	Name  [16]byte
 	Flags uint16
@@ -87,10 +238,8 @@ type ifreqQLEN struct {
 	pad   [8]byte
 }
 
-func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix, unsafeNetworks []netip.Prefix) (*tun, error) {
+func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
-	file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
+	t, err := newTunGeneric(c, l, deviceFd, vpnNetworks)
-
-	t, err := newTunGeneric(c, l, file, vpnNetworks, unsafeNetworks)
 	if err != nil {
 		return nil, err
 	}
@@ -100,7 +249,7 @@ func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []net
 	return t, nil
 }
 
-func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNetworks []netip.Prefix, multiqueue bool) (*tun, error) {
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) {
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
 	if err != nil {
 		// If /dev/net/tun doesn't exist, try to create it (will happen in docker)
@@ -131,6 +280,7 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNet
 	nameStr := c.GetString("tun.dev", "")
 	copy(req.Name[:], nameStr)
 	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
+		_ = unix.Close(fd)
 		return nil, &NameError{
 			Name:       nameStr,
 			Underlying: err,
@@ -138,8 +288,7 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNet
 	}
 	name := strings.Trim(string(req.Name[:]), "\x00")
 
-	file := os.NewFile(uintptr(fd), "/dev/net/tun")
+	t, err := newTunGeneric(c, l, fd, vpnNetworks)
-	t, err := newTunGeneric(c, l, file, vpnNetworks, unsafeNetworks)
 	if err != nil {
 		return nil, err
 	}
@@ -149,12 +298,18 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNet
 	return t, nil
 }
 
-func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, vpnNetworks []netip.Prefix, unsafeNetworks []netip.Prefix) (*tun, error) {
+// newTunGeneric does all the stuff common to different tun initialization paths. It will close your files on error.
+func newTunGeneric(c *config.C, l *logrus.Logger, fd int, vpnNetworks []netip.Prefix) (*tun, error) {
+	tfd, err := newTunFd(fd)
+	if err != nil {
+		_ = unix.Close(fd)
+		return nil, err
+	}
 	t := &tun{
-		ReadWriteCloser:           file,
+		tunFile:                   tfd,
-		fd:                        int(file.Fd()),
+		readers:                   []*tunFile{tfd},
+		closeLock:                 sync.Mutex{},
 		vpnNetworks:               vpnNetworks,
-		unsafeNetworks:            unsafeNetworks,
 		TXQueueLen:                c.GetInt("tun.tx_queue", 500),
 		useSystemRoutes:           c.GetBool("tun.use_system_route_table", false),
 		useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0),
@@ -162,8 +317,8 @@ func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, vpnNetworks []n
 		l:                         l,
 	}
 
-	err := t.reload(c, true)
+	if err = t.reload(c, true); err != nil {
-	if err != nil {
+		_ = t.Close()
 		return nil, err
 	}
 
@@ -187,11 +342,6 @@ func (t *tun) reload(c *config.C, initial bool) error {
 		return nil
 	}
 
-	if initial {
-		t.unsafeIPv4Origin = prepareUnsafeOriginAddr(t, t.l, c, routes) //todo MUST be different from t.snatAddr!
-		t.snatAddr = prepareSnatAddr(t, t.l, c)
-	}
-
 	routeTree, err := makeRouteTree(t.l, routes, true)
 	if err != nil {
 		return err
@@ -261,6 +411,9 @@ func (t *tun) SupportsMultiqueue() bool {
 }
 
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	t.closeLock.Lock()
+	defer t.closeLock.Unlock()
+
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
 	if err != nil {
 		return nil, err
@@ -270,12 +423,19 @@ func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
 	copy(req.Name[:], t.Device)
 	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
+		_ = unix.Close(fd)
 		return nil, err
 	}
 
-	file := os.NewFile(uintptr(fd), "/dev/net/tun")
+	out, err := t.tunFile.newFriend(fd)
+	if err != nil {
+		_ = unix.Close(fd)
+		return nil, err
+	}
 
-	return file, nil
+	t.readers = append(t.readers, out)
+
+	return out, nil
 }
 
 func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
@@ -283,29 +443,6 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
 	return r
 }
 
-func (t *tun) Write(b []byte) (int, error) {
-	var nn int
-	maximum := len(b)
-
-	for {
-		n, err := unix.Write(t.fd, b[nn:maximum])
-		if n > 0 {
-			nn += n
-		}
-		if nn == len(b) {
-			return nn, err
-		}
-
-		if err != nil {
-			return nn, err
-		}
-
-		if n == 0 {
-			return nn, io.ErrUnexpectedEOF
-		}
-	}
-}
-
 func (t *tun) deviceBytes() (o [16]byte) {
 	for i, c := range t.Device {
 		o[i] = byte(c)
@@ -335,17 +472,6 @@ func (t *tun) addIPs(link netlink.Link) error {
 		}
 	}
 
-	if t.unsafeIPv4Origin.IsValid() {
-		newAddrs = append(newAddrs, &netlink.Addr{
-			IPNet: &net.IPNet{
-				IP:   t.unsafeIPv4Origin.Addr().AsSlice(),
-				Mask: net.CIDRMask(t.unsafeIPv4Origin.Bits(), t.unsafeIPv4Origin.Addr().BitLen()),
-			},
-			Label: t.unsafeIPv4Origin.Addr().Zone(),
-		})
-		t.l.WithField("address", t.unsafeIPv4Origin).Info("Adding origin address for IPv4 unsafe_routes")
-	}
-
 	//add all new addresses
 	for i := range newAddrs {
 		//AddrReplace still adds new IPs, but if their properties change it will change them as well
@@ -433,13 +559,7 @@ func (t *tun) Activate() error {
 	//set route MTU
 	for i := range t.vpnNetworks {
 		if err = t.setDefaultRoute(t.vpnNetworks[i]); err != nil {
-			return fmt.Errorf("failed to set default route MTU for %s: %w", t.vpnNetworks[i], err)
+			return fmt.Errorf("failed to set default route MTU: %w", err)
-		}
-	}
-
-	if t.unsafeIPv4Origin.IsValid() {
-		if err = t.setDefaultRoute(t.unsafeIPv4Origin); err != nil {
-			return fmt.Errorf("failed to set default route MTU for %s: %w", t.unsafeIPv4Origin, err)
 		}
 	}
 
@@ -466,23 +586,6 @@ func (t *tun) setMTU() {
 	}
 }
 
-func (t *tun) setSnatRoute() error {
-	dr := &net.IPNet{
-		IP:   t.snatAddr.Masked().Addr().AsSlice(),
-		Mask: net.CIDRMask(t.snatAddr.Bits(), t.snatAddr.Addr().BitLen()),
-	}
-
-	nr := netlink.Route{
-		LinkIndex: t.deviceIndex,
-		Dst:       dr,
-		Scope:     unix.RT_SCOPE_LINK,
-		//Protocol: unix.RTPROT_KERNEL,
-		Table: unix.RT_TABLE_MAIN,
-		Type:  unix.RTN_UNICAST,
-	}
-	return netlink.RouteReplace(&nr)
-}
-
 func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
 	dr := &net.IPNet{
 		IP:   cidr.Masked().Addr().AsSlice(),
@@ -559,13 +662,6 @@ func (t *tun) addRoutes(logErrors bool) error {
 		}
 	}
 
-	if t.snatAddr.IsValid() {
-		//at least for Linux, we need to set a return route for the SNATted traffic in order to satisfy the reverse-path filter,
-		//and to help the kernel deliver our reply traffic to the tun device.
-		//however, it is important that we do not actually /assign/ the SNAT address,
-		//since link-local addresses will not be routed between interfaces without significant trickery.
-		return t.setSnatRoute()
-	}
 	return nil
 }
 
@@ -770,18 +866,40 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
 }
 
 func (t *tun) Close() error {
+	t.closeLock.Lock()
+	defer t.closeLock.Unlock()
+
 	if t.routeChan != nil {
 		close(t.routeChan)
+		t.routeChan = nil
 	}
 
-	if t.ReadWriteCloser != nil {
+	// Signal all readers blocked in poll to wake up and exit
-		_ = t.ReadWriteCloser.Close()
+	_ = t.tunFile.wakeForShutdown()
-	}
 
 	if t.ioctlFd > 0 {
-		_ = os.NewFile(t.ioctlFd, "ioctlFd").Close()
+		_ = unix.Close(int(t.ioctlFd))
 		t.ioctlFd = 0
 	}
 
-	return nil
+	for i := range t.readers {
+		if i == 0 {
+			continue //we want to close the zeroth reader last
+		}
+		err := t.readers[i].Close()
+		if err != nil {
+			t.l.WithField("reader", i).WithError(err).Error("error closing tun reader")
+		} else {
+			t.l.WithField("reader", i).Info("closed tun reader")
+		}
+	}
+
+	//this is t.readers[0] too
+	err := t.tunFile.Close()
+	if err != nil {
+		t.l.WithField("reader", 0).WithError(err).Error("error closing tun reader")
+	} else {
+		t.l.WithField("reader", 0).Info("closed tun reader")
+	}
+	return err
 }

overlay/tun_netbsd.go

@@ -58,25 +58,23 @@ type addrLifetime struct {
 }
 
 type tun struct {
-	Device           string
+	Device      string
-	vpnNetworks      []netip.Prefix
+	vpnNetworks []netip.Prefix
-	unsafeNetworks   []netip.Prefix
+	MTU         int
-	unsafeIPv4Origin netip.Prefix
+	Routes      atomic.Pointer[[]Route]
-	MTU              int
+	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
-	Routes           atomic.Pointer[[]Route]
+	l           *logrus.Logger
-	routeTree        atomic.Pointer[bart.Table[routing.Gateways]]
+	f           *os.File
-	l                *logrus.Logger
+	fd          int
-	f                *os.File
-	fd               int
 }
 
 var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
 
-func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix, _ []netip.Prefix) (*tun, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in NetBSD")
 }
 
-func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNetworks []netip.Prefix, _ bool) (*tun, error) {
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
 	// Try to open tun device
 	var err error
 	deviceName := c.GetString("tun.dev", "")
@@ -352,10 +350,6 @@ func (t *tun) reload(c *config.C, initial bool) error {
 		return nil
 	}
 
-	if initial {
-		t.unsafeIPv4Origin = prepareUnsafeOriginAddr(t, t.l, c, routes)
-	}
-
 	routeTree, err := makeRouteTree(t.l, routes, false)
 	if err != nil {
 		return err
@@ -392,18 +386,6 @@ func (t *tun) Networks() []netip.Prefix {
 	return t.vpnNetworks
 }
 
-func (t *tun) UnsafeNetworks() []netip.Prefix {
-	return t.unsafeNetworks
-}
-
-func (t *tun) UnsafeIPv4OriginAddress() netip.Prefix {
-	return t.unsafeIPv4Origin
-}
-
-func (t *tun) SNATAddress() netip.Prefix {
-	return netip.Prefix{}
-}
-
 func (t *tun) Name() string {
 	return t.Device
 }

overlay/tun_openbsd.go

@@ -49,27 +49,25 @@ type ifreq struct {
 }
 
 type tun struct {
-	Device           string
+	Device      string
-	vpnNetworks      []netip.Prefix
+	vpnNetworks []netip.Prefix
-	unsafeNetworks   []netip.Prefix
+	MTU         int
-	unsafeIPv4Origin netip.Prefix
+	Routes      atomic.Pointer[[]Route]
-	MTU              int
+	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
-	Routes           atomic.Pointer[[]Route]
+	l           *logrus.Logger
-	routeTree        atomic.Pointer[bart.Table[routing.Gateways]]
+	f           *os.File
-	l                *logrus.Logger
+	fd          int
-	f                *os.File
-	fd               int
 	// cache out buffer since we need to prepend 4 bytes for tun metadata
 	out []byte
 }
 
 var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
 
-func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix, _ []netip.Prefix) (*tun, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*tun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in openbsd")
 }
 
-func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNetworks []netip.Prefix, _ bool) (*tun, error) {
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) {
 	// Try to open tun device
 	var err error
 	deviceName := c.GetString("tun.dev", "")
@@ -91,13 +89,12 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNet
 	}
 
 	t := &tun{
-		f:              os.NewFile(uintptr(fd), ""),
+		f:           os.NewFile(uintptr(fd), ""),
-		fd:             fd,
+		fd:          fd,
-		Device:         deviceName,
+		Device:      deviceName,
-		vpnNetworks:    vpnNetworks,
+		vpnNetworks: vpnNetworks,
-		unsafeNetworks: unsafeNetworks,
+		MTU:         c.GetInt("tun.mtu", DefaultMTU),
-		MTU:            c.GetInt("tun.mtu", DefaultMTU),
+		l:           l,
-		l:              l,
 	}
 
 	err = t.reload(c, true)
@@ -273,10 +270,6 @@ func (t *tun) reload(c *config.C, initial bool) error {
 		return nil
 	}
 
-	if initial {
-		t.unsafeIPv4Origin = prepareUnsafeOriginAddr(t, t.l, c, routes)
-	}
-
 	routeTree, err := makeRouteTree(t.l, routes, false)
 	if err != nil {
 		return err
@@ -313,18 +306,6 @@ func (t *tun) Networks() []netip.Prefix {
 	return t.vpnNetworks
 }
 
-func (t *tun) UnsafeNetworks() []netip.Prefix {
-	return t.unsafeNetworks
-}
-
-func (t *tun) UnsafeIPv4OriginAddress() netip.Prefix {
-	return t.unsafeIPv4Origin
-}
-
-func (t *tun) SNATAddress() netip.Prefix {
-	return netip.Prefix{}
-}
-
 func (t *tun) Name() string {
 	return t.Device
 }

overlay/tun_snat_test.go

@@ -1,179 +0,0 @@
-package overlay
-
-import (
-	"io"
-	"net/netip"
-	"testing"
-
-	"github.com/sirupsen/logrus"
-	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/routing"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// mockDevice is a minimal Device implementation for testing prepareUnsafeOriginAddr.
-type mockDevice struct {
-	networks       []netip.Prefix
-	unsafeNetworks []netip.Prefix
-	snatAddr       netip.Prefix
-	unsafeSnatAddr netip.Prefix
-}
-
-func (d *mockDevice) Read([]byte) (int, error)                         { return 0, nil }
-func (d *mockDevice) Write([]byte) (int, error)                        { return 0, nil }
-func (d *mockDevice) Close() error                                     { return nil }
-func (d *mockDevice) Activate() error                                  { return nil }
-func (d *mockDevice) Networks() []netip.Prefix                         { return d.networks }
-func (d *mockDevice) UnsafeNetworks() []netip.Prefix                   { return d.unsafeNetworks }
-func (d *mockDevice) SNATAddress() netip.Prefix                        { return d.snatAddr }
-func (d *mockDevice) UnsafeIPv4OriginAddress() netip.Prefix            { return d.unsafeSnatAddr }
-func (d *mockDevice) Name() string                                     { return "mock" }
-func (d *mockDevice) RoutesFor(netip.Addr) routing.Gateways            { return routing.Gateways{} }
-func (d *mockDevice) SupportsMultiqueue() bool                         { return false }
-func (d *mockDevice) NewMultiQueueReader() (io.ReadWriteCloser, error) { return nil, nil }
-
-func TestPrepareSnatAddr_V4Primary_NoSnat(t *testing.T) {
-	l := logrus.New()
-	l.SetLevel(logrus.PanicLevel)
-	c := config.NewC(l)
-
-	// If the device has an IPv4 primary address, no SNAT needed
-	d := &mockDevice{
-		networks: []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")},
-	}
-	result := prepareUnsafeOriginAddr(d, l, c, nil)
-	assert.Equal(t, netip.Prefix{}, result, "should not assign SNAT addr when device has IPv4 primary")
-}
-
-func TestPrepareSnatAddr_V6Primary_NoUnsafeOrRoutes(t *testing.T) {
-	l := logrus.New()
-	l.SetLevel(logrus.PanicLevel)
-	c := config.NewC(l)
-
-	// IPv6 primary but no unsafe networks or IPv4 routes
-	d := &mockDevice{
-		networks: []netip.Prefix{netip.MustParsePrefix("fd00::1/128")},
-	}
-	result := prepareUnsafeOriginAddr(d, l, c, nil)
-	assert.Equal(t, netip.Prefix{}, result, "should not assign SNAT addr without IPv4 unsafe networks or routes")
-}
-
-func TestPrepareSnatAddr_V6Primary_WithV4Unsafe(t *testing.T) {
-	l := logrus.New()
-	l.SetLevel(logrus.PanicLevel)
-	c := config.NewC(l)
-
-	// IPv6 primary with IPv4 unsafe network -> should get SNAT addr
-	d := &mockDevice{
-		networks:       []netip.Prefix{netip.MustParsePrefix("fd00::1/128")},
-		unsafeNetworks: []netip.Prefix{netip.MustParsePrefix("192.168.0.0/16")},
-	}
-	result := prepareSnatAddr(d, l, c)
-	require.True(t, result.IsValid(), "should assign SNAT addr")
-	assert.True(t, result.Addr().Is4(), "SNAT addr should be IPv4")
-	assert.True(t, result.Addr().IsLinkLocalUnicast(), "SNAT addr should be link-local")
-	assert.Equal(t, 32, result.Bits(), "SNAT addr should be /32")
-
-	result = prepareUnsafeOriginAddr(d, l, c, nil)
-	require.False(t, result.IsValid(), "no routes = no origin addr needed")
-}
-
-func TestPrepareUnsafeOriginAddr_V6Primary_WithV4Route(t *testing.T) {
-	l := logrus.New()
-	l.SetLevel(logrus.PanicLevel)
-	c := config.NewC(l)
-
-	// IPv6 primary with IPv4 route -> should get SNAT addr
-	d := &mockDevice{
-		networks: []netip.Prefix{netip.MustParsePrefix("fd00::1/128")},
-	}
-	routes := []Route{
-		{Cidr: netip.MustParsePrefix("10.0.0.0/8")},
-	}
-	result := prepareUnsafeOriginAddr(d, l, c, routes)
-	require.True(t, result.IsValid(), "should assign SNAT addr when IPv4 route exists")
-	assert.True(t, result.Addr().Is4())
-	assert.True(t, result.Addr().IsLinkLocalUnicast())
-
-	result = prepareSnatAddr(d, l, c)
-	require.False(t, result.IsValid(), "no UnsafeNetworks = no snat addr needed")
-}
-
-func TestPrepareSnatAddr_V6Primary_V6UnsafeOnly(t *testing.T) {
-	l := logrus.New()
-	l.SetLevel(logrus.PanicLevel)
-	c := config.NewC(l)
-
-	// IPv6 primary with only IPv6 unsafe network -> no SNAT needed
-	d := &mockDevice{
-		networks:       []netip.Prefix{netip.MustParsePrefix("fd00::1/128")},
-		unsafeNetworks: []netip.Prefix{netip.MustParsePrefix("fd01::/64")},
-	}
-	result := prepareUnsafeOriginAddr(d, l, c, nil)
-	assert.Equal(t, netip.Prefix{}, result, "should not assign SNAT addr for IPv6-only unsafe networks")
-}
-
-func TestPrepareSnatAddr_ManualAddress(t *testing.T) {
-	l := logrus.New()
-	l.SetLevel(logrus.PanicLevel)
-	c := config.NewC(l)
-	c.Settings["tun"] = map[string]any{
-		"snat_address_for_4over6": "169.254.42.42",
-	}
-
-	d := &mockDevice{
-		networks:       []netip.Prefix{netip.MustParsePrefix("fd00::1/128")},
-		unsafeNetworks: []netip.Prefix{netip.MustParsePrefix("192.168.0.0/16")},
-	}
-	result := prepareSnatAddr(d, l, c)
-	require.True(t, result.IsValid())
-	assert.Equal(t, netip.MustParseAddr("169.254.42.42"), result.Addr())
-	assert.Equal(t, 32, result.Bits())
-}
-
-func TestPrepareSnatAddr_InvalidManualAddress_Fallback(t *testing.T) {
-	l := logrus.New()
-	l.SetLevel(logrus.PanicLevel)
-	c := config.NewC(l)
-	c.Settings["tun"] = map[string]any{
-		"snat_address_for_4over6": "not-an-ip",
-	}
-
-	d := &mockDevice{
-		networks:       []netip.Prefix{netip.MustParsePrefix("fd00::1/128")},
-		unsafeNetworks: []netip.Prefix{netip.MustParsePrefix("192.168.0.0/16")},
-	}
-	result := prepareSnatAddr(d, l, c)
-	// Should fall back to auto-assignment
-	require.True(t, result.IsValid(), "should fall back to auto-assigned address")
-	assert.True(t, result.Addr().Is4())
-	assert.True(t, result.Addr().IsLinkLocalUnicast())
-}
-
-func TestPrepareSnatAddr_AutoGenerated_Range(t *testing.T) {
-	l := logrus.New()
-	l.SetLevel(logrus.PanicLevel)
-	c := config.NewC(l)
-
-	d := &mockDevice{
-		networks:       []netip.Prefix{netip.MustParsePrefix("fd00::1/128")},
-		unsafeNetworks: []netip.Prefix{netip.MustParsePrefix("192.168.0.0/16")},
-	}
-
-	// Generate several addresses and verify they're all in the expected range
-	for i := 0; i < 100; i++ {
-		result := prepareSnatAddr(d, l, c)
-		require.True(t, result.IsValid())
-		addr := result.Addr()
-		octets := addr.As4()
-		assert.Equal(t, byte(169), octets[0], "first octet should be 169")
-		assert.Equal(t, byte(254), octets[1], "second octet should be 254")
-		// Should not have .0 in the last octet
-		assert.NotEqual(t, byte(0), octets[3], "last octet should not be 0")
-		// Should not be 169.254.255.255 (broadcast)
-		if octets[2] == 255 {
-			assert.NotEqual(t, byte(255), octets[3], "should not be broadcast address")
-		}
-	}
-}

overlay/tun_test.go

@@ -1,37 +0,0 @@
-package overlay
-
-import (
-	"bytes"
-	"net/netip"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestLinkLocal(t *testing.T) {
-	r := bytes.NewReader([]byte{42, 99})
-	result := genLinkLocal(r)
-	assert.Equal(t, netip.MustParsePrefix("169.254.42.99/32"), result, "genLinkLocal with a deterministic randomizer")
-
-	result = genLinkLocal(nil)
-	assert.True(t, result.IsValid(), "genLinkLocal with nil randomizer should be valid")
-	assert.True(t, result.Addr().IsLinkLocalUnicast(), "genLinkLocal with nil randomizer should be link-local")
-
-	result = coerceLinkLocal([]byte{169, 254, 100, 50})
-	assert.Equal(t, netip.MustParsePrefix("169.254.100.50/32"), result, "coerceLinkLocal should pass through normal values")
-
-	result = coerceLinkLocal([]byte{169, 254, 0, 0})
-	assert.Equal(t, netip.MustParsePrefix("169.254.0.1/32"), result, "coerceLinkLocal should bump .0 last octet to .1")
-
-	result = coerceLinkLocal([]byte{169, 254, 255, 255})
-	assert.Equal(t, netip.MustParsePrefix("169.254.255.254/32"), result, "coerceLinkLocal should bump broadcast 255.255 to 255.254")
-
-	result = coerceLinkLocal([]byte{169, 254, 0, 1})
-	assert.Equal(t, netip.MustParsePrefix("169.254.0.1/32"), result, "coerceLinkLocal should leave .1 last octet unchanged")
-
-	result = coerceLinkLocal([]byte{169, 254, 255, 254})
-	assert.Equal(t, netip.MustParsePrefix("169.254.255.254/32"), result, "coerceLinkLocal should leave 255.254 unchanged")
-
-	result = coerceLinkLocal([]byte{169, 254, 255, 100})
-	assert.Equal(t, netip.MustParsePrefix("169.254.255.100/32"), result, "coerceLinkLocal should leave 255.100 unchanged")
-}

overlay/tun_tester.go

@@ -17,21 +17,18 @@ import (
 )
 
 type TestTun struct {
-	Device           string
+	Device      string
-	vpnNetworks      []netip.Prefix
+	vpnNetworks []netip.Prefix
-	unsafeNetworks   []netip.Prefix
+	Routes      []Route
-	snatAddr         netip.Prefix
+	routeTree   *bart.Table[routing.Gateways]
-	unsafeIPv4Origin netip.Prefix
+	l           *logrus.Logger
-	Routes           []Route
-	routeTree        *bart.Table[routing.Gateways]
-	l                *logrus.Logger
 
 	closed    atomic.Bool
 	rxPackets chan []byte // Packets to receive into nebula
 	TxPackets chan []byte // Packets transmitted outside by nebula
 }
 
-func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNetworks []netip.Prefix, _ bool) (*TestTun, error) {
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*TestTun, error) {
 	_, routes, err := getAllRoutesFromConfig(c, vpnNetworks, true)
 	if err != nil {
 		return nil, err
@@ -41,22 +38,18 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNet
 		return nil, err
 	}
 
-	tt := &TestTun{
+	return &TestTun{
-		Device:         c.GetString("tun.dev", ""),
+		Device:      c.GetString("tun.dev", ""),
-		vpnNetworks:    vpnNetworks,
+		vpnNetworks: vpnNetworks,
-		unsafeNetworks: unsafeNetworks,
+		Routes:      routes,
-		Routes:         routes,
+		routeTree:   routeTree,
-		routeTree:      routeTree,
+		l:           l,
-		l:              l,
+		rxPackets:   make(chan []byte, 10),
-		rxPackets:      make(chan []byte, 10),
+		TxPackets:   make(chan []byte, 10),
-		TxPackets:      make(chan []byte, 10),
+	}, nil
-	}
-	tt.unsafeIPv4Origin = prepareUnsafeOriginAddr(tt, l, c, routes)
-	tt.snatAddr = prepareSnatAddr(tt, tt.l, c)
-	return tt, nil
 }
 
-func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix, _ []netip.Prefix) (*TestTun, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (*TestTun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported")
 }
 
@@ -146,15 +139,3 @@ func (t *TestTun) SupportsMultiqueue() bool {
 func (t *TestTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented")
 }
-
-func (t *TestTun) UnsafeNetworks() []netip.Prefix {
-	return t.unsafeNetworks
-}
-
-func (t *TestTun) UnsafeIPv4OriginAddress() netip.Prefix {
-	return t.unsafeIPv4Origin
-}
-
-func (t *TestTun) SNATAddress() netip.Prefix {
-	return t.snatAddr
-}

overlay/tun_windows.go

@@ -28,23 +28,21 @@ import (
 const tunGUIDLabel = "Fixed Nebula Windows GUID v1"
 
 type winTun struct {
-	Device           string
+	Device      string
-	vpnNetworks      []netip.Prefix
+	vpnNetworks []netip.Prefix
-	unsafeNetworks   []netip.Prefix
+	MTU         int
-	unsafeIPv4Origin netip.Prefix
+	Routes      atomic.Pointer[[]Route]
-	MTU              int
+	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
-	Routes           atomic.Pointer[[]Route]
+	l           *logrus.Logger
-	routeTree        atomic.Pointer[bart.Table[routing.Gateways]]
-	l                *logrus.Logger
 
 	tun *wintun.NativeTun
 }
 
-func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix, _ []netip.Prefix) (Device, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ []netip.Prefix) (Device, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in Windows")
 }
 
-func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNetworks []netip.Prefix, _ bool) (*winTun, error) {
+func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*winTun, error) {
 	err := checkWinTunExists()
 	if err != nil {
 		return nil, fmt.Errorf("can not load the wintun driver: %w", err)
@@ -57,11 +55,10 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNet
 	}
 
 	t := &winTun{
-		Device:         deviceName,
+		Device:      deviceName,
-		vpnNetworks:    vpnNetworks,
+		vpnNetworks: vpnNetworks,
-		unsafeNetworks: unsafeNetworks,
+		MTU:         c.GetInt("tun.mtu", DefaultMTU),
-		MTU:            c.GetInt("tun.mtu", DefaultMTU),
+		l:           l,
-		l:              l,
 	}
 
 	err = t.reload(c, true)
@@ -105,10 +102,6 @@ func (t *winTun) reload(c *config.C, initial bool) error {
 		return nil
 	}
 
-	if initial {
-		t.unsafeIPv4Origin = prepareUnsafeOriginAddr(t, t.l, c, routes)
-	}
-
 	routeTree, err := makeRouteTree(t.l, routes, false)
 	if err != nil {
 		return err
@@ -139,12 +132,7 @@ func (t *winTun) reload(c *config.C, initial bool) error {
 func (t *winTun) Activate() error {
 	luid := winipcfg.LUID(t.tun.LUID())
 
-	prefixes := t.vpnNetworks
+	err := luid.SetIPAddresses(t.vpnNetworks)
-	if t.unsafeIPv4Origin.IsValid() {
-		prefixes = append(prefixes, t.unsafeIPv4Origin)
-	}
-
-	err := luid.SetIPAddresses(prefixes)
 	if err != nil {
 		return fmt.Errorf("failed to set address: %w", err)
 	}
@@ -237,18 +225,6 @@ func (t *winTun) Networks() []netip.Prefix {
 	return t.vpnNetworks
 }
 
-func (t *winTun) UnsafeNetworks() []netip.Prefix {
-	return t.unsafeNetworks
-}
-
-func (t *winTun) UnsafeIPv4OriginAddress() netip.Prefix {
-	return t.unsafeIPv4Origin
-}
-
-func (t *winTun) SNATAddress() netip.Prefix {
-	return netip.Prefix{}
-}
-
 func (t *winTun) Name() string {
 	return t.Device
 }

overlay/user.go

@@ -9,7 +9,7 @@ import (
 	"github.com/slackhq/nebula/routing"
 )
 
-func NewUserDeviceFromConfig(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, unsafeNetworks []netip.Prefix, routines int) (Device, error) {
+func NewUserDeviceFromConfig(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
 	return NewUserDevice(vpnNetworks)
 }
 
@@ -36,17 +36,6 @@ type UserDevice struct {
 	inboundWriter *io.PipeWriter
 }
 
-func (d *UserDevice) UnsafeNetworks() []netip.Prefix {
-	return nil
-}
-
-func (d *UserDevice) SNATAddress() netip.Prefix {
-	return netip.Prefix{}
-}
-func (d *UserDevice) UnsafeIPv4OriginAddress() netip.Prefix {
-	return netip.Prefix{}
-}
-
 func (d *UserDevice) Activate() error {
 	return nil
 }

pki.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net"
 	"net/netip"
 	"os"
@@ -91,7 +92,7 @@ func (p *PKI) reload(c *config.C, initial bool) error {
 }
 
 func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
-	newState, err := newCertStateFromConfig(c, p.l)
+	newState, err := newCertStateFromConfig(c)
 	if err != nil {
 		return util.NewContextualError("Could not load client cert", nil, err)
 	}
@@ -102,7 +103,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
 			if currentState.v1Cert == nil {
 				//adding certs is fine, actually. Networks-in-common confirmed in newCertState().
 			} else {
-				// did IP in cert change? if so, don't set. If we ever allow this, need to set p.firewallReloadNeeded
+				// did IP in cert change? if so, don't set
 				if !slices.Equal(currentState.v1Cert.Networks(), newState.v1Cert.Networks()) {
 					return util.NewContextualError(
 						"Networks in new cert was different from old",
@@ -158,14 +159,6 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError {
 			}
 		}
 
-		newUN := newState.GetDefaultCertificate().UnsafeNetworks()
-		oldUN := currentState.GetDefaultCertificate().UnsafeNetworks()
-		if !slices.Equal(newUN, oldUN) {
-			//todo I don't love this, because other clients will see the new assignments and act on them, but we will not be able to.
-			//I think we need to wire this into the firewall reload.
-			p.l.WithFields(m{"previous": oldUN, "new": newUN}).Warning("UnsafeNetworks assignments differ. A restart is required in order for this to take effect.")
-		}
-
 		// Cipher cant be hot swapped so just leave it at what it was before
 		newState.cipher = currentState.cipher
 
@@ -268,7 +261,7 @@ func (cs *CertState) MarshalJSON() ([]byte, error) {
 	return json.Marshal(msg)
 }
 
-func newCertStateFromConfig(c *config.C, l *logrus.Logger) (*CertState, error) {
+func newCertStateFromConfig(c *config.C) (*CertState, error) {
 	var err error
 
 	privPathOrPEM := c.GetString("pki.key", "")
@@ -352,33 +345,10 @@ func newCertStateFromConfig(c *config.C, l *logrus.Logger) (*CertState, error) {
 		return nil, fmt.Errorf("unknown pki.initiating_version: %v", rawInitiatingVersion)
 	}
 
-	return newCertState(l, initiatingVersion, v1, v2, isPkcs11, curve, rawKey)
+	return newCertState(initiatingVersion, v1, v2, isPkcs11, curve, rawKey)
 }
 
-func compareUnsafeNetworksAcrossCertVersions(v1, v2 cert.Certificate) error {
+func newCertState(dv cert.Version, v1, v2 cert.Certificate, pkcs11backed bool, privateKeyCurve cert.Curve, privateKey []byte) (*CertState, error) {
-	if v1 == nil || v2 == nil {
-		return nil //can't be a problem if we don't have one of the kinds of cert
-	}
-
-	v4UnsafeNets := 0
-	for _, n := range v2.UnsafeNetworks() {
-		if n.Addr().Is6() {
-			continue // V1 certs can't have IPv6 unsafe networks
-		} else {
-			v4UnsafeNets++
-		}
-		if !slices.Contains(v1.UnsafeNetworks(), n) {
-			return errors.New("UnsafeNetworks mismatch")
-		}
-	}
-	if len(v1.UnsafeNetworks()) != v4UnsafeNets {
-		return errors.New("UnsafeNetworks mismatch")
-	}
-
-	return nil
-}
-
-func newCertState(l *logrus.Logger, dv cert.Version, v1, v2 cert.Certificate, pkcs11backed bool, privateKeyCurve cert.Curve, privateKey []byte) (*CertState, error) {
 	cs := CertState{
 		privateKey:               privateKey,
 		pkcs11Backed:             pkcs11backed,
@@ -401,12 +371,6 @@ func newCertState(l *logrus.Logger, dv cert.Version, v1, v2 cert.Certificate, pk
 		}
 
 		cs.initiatingVersion = dv
-
-		warn := compareUnsafeNetworksAcrossCertVersions(v1, v2)
-		if warn != nil {
-			l.WithFields(m{"UnsafeNetworksV1": v1.UnsafeNetworks(), "UnsafeNetworksV2": v2.UnsafeNetworks()}).
-				Warning("the IPv4 UnsafeNetworks assigned in the V1 certificate do not match the ones in V2")
-		}
 	}
 
 	if v1 != nil {
@@ -524,25 +488,25 @@ func loadCertificate(b []byte) (cert.Certificate, []byte, error) {
 }
 
 func loadCAPoolFromConfig(l *logrus.Logger, c *config.C) (*cert.CAPool, error) {
-	var rawCA []byte
-	var err error
-
 	caPathOrPEM := c.GetString("pki.ca", "")
 	if caPathOrPEM == "" {
 		return nil, errors.New("no pki.ca path or PEM data provided")
 	}
 
-	if strings.Contains(caPathOrPEM, "-----BEGIN") {
+	var caReader io.ReadCloser
-		rawCA = []byte(caPathOrPEM)
+	var err error
 
+	if strings.Contains(caPathOrPEM, "-----BEGIN") {
+		caReader = io.NopCloser(strings.NewReader(caPathOrPEM))
 	} else {
-		rawCA, err = os.ReadFile(caPathOrPEM)
+		caReader, err = os.Open(caPathOrPEM)
 		if err != nil {
 			return nil, fmt.Errorf("unable to read pki.ca file %s: %s", caPathOrPEM, err)
 		}
 	}
+	defer caReader.Close()
 
-	caPool, err := cert.NewCAPoolFromPEM(rawCA)
+	caPool, err := cert.NewCAPoolFromPEMReader(caReader)
 	if errors.Is(err, cert.ErrExpired) {
 		var expired int
 		for _, crt := range caPool.CAs {

pki_hup_benchmark_test.go

@@ -0,0 +1,121 @@
+package nebula
+
+import (
+	"bytes"
+	"fmt"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+	"time"
+
+	"github.com/slackhq/nebula/cert"
+	cert_test "github.com/slackhq/nebula/cert_test"
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/test"
+	"github.com/stretchr/testify/require"
+)
+
+func BenchmarkReloadConfigWithCAs(b *testing.B) {
+	prevProcs := runtime.GOMAXPROCS(1)
+	b.Cleanup(func() { runtime.GOMAXPROCS(prevProcs) })
+
+	for _, size := range []int{100, 250, 500, 1000, 5000} {
+		b.Run(fmt.Sprintf("%dCAs", size), func(b *testing.B) {
+			l := test.NewLogger()
+			dir := b.TempDir()
+
+			ca, caKey, caBundle := buildCABundle(b, size)
+			caPath, certPath, keyPath := writePKIFiles(b, dir, ca, caKey, caBundle)
+
+			configBody := fmt.Sprintf(`pki:
+  ca: %s
+  cert: %s
+  key: %s
+`, caPath, certPath, keyPath)
+
+			configPath := filepath.Join(dir, "config.yml")
+			require.NoError(b, os.WriteFile(configPath, []byte(configBody), 0o600))
+
+			c := config.NewC(l)
+			require.NoError(b, c.Load(dir))
+
+			_, err := NewPKIFromConfig(l, c)
+			require.NoError(b, err)
+
+			b.ReportAllocs()
+			b.ResetTimer()
+
+			for b.Loop() {
+				c.ReloadConfig()
+			}
+		})
+	}
+}
+
+func buildCABundle(b *testing.B, count int) (cert.Certificate, []byte, []byte) {
+	b.Helper()
+	require.GreaterOrEqual(b, count, 1)
+
+	before := time.Now().Add(-24 * time.Hour)
+	after := time.Now().Add(24 * time.Hour)
+
+	ca, _, caKey, pem := cert_test.NewTestCaCert(
+		cert.Version2,
+		cert.Curve_CURVE25519,
+		before,
+		after,
+		nil,
+		nil,
+		nil,
+	)
+
+	buf := bytes.NewBuffer(pem)
+	buf.Write([]byte("\n# a comment!\n"))
+
+	for i := 1; i < count; i++ {
+		_, _, _, extraPEM := cert_test.NewTestCaCert(
+			cert.Version2,
+			cert.Curve_CURVE25519,
+			time.Now(),
+			time.Now().Add(time.Hour),
+			nil,
+			nil,
+			nil,
+		)
+		buf.Write([]byte("\n# a comment!\n"))
+		buf.Write(extraPEM)
+	}
+
+	return ca, caKey, buf.Bytes()
+}
+
+func writePKIFiles(b *testing.B, dir string, ca cert.Certificate, caKey []byte, caBundle []byte) (string, string, string) {
+	b.Helper()
+
+	networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
+
+	_, _, keyPEM, certPEM := cert_test.NewTestCert(
+		cert.Version2,
+		cert.Curve_CURVE25519,
+		ca,
+		caKey,
+		"reload-benchmark",
+		time.Now(),
+		time.Now().Add(time.Hour),
+		networks,
+		nil,
+		nil,
+	)
+
+	caPath := filepath.Join(dir, "ca.pem")
+	certPath := filepath.Join(dir, "cert.pem")
+	keyPath := filepath.Join(dir, "key.pem")
+
+	require.NoError(b, os.WriteFile(caPath, caBundle, 0o600))
+	require.NoError(b, os.WriteFile(certPath, certPEM, 0o600))
+	require.NoError(b, os.WriteFile(keyPath, keyPEM, 0o600))
+
+	return caPath, certPath, keyPath
+}

service/service.go

@@ -44,7 +44,10 @@ type Service struct {
 }
 
 func New(control *nebula.Control) (*Service, error) {
-	control.Start()
+	wait, err := control.Start()
+	if err != nil {
+		return nil, err
+	}
 
 	ctx := control.Context()
 	eg, ctx := errgroup.WithContext(ctx)
@@ -141,6 +144,12 @@ func New(control *nebula.Control) (*Service, error) {
 		}
 	})
 
+	// Add the nebula wait function to the group so a fatal reader error
+	// propagates out through errgroup.Wait().
+	eg.Go(func() error {
+		return wait()
+	})
+
 	return &s, nil
 }
 

snat.go

@@ -1,91 +0,0 @@
-package nebula
-
-import (
-	"encoding/binary"
-	"net/netip"
-)
-
-func recalcIPv4Checksum(data []byte, oldSrcIP netip.Addr, newSrcIP netip.Addr) {
-	oldChecksum := binary.BigEndian.Uint16(data[10:12])
-	//because of how checksums work, we can re-use this function
-	checksum := calcNewTransportChecksum(oldChecksum, oldSrcIP, 0, newSrcIP, 0)
-	binary.BigEndian.PutUint16(data[10:12], checksum)
-}
-
-func calcNewTransportChecksum(oldChecksum uint16, oldSrcIP netip.Addr, oldSrcPort uint16, newSrcIP netip.Addr, newSrcPort uint16) uint16 {
-	oldIP := binary.BigEndian.Uint32(oldSrcIP.AsSlice())
-	newIP := binary.BigEndian.Uint32(newSrcIP.AsSlice())
-
-	// Start with inverted checksum
-	checksum := uint32(^oldChecksum)
-
-	// Subtract old IP (as two 16-bit words)
-	checksum += uint32(^uint16(oldIP >> 16))
-	checksum += uint32(^uint16(oldIP & 0xFFFF))
-
-	// Subtract old port
-	checksum += uint32(^oldSrcPort)
-
-	// Add new IP (as two 16-bit words)
-	checksum += uint32(newIP >> 16)
-	checksum += uint32(newIP & 0xFFFF)
-
-	// Add new port
-	checksum += uint32(newSrcPort)
-
-	// Fold carries
-	for checksum > 0xFFFF {
-		checksum = (checksum & 0xFFFF) + (checksum >> 16)
-	}
-
-	// Return ones' complement
-	return ^uint16(checksum)
-}
-
-func recalcV4TransportChecksum(offsetInsideHeader int, data []byte, oldSrcIP netip.AddrPort, newSrcIP netip.AddrPort) {
-	ipHeaderOffset := int(data[0]&0x0F) * 4
-	offset := ipHeaderOffset + offsetInsideHeader
-	oldcsum := binary.BigEndian.Uint16(data[offset : offset+2])
-	checksum := calcNewTransportChecksum(oldcsum, oldSrcIP.Addr(), oldSrcIP.Port(), newSrcIP.Addr(), newSrcIP.Port())
-	binary.BigEndian.PutUint16(data[offset:offset+2], checksum)
-}
-
-func recalcUDPv4Checksum(data []byte, oldSrcIP netip.AddrPort, newSrcIP netip.AddrPort) {
-	const offsetInsideHeader = 6
-	recalcV4TransportChecksum(offsetInsideHeader, data, oldSrcIP, newSrcIP)
-}
-
-func recalcTCPv4Checksum(data []byte, oldSrcIP netip.AddrPort, newSrcIP netip.AddrPort) {
-	const offsetInsideHeader = 16
-	recalcV4TransportChecksum(offsetInsideHeader, data, oldSrcIP, newSrcIP)
-}
-
-func calcNewICMPChecksum(oldChecksum uint16, oldCode uint16, newCode uint16, oldID uint16, newID uint16) uint16 {
-	// Start with inverted checksum
-	checksum := uint32(^oldChecksum)
-
-	// Subtract old stuff
-	checksum += uint32(^oldCode)
-	checksum += uint32(^oldID)
-
-	// Add new stuff
-	checksum += uint32(newCode)
-	checksum += uint32(newID)
-
-	// Fold carries
-	for checksum > 0xFFFF {
-		checksum = (checksum & 0xFFFF) + (checksum >> 16)
-	}
-
-	// Return ones' complement
-	return ^uint16(checksum)
-}
-
-func recalcICMPv4Checksum(data []byte, oldCode uint16, newCode uint16, oldID uint16, newID uint16) {
-	const offsetInsideHeader = 2
-	ipHeaderOffset := int(data[0]&0x0F) * 4
-	offset := ipHeaderOffset + offsetInsideHeader
-	oldChecksum := binary.BigEndian.Uint16(data[offset : offset+2])
-	checksum := calcNewICMPChecksum(oldChecksum, oldCode, newCode, oldID, newID)
-	binary.BigEndian.PutUint16(data[offset:offset+2], checksum)
-}

ssh.go

@@ -10,6 +10,7 @@ import (
 	"net"
 	"net/netip"
 	"os"
+	"path/filepath"
 	"reflect"
 	"runtime"
 	"runtime/pprof"
@@ -188,6 +189,12 @@ func configSSH(l *logrus.Logger, ssh *sshd.SSHServer, c *config.C) (func(), erro
 }
 
 func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Interface) {
+	// sandboxDir defaults to a dir in temp. The intention is that end user will
+	// create this dir as needed. Overriding this config value to "" allows
+	// writing to anywhere in the system.
+	defaultDir := filepath.Join(os.TempDir(), "nebula-debug")
+	sandboxDir := c.GetString("sshd.sandbox_dir", defaultDir)
+
 	ssh.RegisterCommand(&sshd.Command{
 		Name:             "list-hostmap",
 		ShortDescription: "List all known previously connected hosts",
@@ -246,7 +253,9 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter
 	ssh.RegisterCommand(&sshd.Command{
 		Name:             "start-cpu-profile",
 		ShortDescription: "Starts a cpu profile and write output to the provided file, ex: `cpu-profile.pb.gz`",
-		Callback:         sshStartCpuProfile,
+		Callback: func(fs any, a []string, w sshd.StringWriter) error {
+			return sshStartCpuProfile(sandboxDir, fs, a, w)
+		},
 	})
 
 	ssh.RegisterCommand(&sshd.Command{
@@ -261,7 +270,9 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter
 	ssh.RegisterCommand(&sshd.Command{
 		Name:             "save-heap-profile",
 		ShortDescription: "Saves a heap profile to the provided path, ex: `heap-profile.pb.gz`",
-		Callback:         sshGetHeapProfile,
+		Callback: func(fs any, a []string, w sshd.StringWriter) error {
+			return sshGetHeapProfile(sandboxDir, fs, a, w)
+		},
 	})
 
 	ssh.RegisterCommand(&sshd.Command{
@@ -273,7 +284,9 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter
 	ssh.RegisterCommand(&sshd.Command{
 		Name:             "save-mutex-profile",
 		ShortDescription: "Saves a mutex profile to the provided path, ex: `mutex-profile.pb.gz`",
-		Callback:         sshGetMutexProfile,
+		Callback: func(fs any, a []string, w sshd.StringWriter) error {
+			return sshGetMutexProfile(sandboxDir, fs, a, w)
+		},
 	})
 
 	ssh.RegisterCommand(&sshd.Command{
@@ -506,13 +519,43 @@ func sshListLighthouseMap(lightHouse *LightHouse, a any, w sshd.StringWriter) er
 	return nil
 }
 
-func sshStartCpuProfile(fs any, a []string, w sshd.StringWriter) error {
+// sshSanitizeFilePath validates that the given file path is within the sandbox directory.
+// If sandboxDir is empty, the path is returned as-is for backwards compatibility.
+func sshSanitizeFilePath(sandboxDir, filePath string) (string, error) {
+	if sandboxDir == "" {
+		return filePath, nil
+	}
+
+	// Clean and resolve the path relative to the sandbox directory
+	if !filepath.IsAbs(filePath) {
+		filePath = filepath.Join(sandboxDir, filePath)
+	}
+	cleaned := filepath.Clean(filePath)
+
+	// Ensure the resolved path is within the sandbox directory
+	cleanedSandbox := filepath.Clean(sandboxDir)
+	if cleaned == cleanedSandbox {
+		return "", fmt.Errorf("path %q resolves to the sandbox directory itself %q", filePath, sandboxDir)
+	}
+	if !strings.HasPrefix(cleaned, cleanedSandbox+string(filepath.Separator)) {
+		return "", fmt.Errorf("path %q is outside the sandbox directory %q", filePath, sandboxDir)
+	}
+
+	return cleaned, nil
+}
+
+func sshStartCpuProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
 	if len(a) == 0 {
 		err := w.WriteLine("No path to write profile provided")
 		return err
 	}
 
-	file, err := os.Create(a[0])
+	filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
+	if err != nil {
+		return w.WriteLine(err.Error())
+	}
+
+	file, err := os.Create(filePath)
 	if err != nil {
 		err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
 		return err
@@ -676,12 +719,17 @@ func sshChangeRemote(ifce *Interface, fs any, a []string, w sshd.StringWriter) e
 	return w.WriteLine("Changed")
 }
 
-func sshGetHeapProfile(fs any, a []string, w sshd.StringWriter) error {
+func sshGetHeapProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
 	if len(a) == 0 {
 		return w.WriteLine("No path to write profile provided")
 	}
 
-	file, err := os.Create(a[0])
+	filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
+	if err != nil {
+		return w.WriteLine(err.Error())
+	}
+
+	file, err := os.Create(filePath)
 	if err != nil {
 		err = w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
 		return err
@@ -712,12 +760,17 @@ func sshMutexProfileFraction(fs any, a []string, w sshd.StringWriter) error {
 	return w.WriteLine(fmt.Sprintf("New value: %d. Old value: %d", newRate, oldRate))
 }
 
-func sshGetMutexProfile(fs any, a []string, w sshd.StringWriter) error {
+func sshGetMutexProfile(sandboxDir string, fs any, a []string, w sshd.StringWriter) error {
 	if len(a) == 0 {
 		return w.WriteLine("No path to write profile provided")
 	}
 
-	file, err := os.Create(a[0])
+	filePath, err := sshSanitizeFilePath(sandboxDir, a[0])
+	if err != nil {
+		return w.WriteLine(err.Error())
+	}
+
+	file, err := os.Create(filePath)
 	if err != nil {
 		return w.WriteLine(fmt.Sprintf("Unable to create profile file: %s", err))
 	}

test/tun.go

@@ -10,18 +10,6 @@ import (
 
 type NoopTun struct{}
 
-func (NoopTun) UnsafeNetworks() []netip.Prefix {
-	return nil
-}
-
-func (NoopTun) SNATAddress() netip.Prefix {
-	return netip.Prefix{}
-}
-
-func (NoopTun) UnsafeIPv4OriginAddress() netip.Prefix {
-	return netip.Prefix{}
-}
-
 func (NoopTun) RoutesFor(addr netip.Addr) routing.Gateways {
 	return routing.Gateways{}
 }

udp/conn.go

@@ -16,7 +16,7 @@ type EncReader func(
 type Conn interface {
 	Rebind() error
 	LocalAddr() (netip.AddrPort, error)
-	ListenOut(r EncReader)
+	ListenOut(r EncReader) error
 	WriteTo(b []byte, addr netip.AddrPort) error
 	ReloadConfig(c *config.C)
 	SupportsMultipleReaders() bool
@@ -31,8 +31,8 @@ func (NoopConn) Rebind() error {
 func (NoopConn) LocalAddr() (netip.AddrPort, error) {
 	return netip.AddrPort{}, nil
 }
-func (NoopConn) ListenOut(_ EncReader) {
+func (NoopConn) ListenOut(_ EncReader) error {
-	return
+	return nil
 }
 func (NoopConn) SupportsMultipleReaders() bool {
 	return false

udp/udp_darwin.go

@@ -165,7 +165,7 @@ func NewUDPStatsEmitter(udpConns []Conn) func() {
 	return func() {}
 }
 
-func (u *StdConn) ListenOut(r EncReader) {
+func (u *StdConn) ListenOut(r EncReader) error {
 	buffer := make([]byte, MTU)
 
 	for {
@@ -173,8 +173,7 @@ func (u *StdConn) ListenOut(r EncReader) {
 		n, rua, err := u.ReadFromUDPAddrPort(buffer)
 		if err != nil {
 			if errors.Is(err, net.ErrClosed) {
-				u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
+				return err
-				return
 			}
 
 			u.l.WithError(err).Error("unexpected udp socket receive error")

udp/udp_generic.go

@@ -73,7 +73,7 @@ type rawMessage struct {
 	Len uint32
 }
 
-func (u *GenericConn) ListenOut(r EncReader) {
+func (u *GenericConn) ListenOut(r EncReader) error {
 	buffer := make([]byte, MTU)
 
 	var lastRecvErr time.Time
@@ -83,8 +83,7 @@ func (u *GenericConn) ListenOut(r EncReader) {
 		n, rua, err := u.ReadFromUDPAddrPort(buffer)
 		if err != nil {
 			if errors.Is(err, net.ErrClosed) {
-				u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
+				return err
-				return
 			}
 			// Dampen unexpected message warns to once per minute
 			if lastRecvErr.IsZero() || time.Since(lastRecvErr) > time.Minute {

udp/udp_linux.go

@@ -4,6 +4,7 @@
 package udp
 
 import (
+	"context"
 	"encoding/binary"
 	"fmt"
 	"net"
@@ -18,58 +19,58 @@ import (
 )
 
 type StdConn struct {
-	sysFd int
+	udpConn *net.UDPConn
-	isV4  bool
+	rawConn syscall.RawConn
-	l     *logrus.Logger
+	isV4    bool
-	batch int
+	l       *logrus.Logger
+	batch   int
 }
 
-func maybeIPV4(ip net.IP) (net.IP, bool) {
+func setReusePort(network, address string, c syscall.RawConn) error {
-	ip4 := ip.To4()
+	var opErr error
-	if ip4 != nil {
+	err := c.Control(func(fd uintptr) {
-		return ip4, true
+		opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
+		//CloseOnExec already set by the runtime
+	})
+	if err != nil {
+		return err
 	}
-	return ip, false
+	return opErr
 }
 
 func NewListener(l *logrus.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
-	af := unix.AF_INET6
+	listen := netip.AddrPortFrom(ip, uint16(port))
-	if ip.Is4() {
+	lc := net.ListenConfig{}
-		af = unix.AF_INET
+	if multi {
+		lc.Control = setReusePort
 	}
-	syscall.ForkLock.RLock()
+	//this context is only used during the bind operation, you can't cancel it to kill the socket
-	fd, err := unix.Socket(af, unix.SOCK_DGRAM, unix.IPPROTO_UDP)
+	pc, err := lc.ListenPacket(context.Background(), "udp", listen.String())
-	if err == nil {
-		unix.CloseOnExec(fd)
-	}
-	syscall.ForkLock.RUnlock()
-
 	if err != nil {
-		unix.Close(fd)
 		return nil, fmt.Errorf("unable to open socket: %s", err)
 	}
-
+	udpConn := pc.(*net.UDPConn)
-	if multi {
+	rawConn, err := udpConn.SyscallConn()
-		if err = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
+	if err != nil {
-			return nil, fmt.Errorf("unable to set SO_REUSEPORT: %s", err)
+		_ = udpConn.Close()
-		}
+		return nil, err
+	}
+	//gotta find out if we got an AF_INET6 socket or not:
+	out := &StdConn{
+		udpConn: udpConn,
+		rawConn: rawConn,
+		l:       l,
+		batch:   batch,
 	}
 
-	var sa unix.Sockaddr
+	af, err := out.getSockOptInt(unix.SO_DOMAIN)
-	if ip.Is4() {
+	if err != nil {
-		sa4 := &unix.SockaddrInet4{Port: port}
+		_ = out.Close()
-		sa4.Addr = ip.As4()
+		return nil, err
-		sa = sa4
-	} else {
-		sa6 := &unix.SockaddrInet6{Port: port}
-		sa6.Addr = ip.As16()
-		sa = sa6
-	}
-	if err = unix.Bind(fd, sa); err != nil {
-		return nil, fmt.Errorf("unable to bind to socket: %s", err)
 	}
+	out.isV4 = af == unix.AF_INET
 
-	return &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch}, err
+	return out, nil
 }
 
 func (u *StdConn) SupportsMultipleReaders() bool {
@@ -80,62 +81,133 @@ func (u *StdConn) Rebind() error {
 	return nil
 }
 
+func (u *StdConn) getSockOptInt(opt int) (int, error) {
+	if u.rawConn == nil {
+		return 0, fmt.Errorf("no UDP connection")
+	}
+	var out int
+	var opErr error
+	err := u.rawConn.Control(func(fd uintptr) {
+		out, opErr = unix.GetsockoptInt(int(fd), unix.SOL_SOCKET, opt)
+	})
+	if err != nil {
+		return 0, err
+	}
+	return out, opErr
+}
+
+func (u *StdConn) setSockOptInt(opt int, n int) error {
+	if u.rawConn == nil {
+		return fmt.Errorf("no UDP connection")
+	}
+	var opErr error
+	err := u.rawConn.Control(func(fd uintptr) {
+		opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, opt, n)
+	})
+	if err != nil {
+		return err
+	}
+	return opErr
+}
+
 func (u *StdConn) SetRecvBuffer(n int) error {
-	return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUFFORCE, n)
+	return u.setSockOptInt(unix.SO_RCVBUFFORCE, n)
 }
 
 func (u *StdConn) SetSendBuffer(n int) error {
-	return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUFFORCE, n)
+	return u.setSockOptInt(unix.SO_SNDBUFFORCE, n)
 }
 
 func (u *StdConn) SetSoMark(mark int) error {
-	return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK, mark)
+	return u.setSockOptInt(unix.SO_MARK, mark)
 }
 
 func (u *StdConn) GetRecvBuffer() (int, error) {
-	return unix.GetsockoptInt(int(u.sysFd), unix.SOL_SOCKET, unix.SO_RCVBUF)
+	return u.getSockOptInt(unix.SO_RCVBUF)
 }
 
 func (u *StdConn) GetSendBuffer() (int, error) {
-	return unix.GetsockoptInt(int(u.sysFd), unix.SOL_SOCKET, unix.SO_SNDBUF)
+	return u.getSockOptInt(unix.SO_SNDBUF)
 }
 
 func (u *StdConn) GetSoMark() (int, error) {
-	return unix.GetsockoptInt(int(u.sysFd), unix.SOL_SOCKET, unix.SO_MARK)
+	return u.getSockOptInt(unix.SO_MARK)
 }
 
 func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
-	sa, err := unix.Getsockname(u.sysFd)
+	a := u.udpConn.LocalAddr()
-	if err != nil {
-		return netip.AddrPort{}, err
-	}
 
-	switch sa := sa.(type) {
+	switch v := a.(type) {
-	case *unix.SockaddrInet4:
+	case *net.UDPAddr:
-		return netip.AddrPortFrom(netip.AddrFrom4(sa.Addr), uint16(sa.Port)), nil
+		addr, ok := netip.AddrFromSlice(v.IP)
-
+		if !ok {
-	case *unix.SockaddrInet6:
+			return netip.AddrPort{}, fmt.Errorf("LocalAddr returned invalid IP address: %s", v.IP)
-		return netip.AddrPortFrom(netip.AddrFrom16(sa.Addr), uint16(sa.Port)), nil
+		}
+		return netip.AddrPortFrom(addr, uint16(v.Port)), nil
 
 	default:
-		return netip.AddrPort{}, fmt.Errorf("unsupported sock type: %T", sa)
+		return netip.AddrPort{}, fmt.Errorf("LocalAddr returned: %#v", a)
 	}
 }
 
-func (u *StdConn) ListenOut(r EncReader) {
+func recvmmsg(fd uintptr, msgs []rawMessage) (int, bool, error) {
+	var errno syscall.Errno
+	n, _, errno := unix.Syscall6(
+		unix.SYS_RECVMMSG,
+		fd,
+		uintptr(unsafe.Pointer(&msgs[0])),
+		uintptr(len(msgs)),
+		unix.MSG_WAITFORONE,
+		0,
+		0,
+	)
+	if errno == syscall.EAGAIN || errno == syscall.EWOULDBLOCK {
+		// No data available, block for I/O and try again.
+		return int(n), false, nil
+	}
+	if errno != 0 {
+		return int(n), true, &net.OpError{Op: "recvmmsg", Err: errno}
+	}
+	return int(n), true, nil
+}
+
+func (u *StdConn) listenOutSingle(r EncReader) error {
+	var err error
+	var n int
+	var from netip.AddrPort
+	buffer := make([]byte, MTU)
+
+	for {
+		n, from, err = u.udpConn.ReadFromUDPAddrPort(buffer)
+		if err != nil {
+			return err
+		}
+		from = netip.AddrPortFrom(from.Addr().Unmap(), from.Port())
+		r(from, buffer[:n])
+	}
+}
+
+func (u *StdConn) listenOutBatch(r EncReader) error {
 	var ip netip.Addr
+	var n int
+	var operr error
 
 	msgs, buffers, names := u.PrepareRawMessages(u.batch)
-	read := u.ReadMulti
+
-	if u.batch == 1 {
+	//reader needs to capture variables from this function, since it's used as a lambda with rawConn.Read
-		read = u.ReadSingle
+	//defining it outside the loop so it gets re-used
+	reader := func(fd uintptr) (done bool) {
+		n, done, operr = recvmmsg(fd, msgs)
+		return done
 	}
 
 	for {
-		n, err := read(msgs)
+		err := u.rawConn.Read(reader)
 		if err != nil {
-			u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
+			return err
-			return
+		}
+		if operr != nil {
+			return operr
 		}
 
 		for i := 0; i < n; i++ {
@@ -150,106 +222,17 @@ func (u *StdConn) ListenOut(r EncReader) {
 	}
 }
 
-func (u *StdConn) ReadSingle(msgs []rawMessage) (int, error) {
+func (u *StdConn) ListenOut(r EncReader) error {
-	for {
+	if u.batch == 1 {
-		n, _, err := unix.Syscall6(
+		return u.listenOutSingle(r)
-			unix.SYS_RECVMSG,
+	} else {
-			uintptr(u.sysFd),
+		return u.listenOutBatch(r)
-			uintptr(unsafe.Pointer(&(msgs[0].Hdr))),
-			0,
-			0,
-			0,
-			0,
-		)
-
-		if err != 0 {
-			return 0, &net.OpError{Op: "recvmsg", Err: err}
-		}
-
-		msgs[0].Len = uint32(n)
-		return 1, nil
-	}
-}
-
-func (u *StdConn) ReadMulti(msgs []rawMessage) (int, error) {
-	for {
-		n, _, err := unix.Syscall6(
-			unix.SYS_RECVMMSG,
-			uintptr(u.sysFd),
-			uintptr(unsafe.Pointer(&msgs[0])),
-			uintptr(len(msgs)),
-			unix.MSG_WAITFORONE,
-			0,
-			0,
-		)
-
-		if err != 0 {
-			return 0, &net.OpError{Op: "recvmmsg", Err: err}
-		}
-
-		return int(n), nil
 	}
 }
 
 func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
-	if u.isV4 {
+	_, err := u.udpConn.WriteToUDPAddrPort(b, ip)
-		return u.writeTo4(b, ip)
+	return err
-	}
-	return u.writeTo6(b, ip)
-}
-
-func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
-	var rsa unix.RawSockaddrInet6
-	rsa.Family = unix.AF_INET6
-	rsa.Addr = ip.Addr().As16()
-	binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
-
-	for {
-		_, _, err := unix.Syscall6(
-			unix.SYS_SENDTO,
-			uintptr(u.sysFd),
-			uintptr(unsafe.Pointer(&b[0])),
-			uintptr(len(b)),
-			uintptr(0),
-			uintptr(unsafe.Pointer(&rsa)),
-			uintptr(unix.SizeofSockaddrInet6),
-		)
-
-		if err != 0 {
-			return &net.OpError{Op: "sendto", Err: err}
-		}
-
-		return nil
-	}
-}
-
-func (u *StdConn) writeTo4(b []byte, ip netip.AddrPort) error {
-	if !ip.Addr().Is4() {
-		return ErrInvalidIPv6RemoteForSocket
-	}
-
-	var rsa unix.RawSockaddrInet4
-	rsa.Family = unix.AF_INET
-	rsa.Addr = ip.Addr().As4()
-	binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
-
-	for {
-		_, _, err := unix.Syscall6(
-			unix.SYS_SENDTO,
-			uintptr(u.sysFd),
-			uintptr(unsafe.Pointer(&b[0])),
-			uintptr(len(b)),
-			uintptr(0),
-			uintptr(unsafe.Pointer(&rsa)),
-			uintptr(unix.SizeofSockaddrInet4),
-		)
-
-		if err != 0 {
-			return &net.OpError{Op: "sendto", Err: err}
-		}
-
-		return nil
-	}
 }
 
 func (u *StdConn) ReloadConfig(c *config.C) {
@@ -302,15 +285,28 @@ func (u *StdConn) ReloadConfig(c *config.C) {
 
 func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
 	var vallen uint32 = 4 * unix.SK_MEMINFO_VARS
-	_, _, err := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(u.sysFd), uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
+
-	if err != 0 {
+	if u.rawConn == nil {
+		return fmt.Errorf("no UDP connection")
+	}
+	var opErr error
+	err := u.rawConn.Control(func(fd uintptr) {
+		_, _, syserr := unix.Syscall6(unix.SYS_GETSOCKOPT, fd, uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
+		if syserr != 0 {
+			opErr = syserr
+		}
+	})
+	if err != nil {
 		return err
 	}
-	return nil
+	return opErr
 }
 
 func (u *StdConn) Close() error {
-	return syscall.Close(u.sysFd)
+	if u.udpConn != nil {
+		return u.udpConn.Close()
+	}
+	return nil
 }
 
 func NewUDPStatsEmitter(udpConns []Conn) func() {

udp/udp_rio_windows.go

@@ -140,7 +140,7 @@ func (u *RIOConn) bind(l *logrus.Logger, sa windows.Sockaddr) error {
 	return nil
 }
 
-func (u *RIOConn) ListenOut(r EncReader) {
+func (u *RIOConn) ListenOut(r EncReader) error {
 	buffer := make([]byte, MTU)
 
 	var lastRecvErr time.Time
@@ -151,8 +151,7 @@ func (u *RIOConn) ListenOut(r EncReader) {
 
 		if err != nil {
 			if errors.Is(err, net.ErrClosed) {
-				u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
+				return err
-				return
 			}
 			// Dampen unexpected message warns to once per minute
 			if lastRecvErr.IsZero() || time.Since(lastRecvErr) > time.Minute {

udp/udp_tester.go

@@ -6,6 +6,7 @@ package udp
 import (
 	"io"
 	"net/netip"
+	"os"
 	"sync/atomic"
 
 	"github.com/sirupsen/logrus"
@@ -106,11 +107,11 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
 	return nil
 }
 
-func (u *TesterConn) ListenOut(r EncReader) {
+func (u *TesterConn) ListenOut(r EncReader) error {
 	for {
 		p, ok := <-u.RxPackets
 		if !ok {
-			return
+			return os.ErrClosed
 		}
 		r(p.From, p.Data)
 	}