Experimenting with fw event reporting

.github/workflows/smoke-extra.yml

@@ -18,6 +18,8 @@ jobs:
     if: github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'smoke-test-extra')
     name: Run extra smoke tests
     runs-on: ubuntu-latest
+    env:
+      VAGRANT_DEFAULT_PROVIDER: libvirt
     steps:
 
     - uses: actions/checkout@v6
@@ -30,11 +32,13 @@ jobs:
     - name: add hashicorp source
       run: wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg && echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
 
-    - name: workaround AMD-V issue  # https://github.com/cri-o/packaging/pull/306
+    - name: install vagrant and libvirt
-      run: sudo rmmod kvm_amd
+      run: |
-
+        sudo apt-get update && sudo apt-get install -y vagrant libvirt-daemon-system libvirt-dev
-    - name: install vagrant
+        sudo chmod 666 /dev/kvm
-      run: sudo apt-get update && sudo apt-get install -y vagrant virtualbox
+        sudo usermod -aG libvirt $(whoami)
+        sudo chmod 666 /var/run/libvirt/libvirt-sock
+        vagrant plugin install vagrant-libvirt
 
     - name: freebsd-amd64
       run: make smoke-vagrant/freebsd-amd64
@@ -45,10 +49,19 @@ jobs:
     - name: netbsd-amd64
       run: make smoke-vagrant/netbsd-amd64
 
-    - name: linux-386
-      run: make smoke-vagrant/linux-386
-
     - name: linux-amd64-ipv6disable
       run: make smoke-vagrant/linux-amd64-ipv6disable
 
+    # linux-386 runs last because it requires disabling KVM to use VirtualBox,
+    # which prevents libvirt (used by the other tests) from working after this point.
+    - name: install virtualbox for i386 test
+      run: |
+        sudo apt-get install -y virtualbox
+        sudo rmmod kvm_amd kvm_intel kvm 2>/dev/null || true
+
+    - name: linux-386
+      env:
+        VAGRANT_DEFAULT_PROVIDER: virtualbox
+      run: make smoke-vagrant/linux-386
+
     timeout-minutes: 30

.github/workflows/smoke/build-relay.sh

@@ -16,8 +16,10 @@ relay:
   am_relay: true
 EOF
 
-    export LIGHTHOUSES="192.168.100.1 172.17.0.2:4242"
+    # TEST-NET-3 placeholder IPs; smoke-relay.sh seds them to real container IPs.
-    export REMOTE_ALLOW_LIST='{"172.17.0.4/32": false, "172.17.0.5/32": false}'
+    # Mapping: .2 lighthouse1, .3 host2, .4 host3, .5 host4.
+    export LIGHTHOUSES="192.168.100.1 203.0.113.2:4242"
+    export REMOTE_ALLOW_LIST='{"203.0.113.4/32": false, "203.0.113.5/32": false}'
 
     HOST="host2" ../genconfig.sh >host2.yml <<EOF
 relay:
@@ -25,7 +27,7 @@ relay:
     - 192.168.100.1
 EOF
 
-    export REMOTE_ALLOW_LIST='{"172.17.0.3/32": false}'
+    export REMOTE_ALLOW_LIST='{"203.0.113.3/32": false}'
 
     HOST="host3" ../genconfig.sh >host3.yml
 

.github/workflows/smoke/build.sh

@@ -5,9 +5,15 @@ set -e -x
 rm -rf ./build
 mkdir ./build
 
-# TODO: Assumes your docker bridge network is a /24, and the first container that launches will be .1
+# Smoke containers run on a dedicated docker network whose subnet is allocated
-# - We could make this better by launching the lighthouse first and then fetching what IP it is.
+# at smoke time, not known at build time. Configs are written with TEST-NET-3
-NET="$(docker network inspect bridge -f '{{ range .IPAM.Config }}{{ .Subnet }}{{ end }}' | cut -d. -f1-3)"
+# placeholder IPs (RFC 5737) and smoke.sh / smoke-vagrant.sh / smoke-relay.sh
+# sed the real container IPs in before starting nebula.
+#
+# Placeholder mapping (last octet == fixed container slot):
+#   203.0.113.2 -> lighthouse1, 203.0.113.3 -> host2,
+#   203.0.113.4 -> host3,       203.0.113.5 -> host4.
+LIGHTHOUSE_IP="203.0.113.2"
 
 (
     cd build
@@ -25,16 +31,16 @@ NET="$(docker network inspect bridge -f '{{ range .IPAM.Config }}{{ .Subnet }}{{
         ../genconfig.sh >lighthouse1.yml
 
     HOST="host2" \
-        LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
+        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
         ../genconfig.sh >host2.yml
 
     HOST="host3" \
-        LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
+        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
         INBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
         ../genconfig.sh >host3.yml
 
     HOST="host4" \
-        LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
+        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
         OUTBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
         ../genconfig.sh >host4.yml
 

.github/workflows/smoke/smoke-relay.sh

@@ -6,6 +6,8 @@ set -o pipefail
 
 mkdir -p logs
 
+NETWORK="nebula-smoke-relay"
+
 cleanup() {
     echo
     echo " *** cleanup"
@@ -16,22 +18,53 @@ cleanup() {
     then
         docker kill lighthouse1 host2 host3 host4
     fi
+    docker network rm "$NETWORK" >/dev/null 2>&1
 }
 
 trap cleanup EXIT
 
-docker run --name lighthouse1 --rm nebula:smoke-relay -config lighthouse1.yml -test
+# Create a dedicated smoke network with an explicit subnet (required for --ip
-docker run --name host2 --rm nebula:smoke-relay -config host2.yml -test
+# below). Probe a short list of candidates so a locally-used range doesn't
-docker run --name host3 --rm nebula:smoke-relay -config host3.yml -test
+# fail the whole test — we only need one to be free.
-docker run --name host4 --rm nebula:smoke-relay -config host4.yml -test
+docker network rm "$NETWORK" >/dev/null 2>&1 || true
+for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
+    if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
+        break
+    fi
+done
+if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
+    echo "failed to create $NETWORK: every candidate subnet is in use" >&2
+    exit 1
+fi
 
-docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
+# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
+# .3 host2, .4 host3, .5 host4 — matches the placeholders in build-relay.sh.
+SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
+PREFIX="${SUBNET%/*}"
+PREFIX="${PREFIX%.*}"
+LIGHTHOUSE_IP="$PREFIX.2"
+HOST2_IP="$PREFIX.3"
+HOST3_IP="$PREFIX.4"
+HOST4_IP="$PREFIX.5"
+
+# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
+for f in build/host2.yml build/host3.yml build/host4.yml; do
+    sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
+    mv "$f.tmp" "$f"
+done
+
+docker run --name lighthouse1 --rm nebula:smoke-relay -config lighthouse1.yml -test
+docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" nebula:smoke-relay -config host2.yml -test
+docker run --name host3 --rm -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" nebula:smoke-relay -config host3.yml -test
+docker run --name host4 --rm -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" nebula:smoke-relay -config host4.yml -test
+
+docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
 sleep 1
-docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
+docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
 sleep 1
-docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
+docker run --name host3 --network "$NETWORK" --ip "$HOST3_IP" -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
 sleep 1
-docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
+docker run --name host4 --network "$NETWORK" --ip "$HOST4_IP" -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
 sleep 1
 
 set +x
@@ -76,7 +109,13 @@ docker exec host4 sh -c 'kill 1'
 docker exec host3 sh -c 'kill 1'
 docker exec host2 sh -c 'kill 1'
 docker exec lighthouse1 sh -c 'kill 1'
-sleep 5
+
+# Wait up to 30s for all backgrounded jobs to exit rather than relying on a
+# fixed sleep.
+for _ in $(seq 1 30); do
+    [ -z "$(jobs -r)" ] && break
+    sleep 1
+done
 
 if [ "$(jobs -r)" ]
 then

.github/workflows/smoke/smoke-vagrant.sh

@@ -8,6 +8,8 @@ export VAGRANT_CWD="$PWD/vagrant-$1"
 
 mkdir -p logs
 
+NETWORK="nebula-smoke"
+
 cleanup() {
     echo
     echo " *** cleanup"
@@ -19,32 +21,51 @@ cleanup() {
         docker kill lighthouse1 host2
     fi
     vagrant destroy -f
+    docker network rm "$NETWORK" >/dev/null 2>&1
 }
 
 trap cleanup EXIT
 
+# Create a dedicated smoke network with an explicit subnet (required for --ip
+# below). Probe a short list of candidates so a locally-used range doesn't
+# fail the whole test — we only need one to be free.
+docker network rm "$NETWORK" >/dev/null 2>&1 || true
+for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
+    if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
+        break
+    fi
+done
+if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
+    echo "failed to create $NETWORK: every candidate subnet is in use" >&2
+    exit 1
+fi
+
+# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
+# .3 host2 — matches the placeholders in build.sh.
+SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
+PREFIX="${SUBNET%/*}"
+PREFIX="${PREFIX%.*}"
+LIGHTHOUSE_IP="$PREFIX.2"
+HOST2_IP="$PREFIX.3"
+
+# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
+# This must happen before `vagrant up` rsyncs build/ into the VM for host3.
+for f in build/host2.yml build/host3.yml; do
+    sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
+    mv "$f.tmp" "$f"
+done
+
 CONTAINER="nebula:${NAME:-smoke}"
 
 docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test
-docker run --name host2 --rm "$CONTAINER" -config host2.yml -test
+docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" "$CONTAINER" -config host2.yml -test
 
 vagrant up
-
-# OpenBSD: synced folders are disabled because Vagrant's rsync installer
-# uses ftp.openbsd.org which no longer hosts packages for older releases.
-# Copy build artifacts in via scp instead.
-case "$1" in
-    openbsd-*)
-        vagrant ssh -c "sudo mkdir -p /nebula" -- -T
-        tar -cf - -C build . | vagrant ssh -c "sudo tar -xf - -C /nebula && sudo chmod -R a+r /nebula" -- -T
-        ;;
-esac
-
 vagrant ssh -c "cd /nebula && /nebula/$1-nebula -config host3.yml -test" -- -T
 
-docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
+docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
 sleep 1
-docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
+docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
 sleep 1
 vagrant ssh -c "cd /nebula && sudo sh -c 'echo \$\$ >/nebula/pid && exec /nebula/$1-nebula -config host3.yml'" 2>&1 -- -T | tee logs/host3 | sed -u 's/^/  [host3]  /' &
 sleep 15
@@ -107,7 +128,14 @@ vagrant ssh -c "ping -c1 192.168.100.2" -- -T
 vagrant ssh -c "sudo xargs kill </nebula/pid" -- -T
 docker exec host2 sh -c 'kill 1'
 docker exec lighthouse1 sh -c 'kill 1'
-sleep 1
+
+# Wait up to 30s for all backgrounded jobs to exit. vagrant ssh in particular
+# takes a beat to tear down after nebula exits on the VM, so a fixed sleep is
+# racy.
+for _ in $(seq 1 30); do
+    [ -z "$(jobs -r)" ] && break
+    sleep 1
+done
 
 if [ "$(jobs -r)" ]
 then

.github/workflows/smoke/smoke.sh

@@ -6,6 +6,8 @@ set -o pipefail
 
 mkdir -p logs
 
+NETWORK="nebula-smoke"
+
 cleanup() {
     echo
     echo " *** cleanup"
@@ -16,24 +18,56 @@ cleanup() {
     then
         docker kill lighthouse1 host2 host3 host4
     fi
+    docker network rm "$NETWORK" >/dev/null 2>&1
 }
 
 trap cleanup EXIT
 
+# Create a dedicated smoke network with an explicit subnet (required for --ip
+# below). Probe a short list of candidates so a locally-used range doesn't
+# fail the whole test — we only need one to be free.
+docker network rm "$NETWORK" >/dev/null 2>&1 || true
+for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
+    if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
+        break
+    fi
+done
+if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
+    echo "failed to create $NETWORK: every candidate subnet is in use" >&2
+    exit 1
+fi
+
+# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
+# .3 host2, .4 host3, .5 host4 — matches the placeholders in build.sh.
+SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
+PREFIX="${SUBNET%/*}"
+PREFIX="${PREFIX%.*}"
+LIGHTHOUSE_IP="$PREFIX.2"
+HOST2_IP="$PREFIX.3"
+HOST3_IP="$PREFIX.4"
+HOST4_IP="$PREFIX.5"
+
+# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
+# build/lighthouse1.yml has no IPs to rewrite so it's skipped.
+for f in build/host2.yml build/host3.yml build/host4.yml; do
+    sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
+    mv "$f.tmp" "$f"
+done
+
 CONTAINER="nebula:${NAME:-smoke}"
 
 docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test
-docker run --name host2 --rm "$CONTAINER" -config host2.yml -test
+docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" "$CONTAINER" -config host2.yml -test
-docker run --name host3 --rm "$CONTAINER" -config host3.yml -test
+docker run --name host3 --rm -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" "$CONTAINER" -config host3.yml -test
-docker run --name host4 --rm "$CONTAINER" -config host4.yml -test
+docker run --name host4 --rm -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" "$CONTAINER" -config host4.yml -test
 
-docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
+docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
 sleep 1
-docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
+docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
 sleep 1
-docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
+docker run --name host3 --network "$NETWORK" --ip "$HOST3_IP" -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
 sleep 1
-docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
+docker run --name host4 --network "$NETWORK" --ip "$HOST4_IP" -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
 sleep 1
 
 # grab tcpdump pcaps for debugging
@@ -131,7 +165,13 @@ docker exec host4 sh -c 'kill 1'
 docker exec host3 sh -c 'kill 1'
 docker exec host2 sh -c 'kill 1'
 docker exec lighthouse1 sh -c 'kill 1'
-sleep 5
+
+# Wait up to 30s for all backgrounded jobs to exit rather than relying on a
+# fixed sleep.
+for _ in $(seq 1 30); do
+    [ -z "$(jobs -r)" ] && break
+    sleep 1
+done
 
 if [ "$(jobs -r)" ]
 then

.github/workflows/smoke/vagrant-linux-amd64-ipv6disable/Vagrantfile

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 Vagrant.configure("2") do |config|
-  config.vm.box = "ubuntu/jammy64"
+  config.vm.box = "bento/ubuntu-24.04"
 
   config.vm.synced_folder "../build", "/nebula"
 

.github/workflows/smoke/vagrant-openbsd-amd64/Vagrantfile

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 Vagrant.configure("2") do |config|
-  config.vm.box = "generic/openbsd7"
+  config.vm.box = "DefinedNet/openbsd78"
 
-  config.vm.synced_folder ".", "/vagrant", disabled: true
+  config.vm.synced_folder "../build", "/nebula", type: "rsync"
 end

cmd/nebula-service/main.go

@@ -50,9 +50,15 @@ func main() {
 		os.Exit(0)
 	}
 
+	l := logrus.New()
+	l.Out = os.Stdout
+
 	if *serviceFlag != "" {
-		doService(configPath, configTest, Build, serviceFlag)
+		if err := doService(configPath, configTest, Build, serviceFlag); err != nil {
-		os.Exit(1)
+			l.WithError(err).Error("Service command failed")
+			os.Exit(1)
+		}
+		return
 	}
 
 	if *configPath == "" {
@@ -61,9 +67,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	l := logrus.New()
-	l.Out = os.Stdout
-
 	c := config.NewC(l)
 	err := c.Load(*configPath)
 	if err != nil {
@@ -78,8 +81,20 @@ func main() {
 	}
 
 	if !*configTest {
-		ctrl.Start()
+		wait, err := ctrl.Start()
-		ctrl.ShutdownBlock()
+		if err != nil {
+			util.LogWithContextIfNeeded("Error while running", err, l)
+			os.Exit(1)
+		}
+
+		go ctrl.ShutdownBlock()
+
+		if err := wait(); err != nil {
+			l.WithError(err).Error("Nebula stopped due to fatal error")
+			os.Exit(2)
+		}
+
+		l.Info("Goodbye")
 	}
 
 	os.Exit(0)

cmd/nebula-service/service.go

@@ -57,11 +57,11 @@ func fileExists(filename string) bool {
 	return true
 }
 
-func doService(configPath *string, configTest *bool, build string, serviceFlag *string) {
+func doService(configPath *string, configTest *bool, build string, serviceFlag *string) error {
 	if *configPath == "" {
 		ex, err := os.Executable()
 		if err != nil {
-			panic(err)
+			return err
 		}
 		*configPath = filepath.Dir(ex) + "/config.yaml"
 		if !fileExists(*configPath) {
@@ -88,13 +88,13 @@ func doService(configPath *string, configTest *bool, build string, serviceFlag *
 	// - above, in `Run` we create a `logrus.Logger` which is what nebula expects to use
 	s, err := service.New(prg, svcConfig)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 
 	errs := make(chan error, 5)
 	logger, err = s.Logger(errs)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 
 	go func() {
@@ -109,18 +109,16 @@ func doService(configPath *string, configTest *bool, build string, serviceFlag *
 
 	switch *serviceFlag {
 	case "run":
-		err = s.Run()
+		if err := s.Run(); err != nil {
-		if err != nil {
 			// Route any errors to the system logger
 			logger.Error(err)
 		}
 	default:
-		err := service.Control(s, *serviceFlag)
+		if err := service.Control(s, *serviceFlag); err != nil {
-		if err != nil {
 			log.Printf("Valid actions: %q\n", service.ControlAction)
-			log.Fatal(err)
+			return err
 		}
-		return
 	}
 
+	return nil
 }

cmd/nebula/main.go

@@ -72,9 +72,21 @@ func main() {
 	}
 
 	if !*configTest {
-		ctrl.Start()
+		wait, err := ctrl.Start()
+		if err != nil {
+			util.LogWithContextIfNeeded("Error while running", err, l)
+			os.Exit(1)
+		}
+
+		go ctrl.ShutdownBlock()
 		notifyReady(l)
-		ctrl.ShutdownBlock()
+
+		if err := wait(); err != nil {
+			l.WithError(err).Error("Nebula stopped due to fatal error")
+			os.Exit(2)
+		}
+
+		l.Info("Goodbye")
 	}
 
 	os.Exit(0)

control.go

@@ -2,17 +2,34 @@ package nebula
 
 import (
 	"context"
+	"errors"
 	"net/netip"
 	"os"
 	"os/signal"
+	"sync"
 	"syscall"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/firewall/events"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/overlay"
 )
 
+type RunState int
+
+const (
+	StateUnknown RunState = iota
+	StateReady
+	StateStarted
+	StateStopping
+	StateStopped
+)
+
+var ErrAlreadyStarted = errors.New("nebula is already started")
+var ErrAlreadyStopped = errors.New("nebula cannot be restarted")
+var ErrUnknownState = errors.New("nebula state is invalid")
+
 // Every interaction here needs to take extra care to copy memory and not return or use arguments "as is" when touching
 // core. This means copying IP objects, slices, de-referencing pointers and taking the actual value, etc
 
@@ -26,6 +43,9 @@ type controlHostLister interface {
 }
 
 type Control struct {
+	stateLock sync.Mutex
+	state     RunState
+
 	f                      *Interface
 	l                      *logrus.Logger
 	ctx                    context.Context
@@ -49,10 +69,31 @@ type ControlHostInfo struct {
 	CurrentRelaysThroughMe []netip.Addr     `json:"currentRelaysThroughMe"`
 }
 
-// Start actually runs nebula, this is a nonblocking call. To block use Control.ShutdownBlock()
+// Start actually runs nebula, this is a nonblocking call.
-func (c *Control) Start() {
+// The returned function blocks until nebula has fully stopped and returns the
+// first fatal reader error (if any). A nil error means nebula shut down
+// gracefully; a non-nil error means a reader hit an unexpected failure that
+// triggered the shutdown.
+func (c *Control) Start() (func() error, error) {
+	c.stateLock.Lock()
+	defer c.stateLock.Unlock()
+	switch c.state {
+	case StateReady:
+		//yay!
+	case StateStopped, StateStopping:
+		return nil, ErrAlreadyStopped
+	case StateStarted:
+		return nil, ErrAlreadyStarted
+	default:
+		return nil, ErrUnknownState
+	}
+
 	// Activate the interface
-	c.f.activate()
+	err := c.f.activate()
+	if err != nil {
+		c.state = StateStopped
+		return nil, err
+	}
 
 	// Call all the delayed funcs that waited patiently for the interface to be created.
 	if c.sshStart != nil {
@@ -71,16 +112,40 @@ func (c *Control) Start() {
 		c.lighthouseStart()
 	}
 
+	c.f.triggerShutdown = c.Stop
+
 	// Start reading packets.
-	c.f.run()
+	out, err := c.f.run()
+	if err != nil {
+		c.state = StateStopped
+		return nil, err
+	}
+	c.state = StateStarted
+	return out, nil
+}
+
+func (c *Control) State() RunState {
+	c.stateLock.Lock()
+	defer c.stateLock.Unlock()
+	return c.state
 }
 
 func (c *Control) Context() context.Context {
 	return c.ctx
 }
 
-// Stop signals nebula to shutdown and close all tunnels, returns after the shutdown is complete
+// Stop is a non-blocking call that signals nebula to close all tunnels and shut down
 func (c *Control) Stop() {
+	c.stateLock.Lock()
+	if c.state != StateStarted {
+		c.stateLock.Unlock()
+		// We are stopping or stopped already
+		return
+	}
+
+	c.state = StateStopping
+	c.stateLock.Unlock()
+
 	// Stop the handshakeManager (and other services), to prevent new tunnels from
 	// being created while we're shutting them all down.
 	c.cancel()
@@ -89,7 +154,9 @@ func (c *Control) Stop() {
 	if err := c.f.Close(); err != nil {
 		c.l.WithError(err).Error("Close interface failed")
 	}
-	c.l.Info("Goodbye")
+	c.stateLock.Lock()
+	c.state = StateStopped
+	c.stateLock.Unlock()
 }
 
 // ShutdownBlock will listen for and block on term and interrupt signals, calling Control.Stop() once signalled
@@ -274,6 +341,36 @@ func (c *Control) Device() overlay.Device {
 	return c.f.inside
 }
 
+// SetFirewallEventReporter installs an event reporter on the current firewall.
+// Passing nil clears any installed reporter. The reporter is carried across
+// firewall rule reloads. Report* methods are invoked while nebula holds
+// internal locks and must be non-blocking; in particular they must not call
+// back into *Control methods that touch the firewall, or deadlock will
+// result.
+//
+// Installation is performed by shallow-copying the current *Firewall,
+// setting the reporter field on the copy, and swapping the pointer under
+// the conntrack lock. Every Firewall the data path sees therefore has an
+// immutable reporter slot, and emit sites can read it without any
+// synchronization of their own.
+func (c *Control) SetFirewallEventReporter(r events.Reporter) {
+	old := c.f.firewall
+	if old == nil {
+		return
+	}
+	old.Conntrack.Lock()
+	defer old.Conntrack.Unlock()
+
+	// Re-read under the lock in case a concurrent reload swapped in a new
+	// Firewall between the unlocked load above and here. Both Firewalls share
+	// the same Conntrack pointer in the normal (non-overflow) reload path,
+	// so the lock we hold is the right one for whichever we see now.
+	current := c.f.firewall
+	fw := *current
+	fw.reporter = r
+	c.f.firewall = &fw
+}
+
 func copyHostInfo(h *HostInfo, preferredRanges []netip.Prefix) ControlHostInfo {
 	chi := ControlHostInfo{
 		VpnAddrs:               make([]netip.Addr, len(h.vpnAddrs)),

control_test.go

@@ -79,6 +79,7 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) {
 	}, &Interface{})
 
 	c := Control{
+		state: StateReady,
 		f: &Interface{
 			hostMap: hm,
 		},

dns_server.go

@@ -1,12 +1,14 @@
 package nebula
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/netip"
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 
 	"github.com/gaissmai/bart"
 	"github.com/miekg/dns"
@@ -14,32 +16,207 @@ import (
 	"github.com/slackhq/nebula/config"
 )
 
-// This whole thing should be rewritten to use context
+type dnsServer struct {
-
-var dnsR *dnsRecords
-var dnsServer *dns.Server
-var dnsAddr string
-
-type dnsRecords struct {
 	sync.RWMutex
 	l               *logrus.Logger
+	ctx             context.Context
 	dnsMap4         map[string]netip.Addr
 	dnsMap6         map[string]netip.Addr
 	hostMap         *HostMap
 	myVpnAddrsTable *bart.Lite
+
+	mux *dns.ServeMux
+
+	// enabled mirrors `lighthouse.serve_dns && lighthouse.am_lighthouse`.
+	// Start, Add, and reload consult it so callers don't need to know the
+	// gating rules. When it toggles off via reload, accumulated records are
+	// cleared so a later re-enable starts with a fresh map populated from
+	// new handshakes.
+	enabled atomic.Bool
+
+	serverMu sync.Mutex
+	server   *dns.Server
+	// started is closed once `server` has finished binding (or after
+	// ListenAndServe returns on a bind failure). Stop waits on it before
+	// calling Shutdown to avoid the miekg/dns "server not started" race
+	// where a Shutdown that arrives before bind completes is silently
+	// ignored, leaving the listener running forever.
+	started chan struct{}
+	addr    string
 }
 
-func newDnsRecords(l *logrus.Logger, cs *CertState, hostMap *HostMap) *dnsRecords {
+// newDnsServerFromConfig builds a dnsServer, applies the initial config, and
-	return &dnsRecords{
+// registers a reload callback. The reload callback is registered before the
+// initial config is applied, so a SIGHUP can later enable, fix, or disable
+// DNS even if the initial application failed.
+//
+// The dnsServer internally gates on `lighthouse.serve_dns &&
+// lighthouse.am_lighthouse`. Start and Add are safe to call unconditionally,
+// they no-op when DNS isn't enabled. Each Start invocation owns a ctx-cancel
+// watcher that tears the listener down on nebula shutdown. The returned
+// pointer is always non-nil, even on error.
+func newDnsServerFromConfig(ctx context.Context, l *logrus.Logger, cs *CertState, hostMap *HostMap, c *config.C) (*dnsServer, error) {
+	ds := &dnsServer{
 		l:               l,
+		ctx:             ctx,
 		dnsMap4:         make(map[string]netip.Addr),
 		dnsMap6:         make(map[string]netip.Addr),
 		hostMap:         hostMap,
 		myVpnAddrsTable: cs.myVpnAddrsTable,
 	}
+	ds.mux = dns.NewServeMux()
+	ds.mux.HandleFunc(".", ds.handleDnsRequest)
+
+	c.RegisterReloadCallback(func(c *config.C) {
+		if err := ds.reload(c, false); err != nil {
+			l.WithError(err).Error("Failed to reload DNS responder from config")
+		}
+	})
+
+	if err := ds.reload(c, true); err != nil {
+		return ds, err
+	}
+	return ds, nil
 }
 
-func (d *dnsRecords) Query(q uint16, data string) netip.Addr {
+// reload applies the latest config and reconciles the running state with it:
+//   - enabled toggled on  -> spawn a runner
+//   - enabled toggled off -> stop the runner
+//   - listen address changed (while running) -> restart on the new address
+//   - everything else     -> no-op
+//
+// On the initial call it only records configuration; Control.Start is what
+// launches the first runner via dnsStart.
+func (d *dnsServer) reload(c *config.C, initial bool) error {
+	wantsDns := c.GetBool("lighthouse.serve_dns", false)
+	amLighthouse := c.GetBool("lighthouse.am_lighthouse", false)
+	enabled := wantsDns && amLighthouse
+	newAddr := getDnsServerAddr(c)
+
+	d.serverMu.Lock()
+	running := d.server
+	runningStarted := d.started
+	sameAddr := d.addr == newAddr
+	d.addr = newAddr
+	d.enabled.Store(enabled)
+	d.serverMu.Unlock()
+
+	if initial {
+		if wantsDns && !amLighthouse {
+			d.l.Warn("DNS server refusing to run because this host is not a lighthouse.")
+		}
+		return nil
+	}
+
+	if !enabled {
+		if running != nil {
+			d.Stop()
+		}
+		// Drop any records that accumulated while enabled; a later re-enable
+		// will repopulate from fresh handshakes.
+		d.clearRecords()
+		return nil
+	}
+
+	if running == nil {
+		// Was disabled (or never started); bring it up now.
+		go d.Start()
+		return nil
+	}
+
+	if sameAddr {
+		return nil
+	}
+
+	d.shutdownServer(running, runningStarted, "reload")
+	// Old Start goroutine has now exited; bring up a fresh listener on the
+	// new address.
+	go d.Start()
+	return nil
+}
+
+// shutdownServer waits for the server to finish binding (so Shutdown actually
+// stops it rather than no-oping) and then shuts it down.
+func (d *dnsServer) shutdownServer(srv *dns.Server, started chan struct{}, reason string) {
+	if srv == nil {
+		return
+	}
+	if started != nil {
+		<-started
+	}
+	if err := srv.Shutdown(); err != nil {
+		d.l.WithError(err).WithField("reason", reason).Warn("Failed to shut down the DNS responder")
+	}
+}
+
+// Start binds and serves the DNS responder. Blocks until Stop is called or
+// the listener errors. Safe to call when DNS is disabled (returns
+// immediately). This is what Control.dnsStart points at.
+//
+// Must be invoked after the tun device is active so that lighthouse.dns.host
+// may bind to a nebula IP.
+func (d *dnsServer) Start() {
+	if !d.enabled.Load() {
+		return
+	}
+
+	started := make(chan struct{})
+	d.serverMu.Lock()
+	if d.ctx.Err() != nil {
+		d.serverMu.Unlock()
+		return
+	}
+	addr := d.addr
+	server := &dns.Server{
+		Addr:              addr,
+		Net:               "udp",
+		Handler:           d.mux,
+		NotifyStartedFunc: func() { close(started) },
+	}
+	d.server = server
+	d.started = started
+	d.serverMu.Unlock()
+
+	// Per-invocation ctx watcher. Exits when Start does, so we don't leak a
+	// watcher per reload-driven restart.
+	done := make(chan struct{})
+	go func() {
+		select {
+		case <-d.ctx.Done():
+			d.shutdownServer(server, started, "shutdown")
+		case <-done:
+		}
+	}()
+
+	d.l.WithField("dnsListener", addr).Info("Starting DNS responder")
+	err := server.ListenAndServe()
+	close(done)
+
+	// If the listener never bound (bind error) NotifyStartedFunc never fires,
+	// so close started here to release any Stop caller waiting on it.
+	select {
+	case <-started:
+	default:
+		close(started)
+	}
+
+	if err != nil {
+		d.l.WithError(err).Warn("Failed to run the DNS responder")
+	}
+}
+
+// Stop shuts down the active server, if any. Idempotent.
+func (d *dnsServer) Stop() {
+	d.serverMu.Lock()
+	srv := d.server
+	started := d.started
+	d.server = nil
+	d.started = nil
+	d.serverMu.Unlock()
+	d.shutdownServer(srv, started, "stop")
+}
+
+func (d *dnsServer) Query(q uint16, data string) netip.Addr {
 	data = strings.ToLower(data)
 	d.RLock()
 	defer d.RUnlock()
@@ -57,7 +234,7 @@ func (d *dnsRecords) Query(q uint16, data string) netip.Addr {
 	return netip.Addr{}
 }
 
-func (d *dnsRecords) QueryCert(data string) string {
+func (d *dnsServer) QueryCert(data string) string {
 	ip, err := netip.ParseAddr(data[:len(data)-1])
 	if err != nil {
 		return ""
@@ -80,8 +257,19 @@ func (d *dnsRecords) QueryCert(data string) string {
 	return string(b)
 }
 
+// clearRecords drops all DNS records.
+func (d *dnsServer) clearRecords() {
+	d.Lock()
+	defer d.Unlock()
+	clear(d.dnsMap4)
+	clear(d.dnsMap6)
+}
+
 // Add adds the first IPv4 and IPv6 address that appears in `addresses` as the record for `host`
-func (d *dnsRecords) Add(host string, addresses []netip.Addr) {
+func (d *dnsServer) Add(host string, addresses []netip.Addr) {
+	if !d.enabled.Load() {
+		return
+	}
 	host = strings.ToLower(host)
 	d.Lock()
 	defer d.Unlock()
@@ -101,7 +289,7 @@ func (d *dnsRecords) Add(host string, addresses []netip.Addr) {
 	}
 }
 
-func (d *dnsRecords) isSelfNebulaOrLocalhost(addr string) bool {
+func (d *dnsServer) isSelfNebulaOrLocalhost(addr string) bool {
 	a, _, _ := net.SplitHostPort(addr)
 	b, err := netip.ParseAddr(a)
 	if err != nil {
@@ -116,7 +304,7 @@ func (d *dnsRecords) isSelfNebulaOrLocalhost(addr string) bool {
 	return d.myVpnAddrsTable.Contains(b)
 }
 
-func (d *dnsRecords) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
+func (d *dnsServer) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
 	for _, q := range m.Question {
 		switch q.Qtype {
 		case dns.TypeA, dns.TypeAAAA:
@@ -150,7 +338,7 @@ func (d *dnsRecords) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
 	}
 }
 
-func (d *dnsRecords) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
+func (d *dnsServer) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
 	m := new(dns.Msg)
 	m.SetReply(r)
 	m.Compress = false
@@ -163,21 +351,6 @@ func (d *dnsRecords) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
 	w.WriteMsg(m)
 }
 
-func dnsMain(l *logrus.Logger, cs *CertState, hostMap *HostMap, c *config.C) func() {
-	dnsR = newDnsRecords(l, cs, hostMap)
-
-	// attach request handler func
-	dns.HandleFunc(".", dnsR.handleDnsRequest)
-
-	c.RegisterReloadCallback(func(c *config.C) {
-		reloadDns(l, c)
-	})
-
-	return func() {
-		startDns(l, c)
-	}
-}
-
 func getDnsServerAddr(c *config.C) string {
 	dnsHost := strings.TrimSpace(c.GetString("lighthouse.dns.host", ""))
 	// Old guidance was to provide the literal `[::]` in `lighthouse.dns.host` but that won't resolve.
@@ -186,25 +359,3 @@ func getDnsServerAddr(c *config.C) string {
 	}
 	return net.JoinHostPort(dnsHost, strconv.Itoa(c.GetInt("lighthouse.dns.port", 53)))
 }
-
-func startDns(l *logrus.Logger, c *config.C) {
-	dnsAddr = getDnsServerAddr(c)
-	dnsServer = &dns.Server{Addr: dnsAddr, Net: "udp"}
-	l.WithField("dnsListener", dnsAddr).Info("Starting DNS responder")
-	err := dnsServer.ListenAndServe()
-	defer dnsServer.Shutdown()
-	if err != nil {
-		l.Errorf("Failed to start server: %s\n ", err.Error())
-	}
-}
-
-func reloadDns(l *logrus.Logger, c *config.C) {
-	if dnsAddr == getDnsServerAddr(c) {
-		l.Debug("No DNS server config change detected")
-		return
-	}
-
-	l.Debug("Restarting DNS server")
-	dnsServer.Shutdown()
-	go startDns(l, c)
-}

dns_server_test.go

@@ -1,19 +1,31 @@
 package nebula
 
 import (
+	"context"
+	"io"
+	"net"
 	"net/netip"
+	"strconv"
 	"testing"
+	"time"
 
 	"github.com/miekg/dns"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestParsequery(t *testing.T) {
 	l := logrus.New()
 	hostMap := &HostMap{}
-	ds := newDnsRecords(l, &CertState{}, hostMap)
+	ds := &dnsServer{
+		l:       l,
+		dnsMap4: make(map[string]netip.Addr),
+		dnsMap6: make(map[string]netip.Addr),
+		hostMap: hostMap,
+	}
+	ds.enabled.Store(true)
 	addrs := []netip.Addr{
 		netip.MustParseAddr("1.2.3.4"),
 		netip.MustParseAddr("1.2.3.5"),
@@ -71,3 +83,209 @@ func Test_getDnsServerAddr(t *testing.T) {
 	}
 	assert.Equal(t, "[::]:1", getDnsServerAddr(c))
 }
+
+func newTestDnsServer(t *testing.T) (*dnsServer, *config.C) {
+	t.Helper()
+	l := logrus.New()
+	l.Out = io.Discard
+	ds := &dnsServer{
+		l:       l,
+		ctx:     context.Background(),
+		dnsMap4: make(map[string]netip.Addr),
+		dnsMap6: make(map[string]netip.Addr),
+		hostMap: &HostMap{},
+	}
+	ds.mux = dns.NewServeMux()
+	ds.mux.HandleFunc(".", ds.handleDnsRequest)
+	return ds, config.NewC(l)
+}
+
+func setDnsConfig(c *config.C, host string, port string, amLighthouse, serveDns bool) {
+	c.Settings["lighthouse"] = map[string]any{
+		"am_lighthouse": amLighthouse,
+		"serve_dns":     serveDns,
+		"dns": map[string]any{
+			"host": host,
+			"port": port,
+		},
+	}
+}
+
+func TestDnsServer_reload_initial_disabled(t *testing.T) {
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", "0", true, false)
+
+	require.NoError(t, ds.reload(c, true))
+	assert.False(t, ds.enabled.Load())
+	assert.Equal(t, "127.0.0.1:0", ds.addr)
+	assert.Nil(t, ds.server)
+}
+
+func TestDnsServer_reload_initial_enabled(t *testing.T) {
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", "0", true, true)
+
+	require.NoError(t, ds.reload(c, true))
+	assert.True(t, ds.enabled.Load())
+	assert.Equal(t, "127.0.0.1:0", ds.addr)
+	// initial never starts a runner; that's Control.Start's job
+	assert.Nil(t, ds.server)
+}
+
+func TestDnsServer_reload_initial_serveDnsWithoutLighthouse(t *testing.T) {
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", "0", false, true)
+
+	require.NoError(t, ds.reload(c, true))
+	// Wants DNS but isn't a lighthouse: gated off, no runner.
+	assert.False(t, ds.enabled.Load())
+}
+
+func TestDnsServer_reload_sameAddr_noOp(t *testing.T) {
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", "0", true, true)
+
+	require.NoError(t, ds.reload(c, true))
+	// No server running yet, no addr change. Reload should not spawn anything.
+	require.NoError(t, ds.reload(c, false))
+	assert.True(t, ds.enabled.Load())
+	assert.Nil(t, ds.server)
+}
+
+func TestDnsServer_StartStop_lifecycle(t *testing.T) {
+	// Bind to a real (random) UDP port so we exercise the actual
+	// ListenAndServe + Shutdown plumbing including the started-chan race fix.
+	port := freeUDPPort(t)
+
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", port, true, true)
+	require.NoError(t, ds.reload(c, true))
+
+	done := make(chan struct{})
+	go func() {
+		ds.Start()
+		close(done)
+	}()
+
+	waitFor(t, func() bool {
+		ds.serverMu.Lock()
+		started := ds.started
+		ds.serverMu.Unlock()
+		if started == nil {
+			return false
+		}
+		select {
+		case <-started:
+			return true
+		default:
+			return false
+		}
+	})
+
+	ds.Stop()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not return after Stop")
+	}
+}
+
+func TestDnsServer_Stop_beforeBind_doesNotHang(t *testing.T) {
+	// Stop called immediately after Start should not deadlock even if bind
+	// hasn't completed yet. This exercises the started-chan close-on-bind-fail
+	// path: by binding to an obviously bad port (privileged) we get a fast
+	// bind error before NotifyStartedFunc fires.
+	ds, c := newTestDnsServer(t)
+	// Use a port that should fail to bind (negative would be invalid, use a
+	// host that won't resolve to ensure listenUDP fails quickly).
+	setDnsConfig(c, "256.256.256.256", "53", true, true)
+	require.NoError(t, ds.reload(c, true))
+
+	done := make(chan struct{})
+	go func() {
+		ds.Start()
+		close(done)
+	}()
+
+	// Give Start a moment to attempt the bind and fail.
+	select {
+	case <-done:
+		// Bind failed and Start returned; Stop should be a no-op.
+	case <-time.After(time.Second):
+		t.Fatal("Start did not return after a bad bind")
+	}
+
+	stopped := make(chan struct{})
+	go func() {
+		ds.Stop()
+		close(stopped)
+	}()
+	select {
+	case <-stopped:
+	case <-time.After(time.Second):
+		t.Fatal("Stop hung after a failed bind")
+	}
+}
+
+func TestDnsServer_reload_disable_stopsRunningServer(t *testing.T) {
+	port := freeUDPPort(t)
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", port, true, true)
+	require.NoError(t, ds.reload(c, true))
+
+	startReturned := make(chan struct{})
+	go func() {
+		ds.Start()
+		close(startReturned)
+	}()
+	waitForBind(t, ds)
+
+	// Toggle serve_dns off; reload should shut the running server down.
+	setDnsConfig(c, "127.0.0.1", port, true, false)
+	require.NoError(t, ds.reload(c, false))
+	select {
+	case <-startReturned:
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not return after reload disabled DNS")
+	}
+	assert.False(t, ds.enabled.Load())
+}
+
+func freeUDPPort(t *testing.T) string {
+	t.Helper()
+	conn, err := net.ListenPacket("udp", "127.0.0.1:0")
+	require.NoError(t, err)
+	port := conn.LocalAddr().(*net.UDPAddr).Port
+	require.NoError(t, conn.Close())
+	return strconv.Itoa(port)
+}
+
+func waitForBind(t *testing.T, ds *dnsServer) {
+	t.Helper()
+	waitFor(t, func() bool {
+		ds.serverMu.Lock()
+		started := ds.started
+		ds.serverMu.Unlock()
+		if started == nil {
+			return false
+		}
+		select {
+		case <-started:
+			return true
+		default:
+			return false
+		}
+	})
+}
+
+func waitFor(t *testing.T, cond func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		if cond() {
+			return
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	t.Fatal("timed out waiting for condition")
+}

firewall.go

@@ -20,6 +20,7 @@ import (
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/firewall"
+	"github.com/slackhq/nebula/firewall/events"
 )
 
 type FirewallInterface interface {
@@ -67,6 +68,14 @@ type Firewall struct {
 	incomingMetrics     firewallMetrics
 	outgoingMetrics     firewallMetrics
 
+	// reporter is the optional embedder-supplied event sink. Immutable for
+	// the lifetime of this Firewall; Control.SetFirewallEventReporter
+	// installs it by shallow-copying the Firewall under the conntrack lock
+	// and swapping the pointer, and reloadFirewall carries it forward.
+	// Read unsynchronized on the data path: the preceding Firewall-pointer
+	// read pins the field's value for the duration of that call.
+	reporter events.Reporter
+
 	l *logrus.Logger
 }
 
@@ -416,23 +425,27 @@ var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
 
 // Drop returns an error if the packet should be dropped, explaining why. It
 // returns nil if the packet should not be dropped.
-func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error {
+func (f *Firewall) Drop(fp firewall.Packet, ctx firewall.PacketContext, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error {
 	// Check if we spoke to this tuple, if we did then allow this packet
 	if f.inConns(fp, h, caPool, localCache) {
 		return nil
 	}
 
+	peerCert := h.ConnectionState.peerCert
+
 	// Make sure remote address matches nebula certificate, and determine how to treat it
 	if h.networks == nil {
 		// Simple case: Certificate has one address and no unsafe networks
 		if h.vpnAddrs[0] != fp.RemoteAddr {
 			f.metrics(incoming).droppedRemoteAddr.Inc(1)
+			f.reportDrop(incoming, events.DropInvalidRemoteIP, fp, ctx, peerCert)
 			return ErrInvalidRemoteIP
 		}
 	} else {
 		nwType, ok := h.networks.Lookup(fp.RemoteAddr)
 		if !ok {
 			f.metrics(incoming).droppedRemoteAddr.Inc(1)
+			f.reportDrop(incoming, events.DropInvalidRemoteIP, fp, ctx, peerCert)
 			return ErrInvalidRemoteIP
 		}
 		switch nwType {
@@ -440,11 +453,13 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
 			break // nothing special
 		case NetworkTypeVPNPeer:
 			f.metrics(incoming).droppedRemoteAddr.Inc(1)
+			f.reportDrop(incoming, events.DropPeerRejected, fp, ctx, peerCert)
 			return ErrPeerRejected // reject for now, one day this may have different FW rules
 		case NetworkTypeUnsafe:
 			break // nothing special, one day this may have different FW rules
 		default:
 			f.metrics(incoming).droppedRemoteAddr.Inc(1)
+			f.reportDrop(incoming, events.DropUnknownNetwork, fp, ctx, peerCert)
 			return ErrUnknownNetworkType //should never happen
 		}
 	}
@@ -452,6 +467,7 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
 	// Make sure we are supposed to be handling this local ip address
 	if !f.routableNetworks.Contains(fp.LocalAddr) {
 		f.metrics(incoming).droppedLocalAddr.Inc(1)
+		f.reportDrop(incoming, events.DropInvalidLocalIP, fp, ctx, peerCert)
 		return ErrInvalidLocalIP
 	}
 
@@ -461,13 +477,14 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
 	}
 
 	// We now know which firewall table to check against
-	if !table.match(fp, incoming, h.ConnectionState.peerCert, caPool) {
+	if !table.match(fp, incoming, peerCert, caPool) {
 		f.metrics(incoming).droppedNoRule.Inc(1)
+		f.reportDrop(incoming, events.DropNoMatchingRule, fp, ctx, peerCert)
 		return ErrNoMatchingRule
 	}
 
 	// We always want to conntrack since it is a faster operation
-	f.addConn(fp, incoming)
+	f.addConn(fp, ctx, incoming, peerCert)
 
 	return nil
 }
@@ -486,6 +503,59 @@ func (f *Firewall) Destroy() {
 	//TODO: clean references if/when needed
 }
 
+func (f *Firewall) reportDrop(incoming bool, reason events.DropReason, fp firewall.Packet, ctx firewall.PacketContext, peerCert *cert.CachedCertificate) {
+	r := f.reporter
+	if r == nil {
+		return
+	}
+	r.ReportDrop(events.DropEvent{
+		Incoming:     incoming,
+		Reason:       reason,
+		Packet:       fp,
+		Context:      ctx,
+		PeerCert:     peerCert,
+		RulesVersion: f.rulesVersion,
+	})
+}
+
+func (f *Firewall) reportFlowCreate(incoming bool, fp firewall.Packet, ctx firewall.PacketContext, peerCert *cert.CachedCertificate) {
+	r := f.reporter
+	if r == nil {
+		return
+	}
+	r.ReportFlowCreate(events.FlowCreateEvent{
+		Incoming:     incoming,
+		Packet:       fp,
+		Context:      ctx,
+		PeerCert:     peerCert,
+		RulesVersion: f.rulesVersion,
+	})
+}
+
+func (f *Firewall) reportFlowEvict(incoming bool, fp firewall.Packet, rulesVersion uint16, expired bool) {
+	r := f.reporter
+	if r == nil {
+		return
+	}
+	r.ReportFlowEvict(events.FlowEvictEvent{
+		Incoming:     incoming,
+		Packet:       fp,
+		RulesVersion: rulesVersion,
+		Expired:      expired,
+	})
+}
+
+func (f *Firewall) reportRulesReload(oldVersion, newVersion uint16) {
+	r := f.reporter
+	if r == nil {
+		return
+	}
+	r.ReportRulesReload(events.RulesReloadEvent{
+		OldVersion: oldVersion,
+		NewVersion: newVersion,
+	})
+}
+
 func (f *Firewall) EmitStats() {
 	conntrack := f.Conntrack
 	conntrack.Lock()
@@ -536,7 +606,9 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 					WithField("oldRulesVersion", c.rulesVersion).
 					Debugln("dropping old conntrack entry, does not match new ruleset")
 			}
+			oldRulesVersion := c.rulesVersion
 			delete(conntrack.Conns, fp)
+			f.reportFlowEvict(c.incoming, fp, oldRulesVersion, false)
 			conntrack.Unlock()
 			return false
 		}
@@ -571,7 +643,7 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 	return true
 }
 
-func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
+func (f *Firewall) addConn(fp firewall.Packet, ctx firewall.PacketContext, incoming bool, peerCert *cert.CachedCertificate) {
 	var timeout time.Duration
 	c := &conn{}
 
@@ -586,7 +658,8 @@ func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
 
 	conntrack := f.Conntrack
 	conntrack.Lock()
-	if _, ok := conntrack.Conns[fp]; !ok {
+	_, existing := conntrack.Conns[fp]
+	if !existing {
 		conntrack.TimerWheel.Advance(time.Now())
 		conntrack.TimerWheel.Add(fp, timeout)
 	}
@@ -597,6 +670,13 @@ func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
 	c.rulesVersion = f.rulesVersion
 	c.Expires = time.Now().Add(timeout)
 	conntrack.Conns[fp] = c
+
+	// Report only when this represents a genuinely new flow. Fires under the
+	// conntrack lock so FlowCreate/FlowEvict events stay ordered relative to
+	// RulesReloadEvent, which also fires under this lock.
+	if !existing {
+		f.reportFlowCreate(incoming, fp, ctx, peerCert)
+	}
 	conntrack.Unlock()
 }
 
@@ -620,7 +700,10 @@ func (f *Firewall) evict(p firewall.Packet) {
 	}
 
 	// This conn is done
+	rulesVersion := t.rulesVersion
+	incoming := t.incoming
 	delete(conntrack.Conns, p)
+	f.reportFlowEvict(incoming, p, rulesVersion, true)
 }
 
 func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {

firewall/cache.go

@@ -1,6 +1,7 @@
 package firewall
 
 import (
+	"context"
 	"sync/atomic"
 	"time"
 
@@ -18,7 +19,7 @@ type ConntrackCacheTicker struct {
 	cache ConntrackCache
 }
 
-func NewConntrackCacheTicker(d time.Duration) *ConntrackCacheTicker {
+func NewConntrackCacheTicker(ctx context.Context, d time.Duration) *ConntrackCacheTicker {
 	if d == 0 {
 		return nil
 	}
@@ -27,15 +28,21 @@ func NewConntrackCacheTicker(d time.Duration) *ConntrackCacheTicker {
 		cache: ConntrackCache{},
 	}
 
-	go c.tick(d)
+	go c.tick(ctx, d)
 
 	return c
 }
 
-func (c *ConntrackCacheTicker) tick(d time.Duration) {
+func (c *ConntrackCacheTicker) tick(ctx context.Context, d time.Duration) {
+	t := time.NewTicker(d)
+	defer t.Stop()
 	for {
-		time.Sleep(d)
+		select {
-		c.cacheTick.Add(1)
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			c.cacheTick.Add(1)
+		}
 	}
 }
 

firewall/events/events.go

@@ -0,0 +1,102 @@
+// Package events defines the opt-in firewall event reporting interface.
+//
+// Nebula emits raw packet-level events (drops, flow creations, flow evictions,
+// rule reloads) and does no aggregation, counting, batching, rule-description,
+// transport, or timestamping. Embedders correlate events back to yaml rules
+// out of band and capture whatever clock they need themselves. All Report*
+// methods are invoked while nebula holds internal locks and must be
+// non-blocking.
+//
+// Events are passed to Report* methods by value. Implementations must not
+// take the address of a received event: doing so forces Go's escape
+// analysis to move the event to the heap and costs one allocation per call.
+// To forward an event, either copy its fields into the reporter's own
+// pooled record or send it through a value-typed channel (chan DropEvent,
+// not chan *DropEvent).
+package events
+
+import (
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/firewall"
+)
+
+type DropReason uint8
+
+const (
+	DropInvalidLocalIP DropReason = iota
+	DropInvalidRemoteIP
+	DropPeerRejected
+	DropUnknownNetwork
+	DropNoMatchingRule
+)
+
+func (r DropReason) String() string {
+	switch r {
+	case DropInvalidLocalIP:
+		return "invalid_local_ip"
+	case DropInvalidRemoteIP:
+		return "invalid_remote_ip"
+	case DropPeerRejected:
+		return "peer_rejected"
+	case DropUnknownNetwork:
+		return "unknown_network"
+	case DropNoMatchingRule:
+		return "no_matching_rule"
+	default:
+		return "unknown"
+	}
+}
+
+// DropEvent is emitted for every packet that fails the firewall check. Drops
+// are not aggregated; every drop produces one event.
+type DropEvent struct {
+	Incoming     bool
+	Reason       DropReason
+	Packet       firewall.Packet
+	Context      firewall.PacketContext
+	PeerCert     *cert.CachedCertificate
+	RulesVersion uint16
+}
+
+// FlowCreateEvent is emitted when a packet is allowed and a new conntrack
+// entry is created. Subsequent packets in the same flow do not re-emit.
+type FlowCreateEvent struct {
+	Incoming     bool
+	Packet       firewall.Packet
+	Context      firewall.PacketContext
+	PeerCert     *cert.CachedCertificate
+	RulesVersion uint16
+}
+
+// FlowEvictEvent is emitted when a conntrack entry is removed. Context is
+// not carried: timer-wheel eviction has no packet in hand, and reload
+// revalidation evicts the OLD flow rather than the triggering packet.
+// RulesVersion is the version under which the flow was originally allowed,
+// which may differ from the current firewall version.
+type FlowEvictEvent struct {
+	Incoming     bool
+	Packet       firewall.Packet
+	RulesVersion uint16
+	// Expired is true when eviction was due to conntrack timeout; false when
+	// the entry was removed because it failed re-validation after a reload.
+	Expired bool
+}
+
+// RulesReloadEvent is emitted once after each successful firewall reload.
+// Reporters that bucket state by RulesVersion should close the old bucket
+// and open a new one on receipt.
+type RulesReloadEvent struct {
+	OldVersion uint16
+	NewVersion uint16
+}
+
+// Reporter is the embedder-supplied sink for firewall events. Implementations
+// that want a timestamp should call time.Now() themselves at the top of the
+// method; nebula does not provide one. See the package doc for the
+// do-not-take-address rule.
+type Reporter interface {
+	ReportDrop(DropEvent)
+	ReportFlowCreate(FlowCreateEvent)
+	ReportFlowEvict(FlowEvictEvent)
+	ReportRulesReload(RulesReloadEvent)
+}

firewall/packet.go

@@ -31,6 +31,27 @@ type Packet struct {
 	Fragment   bool
 }
 
+// PacketContext carries additional parsed details about a packet that are
+// useful for event reporting but deliberately kept out of Packet so Packet
+// can keep being used as a conntrack map key. Populated alongside Packet by
+// newPacket.
+//
+// Fields are interpreted based on Packet.Protocol:
+//   - ProtoTCP: TCPFlags is meaningful; ICMPType / ICMPCode are zero
+//   - ProtoICMP, ProtoICMPv6: ICMPType / ICMPCode are meaningful; TCPFlags is zero
+//   - ProtoUDP and others: only Length is meaningful
+type PacketContext struct {
+	// Length is the total IP packet length in bytes, including headers.
+	Length uint16
+	// TCPFlags is the flag byte from the TCP header (bits for FIN, SYN, RST,
+	// PSH, ACK, URG, ECE, CWR).
+	TCPFlags uint8
+	// ICMPType is the type field of the ICMP / ICMPv6 header.
+	ICMPType uint8
+	// ICMPCode is the code field of the ICMP / ICMPv6 header.
+	ICMPCode uint8
+}
+
 func (fp *Packet) Copy() *Packet {
 	return &Packet{
 		LocalAddr:  fp.LocalAddr,

firewall_events_test.go

@@ -0,0 +1,731 @@
+package nebula
+
+import (
+	"net"
+	"net/netip"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/gaissmai/bart"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/firewall"
+	"github.com/slackhq/nebula/firewall/events"
+	"github.com/slackhq/nebula/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// recordingReporter captures every event fired against it. Its methods take
+// the conntrack lock implicitly (via the firewall code path that invokes
+// them), so we synchronize accumulator mutations with a small mutex to keep
+// the race detector happy across goroutines in case a test introduces any.
+type recordingReporter struct {
+	mu      sync.Mutex
+	drops   []recordedDrop
+	creates []recordedCreate
+	evicts  []recordedEvict
+	reloads []recordedReload
+}
+
+type recordedDrop struct {
+	incoming     bool
+	reason       events.DropReason
+	remote       netip.Addr
+	local        netip.Addr
+	peerName     string
+	rulesVersion uint16
+	ctx          firewall.PacketContext
+}
+
+type recordedCreate struct {
+	incoming     bool
+	remote       netip.Addr
+	local        netip.Addr
+	peerName     string
+	rulesVersion uint16
+	ctx          firewall.PacketContext
+}
+
+type recordedEvict struct {
+	incoming     bool
+	remote       netip.Addr
+	local        netip.Addr
+	rulesVersion uint16
+	expired      bool
+}
+
+type recordedReload struct {
+	oldVersion uint16
+	newVersion uint16
+}
+
+func (r *recordingReporter) ReportDrop(e events.DropEvent) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	name := ""
+	if e.PeerCert != nil && e.PeerCert.Certificate != nil {
+		name = e.PeerCert.Certificate.Name()
+	}
+	r.drops = append(r.drops, recordedDrop{
+		incoming:     e.Incoming,
+		reason:       e.Reason,
+		remote:       e.Packet.RemoteAddr,
+		local:        e.Packet.LocalAddr,
+		peerName:     name,
+		rulesVersion: e.RulesVersion,
+		ctx:          e.Context,
+	})
+}
+
+func (r *recordingReporter) ReportFlowCreate(e events.FlowCreateEvent) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	name := ""
+	if e.PeerCert != nil && e.PeerCert.Certificate != nil {
+		name = e.PeerCert.Certificate.Name()
+	}
+	r.creates = append(r.creates, recordedCreate{
+		incoming:     e.Incoming,
+		remote:       e.Packet.RemoteAddr,
+		local:        e.Packet.LocalAddr,
+		peerName:     name,
+		rulesVersion: e.RulesVersion,
+		ctx:          e.Context,
+	})
+}
+
+func (r *recordingReporter) ReportFlowEvict(e events.FlowEvictEvent) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.evicts = append(r.evicts, recordedEvict{
+		incoming:     e.Incoming,
+		remote:       e.Packet.RemoteAddr,
+		local:        e.Packet.LocalAddr,
+		rulesVersion: e.RulesVersion,
+		expired:      e.Expired,
+	})
+}
+
+func (r *recordingReporter) ReportRulesReload(e events.RulesReloadEvent) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.reloads = append(r.reloads, recordedReload{
+		oldVersion: e.OldVersion,
+		newVersion: e.NewVersion,
+	})
+}
+
+// eventFixture builds a Firewall wired to a Control plus a packet/hostinfo
+// pair that a test can reuse. By default the ruleset allows the packet;
+// callers mutate fw / p / h as needed before invoking Drop.
+type eventFixture struct {
+	ctl *Control
+	fw  *Firewall
+	p   firewall.Packet
+	h   *HostInfo
+	cp  *cert.CAPool
+}
+
+func newEventFixture(t *testing.T) *eventFixture {
+	t.Helper()
+	l := test.NewLogger()
+
+	// myVpnNetworksTable covers our single peer address so buildNetworks takes
+	// the "simple case" path (h.networks stays nil); tests that want a populated
+	// BART table overwrite h.networks directly.
+	vpnNetworks := new(bart.Lite)
+	vpnNetworks.Insert(netip.MustParsePrefix("1.2.3.0/24"))
+
+	// Use the same cert for "peer" and "local" endpoints, matching the
+	// TestFirewall_Drop fixture style: LocalAddr == RemoteAddr == peer vpn addr.
+	c := &dummyCert{
+		name:     "host1",
+		networks: []netip.Prefix{netip.MustParsePrefix("1.2.3.4/24")},
+		groups:   []string{"default-group"},
+		issuer:   "signer-shasum",
+	}
+	h := &HostInfo{
+		ConnectionState: &ConnectionState{
+			peerCert: &cert.CachedCertificate{
+				Certificate:    c,
+				InvertedGroups: map[string]struct{}{"default-group": {}},
+			},
+		},
+		vpnAddrs: []netip.Addr{netip.MustParseAddr("1.2.3.4")},
+	}
+	h.buildNetworks(vpnNetworks, c)
+
+	fw := NewFirewall(l, time.Minute, time.Minute, time.Minute, c)
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
+	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
+
+	ctl := &Control{
+		f: &Interface{firewall: fw},
+		l: l,
+	}
+
+	return &eventFixture{
+		ctl: ctl,
+		fw:  fw,
+		p: firewall.Packet{
+			LocalAddr:  netip.MustParseAddr("1.2.3.4"),
+			RemoteAddr: netip.MustParseAddr("1.2.3.4"),
+			LocalPort:  10,
+			RemotePort: 90,
+			Protocol:   firewall.ProtoUDP,
+		},
+		h:  h,
+		cp: cert.NewCAPool(),
+	}
+}
+
+// firewall() returns the currently-installed firewall. Needed because
+// SetFirewallEventReporter replaces it via shallow-copy swap.
+func (f *eventFixture) firewall() *Firewall {
+	return f.ctl.f.firewall
+}
+
+func TestEvents_ReportDrop_InvalidRemoteIP(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// Packet to an address not in the cert's networks.
+	f.p.RemoteAddr = netip.MustParseAddr("9.9.9.9")
+	assert.Equal(t, ErrInvalidRemoteIP, f.firewall().Drop(f.p, firewall.PacketContext{}, false, f.h, f.cp, nil))
+
+	require.Len(t, r.drops, 1)
+	assert.Equal(t, events.DropInvalidRemoteIP, r.drops[0].reason)
+	assert.False(t, r.drops[0].incoming)
+	assert.Equal(t, "host1", r.drops[0].peerName)
+	assert.Empty(t, r.creates)
+	assert.Empty(t, r.evicts)
+}
+
+func TestEvents_ReportDrop_InvalidLocalIP(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// LocalAddr outside our routable networks.
+	f.p.LocalAddr = netip.MustParseAddr("9.9.9.9")
+	assert.Equal(t, ErrInvalidLocalIP, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+
+	require.Len(t, r.drops, 1)
+	assert.Equal(t, events.DropInvalidLocalIP, r.drops[0].reason)
+	assert.True(t, r.drops[0].incoming)
+}
+
+func TestEvents_ReportDrop_NoMatchingRule(t *testing.T) {
+	f := newEventFixture(t)
+	// Reset to a firewall with no matching rule.
+	l := test.NewLogger()
+	fw := NewFirewall(l, time.Minute, time.Minute, time.Minute, f.h.ConnectionState.peerCert.Certificate)
+	// Rule that won't match (group not in peer's groups).
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", ""))
+	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", ""))
+	f.ctl.f.firewall = fw
+
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	assert.Equal(t, ErrNoMatchingRule, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.drops, 1)
+	assert.Equal(t, events.DropNoMatchingRule, r.drops[0].reason)
+}
+
+func TestEvents_ReportDrop_PeerRejected(t *testing.T) {
+	f := newEventFixture(t)
+	// Re-classify the remote as VPNPeer so it triggers DropPeerRejected.
+	f.h.networks = new(bart.Table[NetworkType])
+	f.h.networks.Insert(netip.MustParsePrefix("1.2.3.0/24"), NetworkTypeVPNPeer)
+
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	assert.Equal(t, ErrPeerRejected, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.drops, 1)
+	assert.Equal(t, events.DropPeerRejected, r.drops[0].reason)
+}
+
+func TestEvents_ReportDrop_UnknownNetwork(t *testing.T) {
+	f := newEventFixture(t)
+	// Insert an unrecognized NetworkType value to hit the default branch.
+	f.h.networks = new(bart.Table[NetworkType])
+	f.h.networks.Insert(netip.MustParsePrefix("1.2.3.0/24"), NetworkTypeUnknown)
+
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	assert.Equal(t, ErrUnknownNetworkType, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.drops, 1)
+	assert.Equal(t, events.DropUnknownNetwork, r.drops[0].reason)
+}
+
+func TestEvents_ReportFlowCreate_OnceOnly(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// First allowed packet creates the conntrack entry.
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	// Second matching packet on the same tuple is short-circuited by conntrack
+	// and must not fire another FlowCreate.
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+
+	require.Len(t, r.creates, 1)
+	assert.True(t, r.creates[0].incoming)
+	assert.Equal(t, f.p.RemoteAddr, r.creates[0].remote)
+	assert.Empty(t, r.drops)
+}
+
+func TestEvents_ReportFlowEvict_OnReloadPurge(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// Create a flow under the current rules.
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.creates, 1)
+
+	// Simulate a reload that produces rules the existing flow no longer
+	// matches. Bump rulesVersion and replace InRules with an empty table so
+	// revalidation fails.
+	fw := f.firewall()
+	fw.Conntrack.Lock()
+	fw.rulesVersion++
+	fw.InRules = newFirewallTable()
+	fw.Conntrack.Unlock()
+
+	// Next packet triggers re-validation, which fails and evicts the entry.
+	err := fw.Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
+	assert.Equal(t, ErrNoMatchingRule, err)
+
+	require.Len(t, r.evicts, 1)
+	assert.False(t, r.evicts[0].expired, "evict from reload purge is not expiration")
+	assert.True(t, r.evicts[0].incoming)
+}
+
+func TestEvents_ReportFlowEvict_OnTimeout(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.creates, 1)
+
+	// Force expiration by rewinding the entry's deadline.
+	fw := f.firewall()
+	fw.Conntrack.Lock()
+	c := fw.Conntrack.Conns[f.p]
+	require.NotNil(t, c)
+	c.Expires = time.Now().Add(-time.Hour)
+	fw.evict(f.p)
+	fw.Conntrack.Unlock()
+
+	require.Len(t, r.evicts, 1)
+	assert.True(t, r.evicts[0].expired)
+}
+
+func TestEvents_SetNil_Clears(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.creates, 1)
+
+	f.ctl.SetFirewallEventReporter(nil)
+	resetConntrack(f.firewall())
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	// No second create should be recorded.
+	assert.Len(t, r.creates, 1)
+}
+
+func TestEvents_ReporterSurvivesSwap(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// Simulate a reload by swapping in a fresh Firewall that carries the
+	// reporter forward. Mirrors what reloadFirewall does with the shared
+	// conntrack pointer.
+	l := test.NewLogger()
+	oldFw := f.firewall()
+	newFw := NewFirewall(l, time.Minute, time.Minute, time.Minute, f.h.ConnectionState.peerCert.Certificate)
+	require.NoError(t, newFw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
+	require.NoError(t, newFw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
+	newFw.Conntrack = oldFw.Conntrack
+	newFw.rulesVersion = oldFw.rulesVersion + 1
+	newFw.reporter = oldFw.reporter
+	f.ctl.f.firewall = newFw
+	newFw.reportRulesReload(oldFw.rulesVersion, newFw.rulesVersion)
+
+	require.Len(t, r.reloads, 1)
+	assert.Equal(t, oldFw.rulesVersion, r.reloads[0].oldVersion)
+	assert.Equal(t, newFw.rulesVersion, r.reloads[0].newVersion)
+
+	// Events on the new firewall should still reach the same reporter.
+	require.NoError(t, newFw.Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.creates, 1)
+	assert.Equal(t, newFw.rulesVersion, r.creates[0].rulesVersion)
+}
+
+func TestEvents_InstallDoesNotMutateOldFirewall(t *testing.T) {
+	f := newEventFixture(t)
+	before := f.firewall()
+
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	after := f.firewall()
+	assert.NotSame(t, before, after, "SetFirewallEventReporter must replace the Firewall pointer")
+	assert.Nil(t, before.reporter, "the pre-install Firewall must remain untouched")
+	assert.NotNil(t, after.reporter)
+}
+
+// --- PacketContext parse tests --------------------------------------------
+
+func mustSerialize(t *testing.T, lrs ...gopacket.SerializableLayer) []byte {
+	t.Helper()
+	buf := gopacket.NewSerializeBuffer()
+	opt := gopacket.SerializeOptions{ComputeChecksums: false, FixLengths: true}
+	require.NoError(t, gopacket.SerializeLayers(buf, opt, lrs...))
+	return buf.Bytes()
+}
+
+func TestPacketContext_IPv4_TCPFlags(t *testing.T) {
+	ip := &layers.IPv4{
+		Version: 4, TTL: 64, Protocol: layers.IPProtocolTCP,
+		SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
+	}
+	tcp := &layers.TCP{SrcPort: 1234, DstPort: 80, SYN: true, ACK: true}
+	require.NoError(t, tcp.SetNetworkLayerForChecksum(ip))
+	data := mustSerialize(t, ip, tcp, gopacket.Payload([]byte("hello")))
+
+	var fp firewall.Packet
+	var ctx firewall.PacketContext
+	require.NoError(t, newPacket(data, true, &fp, &ctx))
+
+	assert.Equal(t, uint8(firewall.ProtoTCP), fp.Protocol)
+	// SYN (0x02) + ACK (0x10) = 0x12
+	assert.Equal(t, uint8(0x12), ctx.TCPFlags)
+	assert.Equal(t, uint16(len(data)), ctx.Length)
+	assert.Equal(t, uint8(0), ctx.ICMPType)
+	assert.Equal(t, uint8(0), ctx.ICMPCode)
+}
+
+func TestPacketContext_IPv4_ICMPTypeCode(t *testing.T) {
+	ip := &layers.IPv4{
+		Version: 4, TTL: 64, Protocol: layers.IPProtocolICMPv4,
+		SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
+	}
+	// Destination Unreachable, code 3 (port unreachable)
+	icmp := &layers.ICMPv4{
+		TypeCode: layers.CreateICMPv4TypeCode(layers.ICMPv4TypeDestinationUnreachable, layers.ICMPv4CodePort),
+	}
+	data := mustSerialize(t, ip, icmp, gopacket.Payload([]byte{0, 0, 0, 0}))
+
+	var fp firewall.Packet
+	var ctx firewall.PacketContext
+	require.NoError(t, newPacket(data, true, &fp, &ctx))
+
+	assert.Equal(t, uint8(firewall.ProtoICMP), fp.Protocol)
+	assert.Equal(t, uint8(layers.ICMPv4TypeDestinationUnreachable), ctx.ICMPType)
+	assert.Equal(t, uint8(layers.ICMPv4CodePort), ctx.ICMPCode)
+	assert.Equal(t, uint16(len(data)), ctx.Length)
+	assert.Equal(t, uint8(0), ctx.TCPFlags)
+}
+
+func TestPacketContext_IPv4_UDPLengthOnly(t *testing.T) {
+	ip := &layers.IPv4{
+		Version: 4, TTL: 64, Protocol: layers.IPProtocolUDP,
+		SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
+	}
+	udp := &layers.UDP{SrcPort: 1234, DstPort: 53}
+	require.NoError(t, udp.SetNetworkLayerForChecksum(ip))
+	data := mustSerialize(t, ip, udp, gopacket.Payload([]byte("query")))
+
+	var fp firewall.Packet
+	var ctx firewall.PacketContext
+	require.NoError(t, newPacket(data, true, &fp, &ctx))
+
+	assert.Equal(t, uint8(firewall.ProtoUDP), fp.Protocol)
+	assert.Equal(t, uint16(len(data)), ctx.Length)
+	assert.Zero(t, ctx.TCPFlags)
+	assert.Zero(t, ctx.ICMPType)
+	assert.Zero(t, ctx.ICMPCode)
+}
+
+func TestPacketContext_IPv6_TCPFlags(t *testing.T) {
+	ip := &layers.IPv6{
+		Version: 6, HopLimit: 64, NextHeader: layers.IPProtocolTCP,
+		SrcIP: net.ParseIP("fd00::1"), DstIP: net.ParseIP("fd00::2"),
+	}
+	tcp := &layers.TCP{SrcPort: 1234, DstPort: 443, FIN: true, ACK: true}
+	require.NoError(t, tcp.SetNetworkLayerForChecksum(ip))
+	data := mustSerialize(t, ip, tcp, gopacket.Payload([]byte("bye")))
+
+	var fp firewall.Packet
+	var ctx firewall.PacketContext
+	require.NoError(t, newPacket(data, true, &fp, &ctx))
+
+	assert.Equal(t, uint8(firewall.ProtoTCP), fp.Protocol)
+	// FIN (0x01) + ACK (0x10) = 0x11
+	assert.Equal(t, uint8(0x11), ctx.TCPFlags)
+	assert.Equal(t, uint16(len(data)), ctx.Length)
+}
+
+func TestPacketContext_IPv6_ICMPv6TypeCode(t *testing.T) {
+	ip := &layers.IPv6{
+		Version: 6, HopLimit: 64, NextHeader: layers.IPProtocolICMPv6,
+		SrcIP: net.ParseIP("fd00::1"), DstIP: net.ParseIP("fd00::2"),
+	}
+	icmp := &layers.ICMPv6{
+		TypeCode: layers.CreateICMPv6TypeCode(layers.ICMPv6TypeDestinationUnreachable, layers.ICMPv6CodePortUnreachable),
+	}
+	require.NoError(t, icmp.SetNetworkLayerForChecksum(ip))
+	data := mustSerialize(t, ip, icmp, gopacket.Payload([]byte{0, 0, 0, 0, 0, 0, 0, 0}))
+
+	var fp firewall.Packet
+	var ctx firewall.PacketContext
+	require.NoError(t, newPacket(data, true, &fp, &ctx))
+
+	assert.Equal(t, uint8(firewall.ProtoICMPv6), fp.Protocol)
+	assert.Equal(t, uint8(layers.ICMPv6TypeDestinationUnreachable), ctx.ICMPType)
+	assert.Equal(t, uint8(layers.ICMPv6CodePortUnreachable), ctx.ICMPCode)
+	assert.Equal(t, uint16(len(data)), ctx.Length)
+}
+
+// TestPacketContext_NilOK confirms a nil context pointer is accepted by
+// newPacket (the hot path may elect not to pass one).
+func TestPacketContext_NilOK(t *testing.T) {
+	ip := &layers.IPv4{
+		Version: 4, TTL: 64, Protocol: layers.IPProtocolUDP,
+		SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
+	}
+	udp := &layers.UDP{SrcPort: 1, DstPort: 2}
+	require.NoError(t, udp.SetNetworkLayerForChecksum(ip))
+	data := mustSerialize(t, ip, udp)
+
+	var fp firewall.Packet
+	require.NoError(t, newPacket(data, true, &fp, nil))
+}
+
+// TestPacketContext_FlowCreateCarriesContext exercises the full Drop -> addConn
+// -> ReportFlowCreate path with a realistic TCP packet and confirms the
+// context makes it into the reporter.
+func TestPacketContext_FlowCreateCarriesContext(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// Hand-construct a matching TCP packet.
+	ctx := firewall.PacketContext{Length: 1500, TCPFlags: 0x12}
+	p := f.p
+	p.Protocol = firewall.ProtoTCP
+	require.NoError(t, f.firewall().Drop(p, ctx, true, f.h, f.cp, nil))
+
+	require.Len(t, r.creates, 1)
+	assert.Equal(t, uint16(1500), r.creates[0].ctx.Length)
+	assert.Equal(t, uint8(0x12), r.creates[0].ctx.TCPFlags)
+}
+
+// --- benchmarks ------------------------------------------------------------
+
+// noopReporter is the cheapest possible reporter. Methods discard the event.
+type noopReporter struct{}
+
+func (noopReporter) ReportDrop(events.DropEvent)             {}
+func (noopReporter) ReportFlowCreate(events.FlowCreateEvent) {}
+func (noopReporter) ReportFlowEvict(events.FlowEvictEvent)   {}
+func (noopReporter) ReportRulesReload(events.RulesReloadEvent) {
+}
+
+// bufferedReporter demonstrates a realistic zero-alloc reporter: each event
+// is forwarded to a value-typed channel. The channel send is a memcpy into
+// the channel's pre-allocated ring buffer -- no heap traffic. A background
+// goroutine would drain these; the bench skips draining to keep the report
+// path pure.
+type bufferedReporter struct {
+	drops  chan events.DropEvent
+	flows  chan events.FlowCreateEvent
+	evicts chan events.FlowEvictEvent
+}
+
+func newBufferedReporter(cap int) *bufferedReporter {
+	return &bufferedReporter{
+		drops:  make(chan events.DropEvent, cap),
+		flows:  make(chan events.FlowCreateEvent, cap),
+		evicts: make(chan events.FlowEvictEvent, cap),
+	}
+}
+
+func (r *bufferedReporter) ReportDrop(e events.DropEvent) {
+	select {
+	case r.drops <- e:
+	default:
+	}
+}
+
+func (r *bufferedReporter) ReportFlowCreate(e events.FlowCreateEvent) {
+	select {
+	case r.flows <- e:
+	default:
+	}
+}
+
+func (r *bufferedReporter) ReportFlowEvict(e events.FlowEvictEvent) {
+	select {
+	case r.evicts <- e:
+	default:
+	}
+}
+
+func (r *bufferedReporter) ReportRulesReload(events.RulesReloadEvent) {}
+
+// pointerReporter is the anti-pattern: it takes the address of the incoming
+// event struct, which forces the callee-side copy onto the heap. Kept for
+// comparison so we can see the alloc cost an unwary reporter would incur.
+type pointerReporter struct {
+	last *events.DropEvent
+}
+
+func (r *pointerReporter) ReportDrop(e events.DropEvent) {
+	r.last = &e
+}
+
+func (r *pointerReporter) ReportFlowCreate(events.FlowCreateEvent) {}
+func (r *pointerReporter) ReportFlowEvict(events.FlowEvictEvent)   {}
+func (r *pointerReporter) ReportRulesReload(events.RulesReloadEvent) {
+}
+
+func newBenchFixture(b *testing.B) *eventFixture {
+	b.Helper()
+	l := test.NewLogger()
+
+	vpnNetworks := new(bart.Lite)
+	vpnNetworks.Insert(netip.MustParsePrefix("1.2.3.0/24"))
+
+	c := &dummyCert{
+		name:     "host1",
+		networks: []netip.Prefix{netip.MustParsePrefix("1.2.3.4/24")},
+		groups:   []string{"default-group"},
+		issuer:   "signer-shasum",
+	}
+	h := &HostInfo{
+		ConnectionState: &ConnectionState{
+			peerCert: &cert.CachedCertificate{
+				Certificate:    c,
+				InvertedGroups: map[string]struct{}{"default-group": {}},
+			},
+		},
+		vpnAddrs: []netip.Addr{netip.MustParseAddr("1.2.3.4")},
+	}
+	h.buildNetworks(vpnNetworks, c)
+
+	fw := NewFirewall(l, time.Minute, time.Minute, time.Minute, c)
+	// Inbound rule that matches our packet; outbound has no match so we can
+	// also benchmark the no-rule drop path.
+	if err := fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""); err != nil {
+		b.Fatal(err)
+	}
+
+	ctl := &Control{f: &Interface{firewall: fw}, l: l}
+	return &eventFixture{
+		ctl: ctl,
+		fw:  fw,
+		p: firewall.Packet{
+			LocalAddr:  netip.MustParseAddr("1.2.3.4"),
+			RemoteAddr: netip.MustParseAddr("1.2.3.4"),
+			LocalPort:  10,
+			RemotePort: 90,
+			Protocol:   firewall.ProtoUDP,
+		},
+		h:  h,
+		cp: cert.NewCAPool(),
+	}
+}
+
+// BenchmarkFirewallDropPath measures the cost of Firewall.Drop on a packet
+// that reaches the no-matching-rule branch (the longest drop path). Compare
+// reporter shapes:
+//
+//	nilReporter      -- no reporter installed (feature cost when off)
+//	noopReporter     -- reporter installed, methods discard args (minimum on-cost)
+//	bufferedReporter -- realistic zero-alloc reporter: value-typed channels
+//	pointerReporter  -- anti-pattern that takes &composite-literal (allocates)
+func BenchmarkFirewallDropPath(b *testing.B) {
+	run := func(b *testing.B, install func(*Control)) {
+		f := newBenchFixture(b)
+		install(f.ctl)
+		b.ReportAllocs()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			_ = f.firewall().Drop(f.p, firewall.PacketContext{}, false, f.h, f.cp, nil)
+		}
+	}
+
+	b.Run("nilReporter", func(b *testing.B) { run(b, func(*Control) {}) })
+	b.Run("noopReporter", func(b *testing.B) {
+		run(b, func(c *Control) { c.SetFirewallEventReporter(noopReporter{}) })
+	})
+	b.Run("bufferedReporter", func(b *testing.B) {
+		run(b, func(c *Control) { c.SetFirewallEventReporter(newBufferedReporter(1024)) })
+	})
+	b.Run("pointerReporter", func(b *testing.B) {
+		run(b, func(c *Control) { c.SetFirewallEventReporter(&pointerReporter{}) })
+	})
+}
+
+// BenchmarkConntrackCreate measures Firewall.Drop for an allowed inbound
+// packet on a fresh conntrack (so addConn fires each iteration).
+func BenchmarkConntrackCreate(b *testing.B) {
+	run := func(b *testing.B, install func(*Control)) {
+		f := newBenchFixture(b)
+		install(f.ctl)
+		b.ReportAllocs()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			resetConntrack(f.firewall())
+			_ = f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
+		}
+	}
+	b.Run("nilReporter", func(b *testing.B) { run(b, func(*Control) {}) })
+	b.Run("noopReporter", func(b *testing.B) {
+		run(b, func(c *Control) { c.SetFirewallEventReporter(noopReporter{}) })
+	})
+	b.Run("bufferedReporter", func(b *testing.B) {
+		run(b, func(c *Control) { c.SetFirewallEventReporter(newBufferedReporter(1024)) })
+	})
+}
+
+// BenchmarkConntrackHit measures the hot path where a flow is already in
+// conntrack and short-circuits rule evaluation. The reporter slot is checked
+// only on create/evict, so this bench should show the reporter having zero
+// impact regardless of install state.
+func BenchmarkConntrackHit(b *testing.B) {
+	b.Run("nilReporter", func(b *testing.B) {
+		f := newBenchFixture(b)
+		// Prime conntrack.
+		require.NoError(b, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+		b.ReportAllocs()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			_ = f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
+		}
+	})
+
+	b.Run("noopReporter", func(b *testing.B) {
+		f := newBenchFixture(b)
+		f.ctl.SetFirewallEventReporter(noopReporter{})
+		require.NoError(b, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+		b.ReportAllocs()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			_ = f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
+		}
+	})
+}

firewall_test.go

@@ -213,44 +213,44 @@ func TestFirewall_Drop(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// Drop outbound
-	assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil))
+	assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 	// Allow inbound
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 	// Allow outbound because conntrack
-	require.NoError(t, fw.Drop(p, false, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 
 	// test remote mismatch
 	oldRemote := p.RemoteAddr
 	p.RemoteAddr = netip.MustParseAddr("1.2.3.10")
-	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil), ErrInvalidRemoteIP)
 	p.RemoteAddr = oldRemote
 
 	// ensure signer doesn't get in the way of group checks
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum"))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad"))
-	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caSha doesn't drop on match
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad"))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum"))
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 
 	// ensure ca name doesn't get in the way of group checks
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", ""))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", ""))
-	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caName doesn't drop on match
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", ""))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", ""))
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 }
 
 func TestFirewall_DropV6(t *testing.T) {
@@ -292,44 +292,44 @@ func TestFirewall_DropV6(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// Drop outbound
-	assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil))
+	assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 	// Allow inbound
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 	// Allow outbound because conntrack
-	require.NoError(t, fw.Drop(p, false, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 
 	// test remote mismatch
 	oldRemote := p.RemoteAddr
 	p.RemoteAddr = netip.MustParseAddr("fd12::56")
-	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil), ErrInvalidRemoteIP)
 	p.RemoteAddr = oldRemote
 
 	// ensure signer doesn't get in the way of group checks
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum"))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad"))
-	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caSha doesn't drop on match
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad"))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum"))
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 
 	// ensure ca name doesn't get in the way of group checks
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", ""))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", ""))
-	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caName doesn't drop on match
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", ""))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", ""))
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 }
 
 func BenchmarkFirewallTable_match(b *testing.B) {
@@ -537,10 +537,10 @@ func TestFirewall_Drop2(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// h1/c1 lacks the proper groups
-	require.ErrorIs(t, fw.Drop(p, true, &h1, cp, nil), ErrNoMatchingRule)
+	require.ErrorIs(t, fw.Drop(p, firewall.PacketContext{}, true, &h1, cp, nil), ErrNoMatchingRule)
 	// c has the proper groups
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 }
 
 func TestFirewall_Drop3(t *testing.T) {
@@ -618,18 +618,18 @@ func TestFirewall_Drop3(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// c1 should pass because host match
-	require.NoError(t, fw.Drop(p, true, &h1, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h1, cp, nil))
 	// c2 should pass because ca sha match
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, true, &h2, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h2, cp, nil))
 	// c3 should fail because no match
 	resetConntrack(fw)
-	assert.Equal(t, fw.Drop(p, true, &h3, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h3, cp, nil), ErrNoMatchingRule)
 
 	// Test a remote address match
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "1.2.3.4/24", "", "", ""))
-	require.NoError(t, fw.Drop(p, true, &h1, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h1, cp, nil))
 }
 
 func TestFirewall_Drop3V6(t *testing.T) {
@@ -667,7 +667,7 @@ func TestFirewall_Drop3V6(t *testing.T) {
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
 	cp := cert.NewCAPool()
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "fd12::34/120", "", "", ""))
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 }
 
 func TestFirewall_DropConntrackReload(t *testing.T) {
@@ -709,12 +709,12 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// Drop outbound
-	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 	// Allow inbound
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 	// Allow outbound because conntrack
-	require.NoError(t, fw.Drop(p, false, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 
 	oldFw := fw
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
@@ -723,7 +723,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	fw.rulesVersion = oldFw.rulesVersion + 1
 
 	// Allow outbound because conntrack and new rules allow port 10
-	require.NoError(t, fw.Drop(p, false, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 
 	oldFw = fw
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
@@ -732,7 +732,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	fw.rulesVersion = oldFw.rulesVersion + 1
 
 	// Drop outbound because conntrack doesn't match new ruleset
-	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 }
 
 func TestFirewall_ICMPPortBehavior(t *testing.T) {
@@ -778,12 +778,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0
 			p.RemotePort = 0
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			require.NoError(t, fw.Drop(*p, true, &h, cp, nil))
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil))
 			//now also allow outbound
-			require.NoError(t, fw.Drop(*p, false, &h, cp, nil))
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil))
 		})
 
 		t.Run("nonzero ports", func(t *testing.T) {
@@ -791,12 +791,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0xabcd
 			p.RemotePort = 0x1234
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			require.NoError(t, fw.Drop(*p, true, &h, cp, nil))
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil))
 			//now also allow outbound
-			require.NoError(t, fw.Drop(*p, false, &h, cp, nil))
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil))
 		})
 	})
 
@@ -808,12 +808,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0
 			p.RemotePort = 0
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 			//now also allow outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 		})
 
 		t.Run("nonzero ports, still blocked", func(t *testing.T) {
@@ -821,12 +821,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0xabcd
 			p.RemotePort = 0x1234
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 			//now also allow outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 		})
 
 		t.Run("nonzero, matching ports, still blocked", func(t *testing.T) {
@@ -834,12 +834,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 80
 			p.RemotePort = 80
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			assert.Equal(t, fw.Drop(*p, true, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 			//now also allow outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 		})
 	})
 	t.Run("Any proto, any port", func(t *testing.T) {
@@ -851,12 +851,12 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0
 			p.RemotePort = 0
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			require.NoError(t, fw.Drop(*p, true, &h, cp, nil))
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil))
 			//now also allow outbound
-			require.NoError(t, fw.Drop(*p, false, &h, cp, nil))
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil))
 		})
 
 		t.Run("nonzero ports, allowed", func(t *testing.T) {
@@ -865,15 +865,15 @@ func TestFirewall_ICMPPortBehavior(t *testing.T) {
 			p.LocalPort = 0xabcd
 			p.RemotePort = 0x1234
 			// Drop outbound
-			assert.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 			// Allow inbound
 			resetConntrack(fw)
-			require.NoError(t, fw.Drop(*p, true, &h, cp, nil))
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil))
 			//now also allow outbound
-			require.NoError(t, fw.Drop(*p, false, &h, cp, nil))
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil))
 			//different ID is blocked
 			p.RemotePort++
-			require.Equal(t, fw.Drop(*p, false, &h, cp, nil), ErrNoMatchingRule)
+			require.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 		})
 	})
 
@@ -922,7 +922,7 @@ func TestFirewall_DropIPSpoofing(t *testing.T) {
 		Protocol:   firewall.ProtoUDP,
 		Fragment:   false,
 	}
-	assert.Equal(t, fw.Drop(p, true, &h1, cp, nil), ErrInvalidRemoteIP)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h1, cp, nil), ErrInvalidRemoteIP)
 }
 
 func BenchmarkLookup(b *testing.B) {
@@ -1336,7 +1336,7 @@ func (c *testcase) Test(t *testing.T, fw *Firewall) {
 	t.Helper()
 	cp := cert.NewCAPool()
 	resetConntrack(fw)
-	err := fw.Drop(c.p, true, c.h, cp, nil)
+	err := fw.Drop(c.p, firewall.PacketContext{}, true, c.h, cp, nil)
 	if c.err == nil {
 		require.NoError(t, err, "failed to not drop remote address %s", c.p.RemoteAddr)
 	} else {

hostmap.go

@@ -604,9 +604,9 @@ func (hm *HostMap) queryVpnAddr(vpnIp netip.Addr, promoteIfce *Interface) *HostI
 // unlockedAddHostInfo assumes you have a write-lock and will add a hostinfo object to the hostmap Indexes and RemoteIndexes maps.
 // If an entry exists for the Hosts table (vpnIp -> hostinfo) then the provided hostinfo will be made primary
 func (hm *HostMap) unlockedAddHostInfo(hostinfo *HostInfo, f *Interface) {
-	if f.serveDns {
+	if f.dnsServer != nil {
 		remoteCert := hostinfo.ConnectionState.peerCert
-		dnsR.Add(remoteCert.Certificate.Name()+".", hostinfo.vpnAddrs)
+		f.dnsServer.Add(remoteCert.Certificate.Name()+".", hostinfo.vpnAddrs)
 	}
 	for _, addr := range hostinfo.vpnAddrs {
 		hm.unlockedInnerAddHostInfo(addr, hostinfo, f)

inside.go

@@ -11,8 +11,8 @@ import (
 	"github.com/slackhq/nebula/routing"
 )
 
-func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache) {
+func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, fwCtx *firewall.PacketContext, nb, out []byte, q int, localCache firewall.ConntrackCache) {
-	err := newPacket(packet, false, fwPacket)
+	err := newPacket(packet, false, fwPacket, fwCtx)
 	if err != nil {
 		if f.l.Level >= logrus.DebugLevel {
 			f.l.WithField("packet", packet).Debugf("Error while validating outbound packet: %s", err)
@@ -66,7 +66,7 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 		return
 	}
 
-	dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
+	dropReason := f.firewall.Drop(*fwPacket, *fwCtx, false, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason == nil {
 		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
 
@@ -211,14 +211,15 @@ func (f *Interface) getOrHandshakeConsiderRouting(fwPacket *firewall.Packet, cac
 
 func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte) {
 	fp := &firewall.Packet{}
-	err := newPacket(p, false, fp)
+	ctx := &firewall.PacketContext{}
+	err := newPacket(p, false, fp, ctx)
 	if err != nil {
 		f.l.Warnf("error while parsing outgoing packet for firewall check; %v", err)
 		return
 	}
 
 	// check if packet is in outbound fw rules
-	dropReason := f.firewall.Drop(*fp, false, hostinfo, f.pki.GetCAPool(), nil)
+	dropReason := f.firewall.Drop(*fp, *ctx, false, hostinfo, f.pki.GetCAPool(), nil)
 	if dropReason != nil {
 		if f.l.Level >= logrus.DebugLevel {
 			f.l.WithField("fwPacket", fp).

interface.go

@@ -6,7 +6,7 @@ import (
 	"fmt"
 	"io"
 	"net/netip"
-	"os"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -29,7 +29,7 @@ type InterfaceConfig struct {
 	pki                *PKI
 	Cipher             string
 	Firewall           *Firewall
-	ServeDns           bool
+	DnsServer          *dnsServer
 	HandshakeManager   *HandshakeManager
 	lightHouse         *LightHouse
 	connectionManager  *connectionManager
@@ -57,7 +57,7 @@ type Interface struct {
 	firewall              *Firewall
 	connectionManager     *connectionManager
 	handshakeManager      *HandshakeManager
-	serveDns              bool
+	dnsServer             *dnsServer
 	createTime            time.Time
 	lightHouse            *LightHouse
 	myBroadcastAddrsTable *bart.Lite
@@ -85,8 +85,16 @@ type Interface struct {
 
 	conntrackCacheTimeout time.Duration
 
+	ctx     context.Context
 	writers []udp.Conn
 	readers []io.ReadWriteCloser
+	wg      sync.WaitGroup
+
+	// fatalErr holds the first unexpected reader error that caused shutdown.
+	// nil means "no fatal error" (yet)
+	fatalErr atomic.Pointer[error]
+	// triggerShutdown is a function that will be run exactly once, when onFatal swaps something non-nil into fatalErr
+	triggerShutdown func()
 
 	metricHandshakes    metrics.Histogram
 	messageMetrics      *MessageMetrics
@@ -163,12 +171,13 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 
 	cs := c.pki.getCertState()
 	ifce := &Interface{
+		ctx:                   ctx,
 		pki:                   c.pki,
 		hostMap:               c.HostMap,
 		outside:               c.Outside,
 		inside:                c.Inside,
 		firewall:              c.Firewall,
-		serveDns:              c.ServeDns,
+		dnsServer:             c.DnsServer,
 		handshakeManager:      c.HandshakeManager,
 		createTime:            time.Now(),
 		lightHouse:            c.lightHouse,
@@ -209,7 +218,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 // activate creates the interface on the host. After the interface is created, any
 // other services that want to bind listeners to its IP may do so successfully. However,
 // the interface isn't going to process anything until run() is called.
-func (f *Interface) activate() {
+func (f *Interface) activate() error {
 	// actually turn on tun dev
 
 	addr, err := f.outside.LocalAddr()
@@ -237,27 +246,54 @@ func (f *Interface) activate() {
 		if i > 0 {
 			reader, err = f.inside.NewMultiQueueReader()
 			if err != nil {
-				f.l.Fatal(err)
+				return err
 			}
 		}
 		f.readers[i] = reader
 	}
 
-	if err := f.inside.Activate(); err != nil {
+	f.wg.Add(1) // for us to wait on Close() to return
+	if err = f.inside.Activate(); err != nil {
+		f.wg.Done()
 		f.inside.Close()
-		f.l.Fatal(err)
+		return err
 	}
+
+	return nil
 }
 
-func (f *Interface) run() {
+func (f *Interface) run() (func() error, error) {
 	// Launch n queues to read packets from udp
 	for i := 0; i < f.routines; i++ {
-		go f.listenOut(i)
+		f.wg.Go(func() {
+			f.listenOut(i)
+		})
 	}
 
 	// Launch n queues to read packets from tun dev
 	for i := 0; i < f.routines; i++ {
-		go f.listenIn(f.readers[i], i)
+		f.wg.Go(func() {
+			f.listenIn(f.readers[i], i)
+		})
+	}
+
+	return func() error {
+		f.wg.Wait()
+		if e := f.fatalErr.Load(); e != nil {
+			return *e
+		}
+		return nil
+	}, nil
+}
+
+// onFatal stores the first fatal reader error, and calls triggerShutdown if it was the first one
+func (f *Interface) onFatal(err error) {
+	swapped := f.fatalErr.CompareAndSwap(nil, &err)
+	if !swapped {
+		return
+	}
+	if f.triggerShutdown != nil {
+		f.triggerShutdown()
 	}
 }
 
@@ -269,40 +305,49 @@ func (f *Interface) listenOut(i int) {
 		li = f.outside
 	}
 
-	ctCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
+	ctCache := firewall.NewConntrackCacheTicker(f.ctx, f.conntrackCacheTimeout)
 	lhh := f.lightHouse.NewRequestHandler()
 	plaintext := make([]byte, udp.MTU)
 	h := &header.H{}
 	fwPacket := &firewall.Packet{}
+	fwCtx := &firewall.PacketContext{}
 	nb := make([]byte, 12, 12)
 
-	li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) {
+	err := li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) {
-		f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, plaintext[:0], payload, h, fwPacket, lhh, nb, i, ctCache.Get(f.l))
+		f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, plaintext[:0], payload, h, fwPacket, fwCtx, lhh, nb, i, ctCache.Get(f.l))
 	})
+
+	if err != nil && !f.closed.Load() {
+		f.l.WithError(err).Error("Error while reading inbound packet, closing")
+		f.onFatal(err)
+	}
+
+	f.l.Debugf("underlay reader %v is done", i)
 }
 
 func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
 	packet := make([]byte, mtu)
 	out := make([]byte, mtu)
 	fwPacket := &firewall.Packet{}
+	fwCtx := &firewall.PacketContext{}
 	nb := make([]byte, 12, 12)
 
-	conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
+	conntrackCache := firewall.NewConntrackCacheTicker(f.ctx, f.conntrackCacheTimeout)
 
 	for {
 		n, err := reader.Read(packet)
 		if err != nil {
-			if errors.Is(err, os.ErrClosed) && f.closed.Load() {
+			if !f.closed.Load() {
-				return
+				f.l.WithError(err).WithField("reader", i).Error("Error while reading outbound packet, closing")
+				f.onFatal(err)
 			}
-
+			break
-			f.l.WithError(err).Error("Error while reading outbound packet")
-			// This only seems to happen when something fatal happens to the fd, so exit.
-			os.Exit(2)
 		}
 
-		f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get(f.l))
+		f.consumeInsidePacket(packet[:n], fwPacket, fwCtx, nb, out, i, conntrackCache.Get(f.l))
 	}
+
+	f.l.Debugf("overlay reader %v is done", i)
 }
 
 func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
@@ -357,8 +402,15 @@ func (f *Interface) reloadFirewall(c *config.C) {
 		fw.Conntrack = conntrack
 	}
 
+	fw.reporter = oldFw.reporter
+
 	f.firewall = fw
 
+	// Fire ReportRulesReload under the conntrack lock so the reporter cannot
+	// observe a FlowCreate/FlowEvict for the new rulesVersion before it
+	// observes the reload marker. Report* must be non-blocking.
+	fw.reportRulesReload(oldFw.rulesVersion, fw.rulesVersion)
+
 	oldFw.Destroy()
 	f.l.WithField("firewallHashes", fw.GetRuleHashes()).
 		WithField("oldFirewallHashes", oldFw.GetRuleHashes()).
@@ -477,15 +529,23 @@ func (f *Interface) GetCertState() *CertState {
 }
 
 func (f *Interface) Close() error {
+	var errs []error
 	f.closed.Store(true)
 
-	for _, u := range f.writers {
+	// Release the udp readers
+	for i, u := range f.writers {
 		err := u.Close()
 		if err != nil {
-			f.l.WithError(err).Error("Error while closing udp socket")
+			f.l.WithError(err).WithField("writer", i).Error("Error while closing udp socket")
+			errs = append(errs, err)
 		}
 	}
 
 	// Release the tun device (closing the tun also closes all readers)
-	return f.inside.Close()
+	closeErr := f.inside.Close()
+	if closeErr != nil {
+		errs = append(errs, closeErr)
+	}
+	f.wg.Done()
+	return errors.Join(errs...)
 }

main.go

@@ -215,13 +215,9 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	handshakeManager := NewHandshakeManager(l, hostMap, lightHouse, udpConns[0], handshakeConfig)
 	lightHouse.handshakeTrigger = handshakeManager.trigger
 
-	serveDns := false
+	ds, err := newDnsServerFromConfig(ctx, l, pki.getCertState(), hostMap, c)
-	if c.GetBool("lighthouse.serve_dns", false) {
+	if err != nil {
-		if c.GetBool("lighthouse.am_lighthouse", false) {
+		l.WithError(err).Warn("Failed to start DNS responder")
-			serveDns = true
-		} else {
-			l.Warn("DNS server refusing to run because this host is not a lighthouse.")
-		}
 	}
 
 	ifConfig := &InterfaceConfig{
@@ -230,7 +226,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		Outside:               udpConns[0],
 		pki:                   pki,
 		Firewall:              fw,
-		ServeDns:              serveDns,
+		DnsServer:             ds,
 		HandshakeManager:      handshakeManager,
 		connectionManager:     connManager,
 		lightHouse:            lightHouse,
@@ -280,23 +276,17 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 
 	attachCommands(l, c, ssh, ifce)
 
-	// Start DNS server last to allow using the nebula IP as lighthouse.dns.host
-	var dnsStart func()
-	if lightHouse.amLighthouse && serveDns {
-		l.Debugln("Starting dns server")
-		dnsStart = dnsMain(l, pki.getCertState(), hostMap, c)
-	}
-
 	return &Control{
-		ifce,
+		state:                  StateReady,
-		l,
+		f:                      ifce,
-		ctx,
+		l:                      l,
-		cancel,
+		ctx:                    ctx,
-		sshStart,
+		cancel:                 cancel,
-		statsStart,
+		sshStart:               sshStart,
-		dnsStart,
+		statsStart:             statsStart,
-		lightHouse.StartUpdateWorker,
+		dnsStart:               ds.Start,
-		connManager.Start,
+		lighthouseStart:        lightHouse.StartUpdateWorker,
+		connectionManagerStart: connManager.Start,
 	}, nil
 }
 

outside.go

@@ -19,7 +19,7 @@ const (
 	minFwPacketLen = 4
 )
 
-func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
+func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, fwCtx *firewall.PacketContext, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
 	err := h.Parse(packet)
 	if err != nil {
 		// Hole punch packets are 0 or 1 byte big, so lets ignore printing those errors
@@ -60,7 +60,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 
 		switch h.Subtype {
 		case header.MessageNone:
-			if !f.decryptToTun(hostinfo, h.MessageCounter, out, packet, fwPacket, nb, q, localCache) {
+			if !f.decryptToTun(hostinfo, h.MessageCounter, out, packet, fwPacket, fwCtx, nb, q, localCache) {
 				return
 			}
 		case header.MessageRelay:
@@ -102,7 +102,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 					relay:     relay,
 					IsRelayed: true,
 				}
-				f.readOutsidePackets(via, out[:0], signedPayload, h, fwPacket, lhf, nb, q, localCache)
+				f.readOutsidePackets(via, out[:0], signedPayload, h, fwPacket, fwCtx, lhf, nb, q, localCache)
 				return
 			case ForwardingType:
 				// Find the target HostInfo relay object
@@ -295,7 +295,10 @@ var (
 )
 
 // newPacket validates and parses the interesting bits for the firewall out of the ip and sub protocol headers
-func newPacket(data []byte, incoming bool, fp *firewall.Packet) error {
+func newPacket(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.PacketContext) error {
+	if ctx != nil {
+		*ctx = firewall.PacketContext{}
+	}
 	if len(data) < 1 {
 		return ErrPacketTooShort
 	}
@@ -303,14 +306,14 @@ func newPacket(data []byte, incoming bool, fp *firewall.Packet) error {
 	version := int((data[0] >> 4) & 0x0f)
 	switch version {
 	case ipv4.Version:
-		return parseV4(data, incoming, fp)
+		return parseV4(data, incoming, fp, ctx)
 	case ipv6.Version:
-		return parseV6(data, incoming, fp)
+		return parseV6(data, incoming, fp, ctx)
 	}
 	return ErrUnknownIPVersion
 }
 
-func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
+func parseV6(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.PacketContext) error {
 	dataLen := len(data)
 	if dataLen < ipv6.HeaderLen {
 		return ErrIPv6PacketTooShort
@@ -355,6 +358,11 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 				fp.RemotePort = 0
 			}
 			fp.Fragment = false
+			if ctx != nil {
+				ctx.Length = binary.BigEndian.Uint16(data[4:6]) + uint16(ipv6.HeaderLen)
+				ctx.ICMPType = data[offset]
+				ctx.ICMPCode = data[offset+1]
+			}
 			return nil
 
 		case layers.IPProtocolTCP, layers.IPProtocolUDP:
@@ -372,6 +380,12 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 			}
 
 			fp.Fragment = false
+			if ctx != nil {
+				ctx.Length = binary.BigEndian.Uint16(data[4:6]) + uint16(ipv6.HeaderLen)
+				if proto == layers.IPProtocolTCP && dataLen >= offset+14 {
+					ctx.TCPFlags = data[offset+13]
+				}
+			}
 			return nil
 
 		case layers.IPProtocolIPv6Fragment:
@@ -423,7 +437,7 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 	return ErrIPv6CouldNotFindPayload
 }
 
-func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
+func parseV4(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.PacketContext) error {
 	// Do we at least have an ipv4 header worth of data?
 	if len(data) < ipv4.HeaderLen {
 		return ErrIPv4PacketTooShort
@@ -480,6 +494,21 @@ func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
 		fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
 	}
 
+	if ctx != nil {
+		ctx.Length = binary.BigEndian.Uint16(data[2:4])
+		if !fp.Fragment {
+			switch fp.Protocol {
+			case firewall.ProtoICMP:
+				ctx.ICMPType = data[ihl]
+				ctx.ICMPCode = data[ihl+1]
+			case firewall.ProtoTCP:
+				if len(data) >= ihl+14 {
+					ctx.TCPFlags = data[ihl+13]
+				}
+			}
+		}
+	}
+
 	return nil
 }
 
@@ -499,7 +528,7 @@ func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, out []byte, packet []
 	return out, nil
 }
 
-func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache) bool {
+func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out []byte, packet []byte, fwPacket *firewall.Packet, fwCtx *firewall.PacketContext, nb []byte, q int, localCache firewall.ConntrackCache) bool {
 	var err error
 
 	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
@@ -508,7 +537,7 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
 		return false
 	}
 
-	err = newPacket(out, true, fwPacket)
+	err = newPacket(out, true, fwPacket, fwCtx)
 	if err != nil {
 		hostinfo.logger(f.l).WithError(err).WithField("packet", out).
 			Warnf("Error while validating inbound packet")
@@ -521,7 +550,7 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
 		return false
 	}
 
-	dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache)
+	dropReason := f.firewall.Drop(*fwPacket, *fwCtx, true, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason != nil {
 		// NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore
 		// This gives us a buffer to build the reject packet in

outside_test.go

@@ -20,13 +20,13 @@ func Test_newPacket(t *testing.T) {
 	p := &firewall.Packet{}
 
 	// length fails
-	err := newPacket([]byte{}, true, p)
+	err := newPacket([]byte{}, true, p, nil)
 	require.ErrorIs(t, err, ErrPacketTooShort)
 
-	err = newPacket([]byte{0x40}, true, p)
+	err = newPacket([]byte{0x40}, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv4PacketTooShort)
 
-	err = newPacket([]byte{0x60}, true, p)
+	err = newPacket([]byte{0x60}, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 
 	// length fail with ip options
@@ -39,15 +39,15 @@ func Test_newPacket(t *testing.T) {
 	}
 
 	b, _ := h.Marshal()
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv4InvalidHeaderLength)
 
 	// not an ipv4 packet
-	err = newPacket([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p)
+	err = newPacket([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p, nil)
 	require.ErrorIs(t, err, ErrUnknownIPVersion)
 
 	// invalid ihl
-	err = newPacket([]byte{4<<4 | (8 >> 2 & 0x0f), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p)
+	err = newPacket([]byte{4<<4 | (8 >> 2 & 0x0f), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv4InvalidHeaderLength)
 
 	// account for variable ip header length - incoming
@@ -62,7 +62,7 @@ func Test_newPacket(t *testing.T) {
 
 	b, _ = h.Marshal()
 	b = append(b, []byte{0, 3, 0, 4}...)
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol)
@@ -84,7 +84,7 @@ func Test_newPacket(t *testing.T) {
 
 	b, _ = h.Marshal()
 	b = append(b, []byte{0, 5, 0, 6}...)
-	err = newPacket(b, false, p)
+	err = newPacket(b, false, p, nil)
 
 	require.NoError(t, err)
 	assert.Equal(t, uint8(2), p.Protocol)
@@ -114,7 +114,7 @@ func Test_newPacket_v6(t *testing.T) {
 	err := gopacket.SerializeLayers(buffer, opt, &ip)
 	require.NoError(t, err)
 
-	err = newPacket(buffer.Bytes(), true, p)
+	err = newPacket(buffer.Bytes(), true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 
 	// A v6 packet with a hop-by-hop extension
@@ -148,12 +148,12 @@ func Test_newPacket_v6(t *testing.T) {
 
 	// A full IPv6 header and 1 byte in the first extension, but missing
 	// the length byte.
-	err = newPacket(buffer.Bytes()[:41], true, p)
+	err = newPacket(buffer.Bytes()[:41], true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 
 	// A full IPv6 header plus 1 full extension, but only 1 byte of the
 	// next layer, missing length byte
-	err = newPacket(buffer.Bytes()[:49], true, p)
+	err = newPacket(buffer.Bytes()[:49], true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 	err = nil
 
@@ -173,7 +173,7 @@ func Test_newPacket_v6(t *testing.T) {
 
 	buffer.Clear()
 	require.NoError(t, gopacket.SerializeLayers(buffer, opt, &ip, &icmp))
-	require.Error(t, newPacket(buffer.Bytes(), true, p))
+	require.Error(t, newPacket(buffer.Bytes(), true, p, nil))
 
 	buffer.Clear()
 	echo := layers.ICMPv6Echo{
@@ -181,7 +181,7 @@ func Test_newPacket_v6(t *testing.T) {
 		SeqNumber:  1234,
 	}
 	require.NoError(t, gopacket.SerializeLayers(buffer, opt, &ip, &icmp, &echo))
-	require.NoError(t, newPacket(buffer.Bytes(), true, p))
+	require.NoError(t, newPacket(buffer.Bytes(), true, p, nil))
 	assert.Equal(t, uint8(layers.IPProtocolICMPv6), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr)
@@ -192,7 +192,7 @@ func Test_newPacket_v6(t *testing.T) {
 	// A good ESP packet
 	b := buffer.Bytes()
 	b[6] = byte(layers.IPProtocolESP)
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(layers.IPProtocolESP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -204,7 +204,7 @@ func Test_newPacket_v6(t *testing.T) {
 	// A good None packet
 	b = buffer.Bytes()
 	b[6] = byte(layers.IPProtocolNoNextHeader)
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(layers.IPProtocolNoNextHeader), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -216,7 +216,7 @@ func Test_newPacket_v6(t *testing.T) {
 	// An unknown protocol packet
 	b = buffer.Bytes()
 	b[6] = 255 // 255 is a reserved protocol number
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 
 	// A good UDP packet
@@ -243,7 +243,7 @@ func Test_newPacket_v6(t *testing.T) {
 	b = buffer.Bytes()
 
 	// incoming
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -253,7 +253,7 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// outgoing
-	err = newPacket(b, false, p)
+	err = newPacket(b, false, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
@@ -263,14 +263,14 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// Too short UDP packet
-	err = newPacket(b[:len(b)-10], false, p) // pull off the last 10 bytes
+	err = newPacket(b[:len(b)-10], false, p, nil) // pull off the last 10 bytes
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 
 	// A good TCP packet
 	b[6] = byte(layers.IPProtocolTCP)
 
 	// incoming
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -280,7 +280,7 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// outgoing
-	err = newPacket(b, false, p)
+	err = newPacket(b, false, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
@@ -290,7 +290,7 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// Too short TCP packet
-	err = newPacket(b[:len(b)-10], false, p) // pull off the last 10 bytes
+	err = newPacket(b[:len(b)-10], false, p, nil) // pull off the last 10 bytes
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 
 	// A good UDP packet with an AH header
@@ -325,7 +325,7 @@ func Test_newPacket_v6(t *testing.T) {
 	b = append(b, ahb...)
 	b = append(b, udpHeader...)
 
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -335,12 +335,12 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// Ensure buffer bounds checking during processing
-	err = newPacket(b[:41], true, p)
+	err = newPacket(b[:41], true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 
 	// Invalid AH header
 	b = buffer.Bytes()
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 }
 
@@ -388,7 +388,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	firstFrag = append(firstFrag, []byte{0xde, 0xad, 0xbe, 0xef}...)
 
 	// Test first fragment incoming
-	err = newPacket(firstFrag, true, p)
+	err = newPacket(firstFrag, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr)
@@ -398,7 +398,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// Test first fragment outgoing
-	err = newPacket(firstFrag, false, p)
+	err = newPacket(firstFrag, false, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.RemoteAddr)
@@ -427,7 +427,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	secondFrag = append(secondFrag, []byte{0xde, 0xad, 0xbe, 0xef}...)
 
 	// Test second fragment incoming
-	err = newPacket(secondFrag, true, p)
+	err = newPacket(secondFrag, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr)
@@ -437,7 +437,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	assert.True(t, p.Fragment)
 
 	// Test second fragment outgoing
-	err = newPacket(secondFrag, false, p)
+	err = newPacket(secondFrag, false, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.RemoteAddr)
@@ -447,7 +447,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	assert.True(t, p.Fragment)
 
 	// Too short of a fragment packet
-	err = newPacket(secondFrag[:len(secondFrag)-10], false, p)
+	err = newPacket(secondFrag[:len(secondFrag)-10], false, p, nil)
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 }
 
@@ -529,7 +529,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("Normal", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(normalPacket, true, fp); err != nil {
+			if err = parseV6(normalPacket, true, fp, nil); err != nil {
 				b.Fatal(err)
 			}
 		}
@@ -537,7 +537,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("FirstFragment", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(firstFrag, true, fp); err != nil {
+			if err = parseV6(firstFrag, true, fp, nil); err != nil {
 				b.Fatal(err)
 			}
 		}
@@ -545,7 +545,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("SecondFragment", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(secondFrag, true, fp); err != nil {
+			if err = parseV6(secondFrag, true, fp, nil); err != nil {
 				b.Fatal(err)
 			}
 		}
@@ -590,7 +590,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("200 HopByHop headers", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(evilBytes, false, fp); err != nil {
+			if err = parseV6(evilBytes, false, fp, nil); err != nil {
 				b.Fatal(err)
 			}
 		}

overlay/tun_file_linux_test.go

@@ -0,0 +1,120 @@
+//go:build linux && !android && !e2e_testing
+// +build linux,!android,!e2e_testing
+
+package overlay
+
+import (
+	"errors"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// newReadPipe returns a read fd. The matching write fd is registered for cleanup.
+// The caller takes ownership of the read fd (pass it to newTunFd / newFriend).
+func newReadPipe(t *testing.T) int {
+	t.Helper()
+	var fds [2]int
+	if err := unix.Pipe2(fds[:], unix.O_CLOEXEC); err != nil {
+		t.Fatalf("pipe2: %v", err)
+	}
+	t.Cleanup(func() { _ = unix.Close(fds[1]) })
+	return fds[0]
+}
+
+func TestTunFile_WakeForShutdown_UnblocksRead(t *testing.T) {
+	tf, err := newTunFd(newReadPipe(t))
+	if err != nil {
+		t.Fatalf("newTunFd: %v", err)
+	}
+	t.Cleanup(func() { _ = tf.Close() })
+
+	done := make(chan error, 1)
+	go func() {
+		_, err := tf.Read(make([]byte, 64))
+		done <- err
+	}()
+
+	// Verify Read is actually blocked in poll.
+	select {
+	case err := <-done:
+		t.Fatalf("Read returned before shutdown signal: %v", err)
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	if err := tf.wakeForShutdown(); err != nil {
+		t.Fatalf("wakeForShutdown: %v", err)
+	}
+
+	select {
+	case err := <-done:
+		if !errors.Is(err, os.ErrClosed) {
+			t.Fatalf("expected os.ErrClosed, got %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("Read did not wake on shutdown")
+	}
+}
+
+func TestTunFile_WakeForShutdown_WakesFriends(t *testing.T) {
+	parent, err := newTunFd(newReadPipe(t))
+	if err != nil {
+		t.Fatalf("newTunFd: %v", err)
+	}
+	friend, err := parent.newFriend(newReadPipe(t))
+	if err != nil {
+		_ = parent.Close()
+		t.Fatalf("newFriend: %v", err)
+	}
+	t.Cleanup(func() {
+		_ = friend.Close()
+		_ = parent.Close()
+	})
+
+	readers := []*tunFile{parent, friend}
+	errs := make([]error, len(readers))
+	var wg sync.WaitGroup
+	for i, r := range readers {
+		wg.Add(1)
+		go func(i int, r *tunFile) {
+			defer wg.Done()
+			_, errs[i] = r.Read(make([]byte, 64))
+		}(i, r)
+	}
+
+	time.Sleep(50 * time.Millisecond)
+
+	if err := parent.wakeForShutdown(); err != nil {
+		t.Fatalf("wakeForShutdown: %v", err)
+	}
+
+	done := make(chan struct{})
+	go func() { wg.Wait(); close(done) }()
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("readers did not wake")
+	}
+
+	for i, err := range errs {
+		if !errors.Is(err, os.ErrClosed) {
+			t.Errorf("reader %d: expected os.ErrClosed, got %v", i, err)
+		}
+	}
+}
+
+func TestTunFile_Close_Idempotent(t *testing.T) {
+	tf, err := newTunFd(newReadPipe(t))
+	if err != nil {
+		t.Fatalf("newTunFd: %v", err)
+	}
+	if err := tf.Close(); err != nil {
+		t.Fatalf("first Close: %v", err)
+	}
+	if err := tf.Close(); err != nil {
+		t.Fatalf("second Close should be a no-op, got %v", err)
+	}
+}

overlay/tun_freebsd.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"io/fs"
 	"net/netip"
+	"os"
 	"sync/atomic"
 	"syscall"
 	"time"
@@ -93,107 +94,184 @@ type tun struct {
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
 	linkAddr    *netroute.LinkAddr
 	l           *logrus.Logger
-	devFd       int
+
+	fd        int
+	shutdownR int // read end of the shutdown pipe; closing the write end wakes blocked polls
+	shutdownW int // write end of the shutdown pipe; closing this signals shutdown to any blocked reader/writer
+	readPoll  [2]unix.PollFd
+	writePoll [2]unix.PollFd
+	closed    atomic.Bool
+}
+
+// blockOnRead waits until the tun fd is readable or shutdown has been signaled.
+// Returns os.ErrClosed if Close was called.
+func (t *tun) blockOnRead() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(t.readPoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	tunEvents := t.readPoll[0].Revents
+	shutdownEvents := t.readPoll[1].Revents
+	t.readPoll[0].Revents = 0
+	t.readPoll[1].Revents = 0
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	}
+	if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
+}
+
+func (t *tun) blockOnWrite() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(t.writePoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	tunEvents := t.writePoll[0].Revents
+	shutdownEvents := t.writePoll[1].Revents
+	t.writePoll[0].Revents = 0
+	t.writePoll[1].Revents = 0
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	}
+	if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
 }
 
 func (t *tun) Read(to []byte) (int, error) {
-	// use readv() to read from the tunnel device, to eliminate the need for copying the buffer
-	if t.devFd < 0 {
-		return -1, syscall.EINVAL
-	}
-
 	// first 4 bytes is protocol family, in network byte order
-	head := make([]byte, 4)
+	var head [4]byte
-
+	iovecs := [2]syscall.Iovec{
-	iovecs := []syscall.Iovec{
 		{&head[0], 4},
 		{&to[0], uint64(len(to))},
 	}
-
+	for {
-	n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
+		n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
-
+		if errno == 0 {
-	var err error
+			bytesRead := int(n)
-	if errno != 0 {
+			if bytesRead < 4 {
-		err = syscall.Errno(errno)
+				return 0, nil
-	} else {
+			}
-		err = nil
+			return bytesRead - 4, nil
-	}
+		}
-	// fix bytes read number to exclude header
+		switch errno {
-	bytesRead := int(n)
+		case unix.EAGAIN:
-	if bytesRead < 0 {
+			if err := t.blockOnRead(); err != nil {
-		return bytesRead, err
+				return 0, err
-	} else if bytesRead < 4 {
+			}
-		return 0, err
+		case unix.EINTR:
-	} else {
+			// retry
-		return bytesRead - 4, err
+		case unix.EBADF:
+			return 0, os.ErrClosed
+		default:
+			return 0, errno
+		}
 	}
 }
 
 // Write is only valid for single threaded use
 func (t *tun) Write(from []byte) (int, error) {
-	// use writev() to write to the tunnel device, to eliminate the need for copying the buffer
-	if t.devFd < 0 {
-		return -1, syscall.EINVAL
-	}
-
 	if len(from) <= 1 {
 		return 0, syscall.EIO
 	}
+
 	ipVer := from[0] >> 4
-	var head []byte
+	var head [4]byte
 	// first 4 bytes is protocol family, in network byte order
-	if ipVer == 4 {
+	switch ipVer {
-		head = []byte{0, 0, 0, syscall.AF_INET}
+	case 4:
-	} else if ipVer == 6 {
+		head[3] = syscall.AF_INET
-		head = []byte{0, 0, 0, syscall.AF_INET6}
+	case 6:
-	} else {
+		head[3] = syscall.AF_INET6
+	default:
 		return 0, fmt.Errorf("unable to determine IP version from packet")
 	}
-	iovecs := []syscall.Iovec{
+
+	iovecs := [2]syscall.Iovec{
 		{&head[0], 4},
 		{&from[0], uint64(len(from))},
 	}
-
+	for {
-	n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
+		n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
-
+		if errno == 0 {
-	var err error
+			return int(n) - 4, nil
-	if errno != 0 {
+		}
-		err = syscall.Errno(errno)
+		switch errno {
-	} else {
+		case unix.EAGAIN:
-		err = nil
+			if err := t.blockOnWrite(); err != nil {
+				return 0, err
+			}
+		case unix.EINTR:
+			// retry
+		case unix.EBADF:
+			return 0, os.ErrClosed
+		default:
+			return 0, errno
+		}
 	}
-
-	return int(n) - 4, err
 }
 
 func (t *tun) Close() error {
-	if t.devFd >= 0 {
+	if t.closed.Swap(true) {
-		err := syscall.Close(t.devFd)
+		return nil
-		if err != nil {
+	}
+
+	// Closing the write end of the shutdown pipe causes any blocked Poll to
+	// return with POLLHUP on the shutdown fd, so readers/writers wake up and
+	// exit with os.ErrClosed.
+	if t.shutdownW >= 0 {
+		_ = unix.Close(t.shutdownW)
+		t.shutdownW = -1
+	}
+
+	if t.fd >= 0 {
+		if err := unix.Close(t.fd); err != nil {
 			t.l.WithError(err).Error("Error closing device")
 		}
-		t.devFd = -1
+		t.fd = -1
+	}
 
-		c := make(chan struct{})
+	if t.shutdownR >= 0 {
-		go func() {
+		_ = unix.Close(t.shutdownR)
-			// destroying the interface can block if a read() is still pending. Do this asynchronously.
+		t.shutdownR = -1
-			defer close(c)
+	}
-			s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
-			if err == nil {
-				defer syscall.Close(s)
-				ifreq := ifreqDestroy{Name: t.deviceBytes()}
-				err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
-			}
-			if err != nil {
-				t.l.WithError(err).Error("Error destroying tunnel")
-			}
-		}()
 
-		// wait up to 1 second so we start blocking at the ioctl
+	c := make(chan struct{})
-		select {
+	go func() {
-		case <-c:
+		// destroying the interface can block if a read() is still pending. Do this asynchronously.
-		case <-time.After(1 * time.Second):
+		defer close(c)
+		s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
+		if err == nil {
+			defer syscall.Close(s)
+			ifreq := ifreqDestroy{Name: t.deviceBytes()}
+			err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
 		}
+		if err != nil {
+			t.l.WithError(err).Error("Error destroying tunnel")
+		}
+	}()
+
+	// wait up to 1 second so we start blocking at the ioctl
+	select {
+	case <-c:
+	case <-time.After(1 * time.Second):
 	}
 
 	return nil
@@ -209,16 +287,38 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 	var err error
 	deviceName := c.GetString("tun.dev", "")
 	if deviceName != "" {
-		fd, err = syscall.Open("/dev/"+deviceName, syscall.O_RDWR, 0)
+		fd, err = unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
 	}
 	if errors.Is(err, fs.ErrNotExist) || deviceName == "" {
 		// If the device doesn't already exist, request a new one and rename it
-		fd, err = syscall.Open("/dev/tun", syscall.O_RDWR, 0)
+		fd, err = unix.Open("/dev/tun", os.O_RDWR, 0)
 	}
 	if err != nil {
 		return nil, err
 	}
 
+	if err = unix.SetNonblock(fd, true); err != nil {
+		_ = unix.Close(fd)
+		return nil, fmt.Errorf("failed to set tun device as nonblocking: %w", err)
+	}
+
+	// Shutdown pipe lets Close wake any reader/writer blocked in Poll.
+	var pipeFds [2]int
+	if err = unix.Pipe2(pipeFds[:], unix.O_CLOEXEC|unix.O_NONBLOCK); err != nil {
+		_ = unix.Close(fd)
+		return nil, fmt.Errorf("failed to create shutdown pipe: %w", err)
+	}
+	shutdownR, shutdownW := pipeFds[0], pipeFds[1]
+
+	closeOnErr := true
+	defer func() {
+		if closeOnErr {
+			_ = unix.Close(fd)
+			_ = unix.Close(shutdownR)
+			_ = unix.Close(shutdownW)
+		}
+	}()
+
 	// Read the name of the interface
 	var name [16]byte
 	arg := fiodgnameArg{length: 16, buf: unsafe.Pointer(&name)}
@@ -237,7 +337,7 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 	}
 
 	if ctrlErr != nil {
-		return nil, err
+		return nil, ctrlErr
 	}
 
 	ifName := string(bytes.TrimRight(name[:], "\x00"))
@@ -253,8 +353,6 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		}
 		defer syscall.Close(s)
 
-		fd := uintptr(s)
-
 		var fromName [16]byte
 		var toName [16]byte
 		copy(fromName[:], ifName)
@@ -266,7 +364,7 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		}
 
 		// Set the device name
-		_ = ioctl(fd, syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr)))
+		_ = ioctl(uintptr(s), syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr)))
 	}
 
 	t := &tun{
@@ -274,13 +372,24 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		vpnNetworks: vpnNetworks,
 		MTU:         c.GetInt("tun.mtu", DefaultMTU),
 		l:           l,
-		devFd:       fd,
+		fd:          fd,
+		shutdownR:   shutdownR,
+		shutdownW:   shutdownW,
+		readPoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLIN},
+			{Fd: int32(shutdownR), Events: unix.POLLIN},
+		},
+		writePoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLOUT},
+			{Fd: int32(shutdownR), Events: unix.POLLIN},
+		},
 	}
 
 	err = t.reload(c, true)
 	if err != nil {
 		return nil, err
 	}
+	closeOnErr = false
 
 	c.RegisterReloadCallback(func(c *config.C) {
 		err := t.reload(c, false)

overlay/tun_linux.go

@@ -28,10 +28,12 @@ import (
 // tunFile wraps a TUN file descriptor with poll-based reads. The FD provided will be changed to non-blocking.
 // A shared eventfd allows Close to wake all readers blocked in poll.
 type tunFile struct {
-	fd      int
+	fd         int
-	lastOne bool
+	shutdownFd int
-	pollFds [2]unix.PollFd
+	lastOne    bool
-	closed  bool
+	readPoll   [2]unix.PollFd
+	writePoll  [2]unix.PollFd
+	closed     bool
 }
 
 // newFriend makes a tunFile for a MultiQueueReader that copies the shutdown eventfd from the parent tun
@@ -40,10 +42,15 @@ func (r *tunFile) newFriend(fd int) (*tunFile, error) {
 		return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
 	}
 	return &tunFile{
-		fd: fd,
+		fd:         fd,
-		pollFds: [2]unix.PollFd{
+		shutdownFd: r.shutdownFd,
+		readPoll: [2]unix.PollFd{
 			{Fd: int32(fd), Events: unix.POLLIN},
-			{Fd: r.pollFds[1].Fd, Events: unix.POLLIN},
+			{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
+		},
+		writePoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLOUT},
+			{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
 		},
 	}, nil
 }
@@ -59,31 +66,62 @@ func newTunFd(fd int) (*tunFile, error) {
 	}
 
 	out := &tunFile{
-		fd:      fd,
+		fd:         fd,
-		lastOne: true,
+		shutdownFd: shutdownFd,
-		pollFds: [2]unix.PollFd{
+		lastOne:    true,
+		readPoll: [2]unix.PollFd{
 			{Fd: int32(fd), Events: unix.POLLIN},
 			{Fd: int32(shutdownFd), Events: unix.POLLIN},
 		},
+		writePoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLOUT},
+			{Fd: int32(shutdownFd), Events: unix.POLLIN},
+		},
 	}
 
 	return out, nil
 }
 
-func (r *tunFile) block() error {
+func (r *tunFile) blockOnRead() error {
 	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
 	var err error
 	for {
-		_, err = unix.Poll(r.pollFds[:], -1)
+		_, err = unix.Poll(r.readPoll[:], -1)
 		if err != unix.EINTR {
 			break
 		}
 	}
 	//always reset these!
-	tunEvents := r.pollFds[0].Revents
+	tunEvents := r.readPoll[0].Revents
-	shutdownEvents := r.pollFds[1].Revents
+	shutdownEvents := r.readPoll[1].Revents
-	r.pollFds[0].Revents = 0
+	r.readPoll[0].Revents = 0
-	r.pollFds[1].Revents = 0
+	r.readPoll[1].Revents = 0
+	//do the err check before trusting the potentially bogus bits we just got
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	} else if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
+}
+
+func (r *tunFile) blockOnWrite() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(r.writePoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	//always reset these!
+	tunEvents := r.writePoll[0].Revents
+	shutdownEvents := r.writePoll[1].Revents
+	r.writePoll[0].Revents = 0
+	r.writePoll[1].Revents = 0
 	//do the err check before trusting the potentially bogus bits we just got
 	if err != nil {
 		return err
@@ -101,10 +139,14 @@ func (r *tunFile) Read(buf []byte) (int, error) {
 		if n, err := unix.Read(r.fd, buf); err == nil {
 			return n, nil
 		} else if err == unix.EAGAIN {
-			if err = r.block(); err != nil {
+			if err = r.blockOnRead(); err != nil {
 				return 0, err
 			}
 			continue
+		} else if err == unix.EINTR {
+			continue
+		} else if err == unix.EBADF {
+			return 0, os.ErrClosed
 		} else {
 			return 0, err
 		}
@@ -112,13 +154,28 @@ func (r *tunFile) Read(buf []byte) (int, error) {
 }
 
 func (r *tunFile) Write(buf []byte) (int, error) {
-	return unix.Write(r.fd, buf)
+	for {
+		if n, err := unix.Write(r.fd, buf); err == nil {
+			return n, nil
+		} else if err == unix.EAGAIN {
+			if err = r.blockOnWrite(); err != nil {
+				return 0, err
+			}
+			continue
+		} else if err == unix.EINTR {
+			continue
+		} else if err == unix.EBADF {
+			return 0, os.ErrClosed
+		} else {
+			return 0, err
+		}
+	}
 }
 
 func (r *tunFile) wakeForShutdown() error {
 	var buf [8]byte
 	binary.NativeEndian.PutUint64(buf[:], 1)
-	_, err := unix.Write(int(r.pollFds[1].Fd), buf[:])
+	_, err := unix.Write(int(r.readPoll[1].Fd), buf[:])
 	return err
 }
 
@@ -128,7 +185,7 @@ func (r *tunFile) Close() error {
 	}
 	r.closed = true
 	if r.lastOne {
-		_ = unix.Close(int(r.pollFds[1].Fd))
+		_ = unix.Close(r.shutdownFd)
 	}
 	return unix.Close(r.fd)
 }
@@ -831,18 +888,18 @@ func (t *tun) Close() error {
 		}
 		err := t.readers[i].Close()
 		if err != nil {
-			t.l.WithField("reader", i).WithError(err).Error("Error closing tun reader")
+			t.l.WithField("reader", i).WithError(err).Error("error closing tun reader")
 		} else {
-			t.l.WithField("reader", i).Info("Closed tun reader")
+			t.l.WithField("reader", i).Info("closed tun reader")
 		}
 	}
 
 	//this is t.readers[0] too
 	err := t.tunFile.Close()
 	if err != nil {
-		t.l.WithField("reader", 0).WithError(err).Error("Error closing tun reader")
+		t.l.WithField("reader", 0).WithError(err).Error("error closing tun reader")
 	} else {
-		t.l.WithField("reader", 0).Info("Closed tun reader")
+		t.l.WithField("reader", 0).Info("closed tun reader")
 	}
 	return err
 }

service/service.go

@@ -44,7 +44,10 @@ type Service struct {
 }
 
 func New(control *nebula.Control) (*Service, error) {
-	control.Start()
+	wait, err := control.Start()
+	if err != nil {
+		return nil, err
+	}
 
 	ctx := control.Context()
 	eg, ctx := errgroup.WithContext(ctx)
@@ -141,6 +144,12 @@ func New(control *nebula.Control) (*Service, error) {
 		}
 	})
 
+	// Add the nebula wait function to the group so a fatal reader error
+	// propagates out through errgroup.Wait().
+	eg.Go(func() error {
+		return wait()
+	})
+
 	return &s, nil
 }
 

udp/conn.go

@@ -16,7 +16,7 @@ type EncReader func(
 type Conn interface {
 	Rebind() error
 	LocalAddr() (netip.AddrPort, error)
-	ListenOut(r EncReader)
+	ListenOut(r EncReader) error
 	WriteTo(b []byte, addr netip.AddrPort) error
 	ReloadConfig(c *config.C)
 	SupportsMultipleReaders() bool
@@ -31,8 +31,8 @@ func (NoopConn) Rebind() error {
 func (NoopConn) LocalAddr() (netip.AddrPort, error) {
 	return netip.AddrPort{}, nil
 }
-func (NoopConn) ListenOut(_ EncReader) {
+func (NoopConn) ListenOut(_ EncReader) error {
-	return
+	return nil
 }
 func (NoopConn) SupportsMultipleReaders() bool {
 	return false

udp/udp_darwin.go

@@ -165,7 +165,7 @@ func NewUDPStatsEmitter(udpConns []Conn) func() {
 	return func() {}
 }
 
-func (u *StdConn) ListenOut(r EncReader) {
+func (u *StdConn) ListenOut(r EncReader) error {
 	buffer := make([]byte, MTU)
 
 	for {
@@ -173,8 +173,7 @@ func (u *StdConn) ListenOut(r EncReader) {
 		n, rua, err := u.ReadFromUDPAddrPort(buffer)
 		if err != nil {
 			if errors.Is(err, net.ErrClosed) {
-				u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
+				return err
-				return
 			}
 
 			u.l.WithError(err).Error("unexpected udp socket receive error")

udp/udp_generic.go

@@ -73,7 +73,7 @@ type rawMessage struct {
 	Len uint32
 }
 
-func (u *GenericConn) ListenOut(r EncReader) {
+func (u *GenericConn) ListenOut(r EncReader) error {
 	buffer := make([]byte, MTU)
 
 	var lastRecvErr time.Time
@@ -83,8 +83,7 @@ func (u *GenericConn) ListenOut(r EncReader) {
 		n, rua, err := u.ReadFromUDPAddrPort(buffer)
 		if err != nil {
 			if errors.Is(err, net.ErrClosed) {
-				u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
+				return err
-				return
 			}
 			// Dampen unexpected message warns to once per minute
 			if lastRecvErr.IsZero() || time.Since(lastRecvErr) > time.Minute {

udp/udp_linux.go

@@ -171,7 +171,7 @@ func recvmmsg(fd uintptr, msgs []rawMessage) (int, bool, error) {
 	return int(n), true, nil
 }
 
-func (u *StdConn) listenOutSingle(r EncReader) {
+func (u *StdConn) listenOutSingle(r EncReader) error {
 	var err error
 	var n int
 	var from netip.AddrPort
@@ -180,15 +180,14 @@ func (u *StdConn) listenOutSingle(r EncReader) {
 	for {
 		n, from, err = u.udpConn.ReadFromUDPAddrPort(buffer)
 		if err != nil {
-			u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
+			return err
-			return
 		}
 		from = netip.AddrPortFrom(from.Addr().Unmap(), from.Port())
 		r(from, buffer[:n])
 	}
 }
 
-func (u *StdConn) listenOutBatch(r EncReader) {
+func (u *StdConn) listenOutBatch(r EncReader) error {
 	var ip netip.Addr
 	var n int
 	var operr error
@@ -205,12 +204,10 @@ func (u *StdConn) listenOutBatch(r EncReader) {
 	for {
 		err := u.rawConn.Read(reader)
 		if err != nil {
-			u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
+			return err
-			return
 		}
 		if operr != nil {
-			u.l.WithError(operr).Debug("operr: udp socket is closed, exiting read loop")
+			return operr
-			return
 		}
 
 		for i := 0; i < n; i++ {
@@ -225,14 +222,11 @@ func (u *StdConn) listenOutBatch(r EncReader) {
 	}
 }
 
-func (u *StdConn) ListenOut(r EncReader) {
+func (u *StdConn) ListenOut(r EncReader) error {
 	if u.batch == 1 {
-		//save some ram by not calling PrepareRawMessages for fields we won't use
+		return u.listenOutSingle(r)
-		//we could also make this path more common by calling recvmmsg with msgs[:1],
-		//but that's still the recvmmsg syscall, which would be a change
-		u.listenOutSingle(r)
 	} else {
-		u.listenOutBatch(r)
+		return u.listenOutBatch(r)
 	}
 }
 

udp/udp_rio_windows.go

@@ -140,7 +140,7 @@ func (u *RIOConn) bind(l *logrus.Logger, sa windows.Sockaddr) error {
 	return nil
 }
 
-func (u *RIOConn) ListenOut(r EncReader) {
+func (u *RIOConn) ListenOut(r EncReader) error {
 	buffer := make([]byte, MTU)
 
 	var lastRecvErr time.Time
@@ -151,8 +151,7 @@ func (u *RIOConn) ListenOut(r EncReader) {
 
 		if err != nil {
 			if errors.Is(err, net.ErrClosed) {
-				u.l.WithError(err).Debug("udp socket is closed, exiting read loop")
+				return err
-				return
 			}
 			// Dampen unexpected message warns to once per minute
 			if lastRecvErr.IsZero() || time.Since(lastRecvErr) > time.Minute {

udp/udp_tester.go

@@ -6,6 +6,7 @@ package udp
 import (
 	"io"
 	"net/netip"
+	"os"
 	"sync/atomic"
 
 	"github.com/sirupsen/logrus"
@@ -106,11 +107,11 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
 	return nil
 }
 
-func (u *TesterConn) ListenOut(r EncReader) {
+func (u *TesterConn) ListenOut(r EncReader) error {
 	for {
 		p, ok := <-u.RxPackets
 		if !ok {
-			return
+			return os.ErrClosed
 		}
 		r(p.From, p.Data)
 	}