Experimenting with fw event reporting

Bumps the zx2c4-dependencies group with 1 update: golang.zx2c4.com/wireguard/windows.


Updates `golang.zx2c4.com/wireguard/windows` from 0.5.3 to 0.6.1

---
updated-dependencies:
- dependency-name: golang.zx2c4.com/wireguard/windows
  dependency-version: 0.6.1
  dependency-type: direct:production
  update-type: version-update:semver-minor
  dependency-group: zx2c4-dependencies
...

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

.github/workflows/release.yml

@@ -24,7 +24,7 @@ jobs:
           mv build/*.tar.gz release
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: linux-latest
           path: release
@@ -55,7 +55,7 @@ jobs:
           mv dist\windows\wintun build\dist\windows\
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: windows-latest
           path: build
@@ -104,7 +104,7 @@ jobs:
           fi
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v5
+        uses: actions/upload-artifact@v6
         with:
           name: darwin-latest
           path: ./release/*
@@ -128,7 +128,7 @@ jobs:
 
       - name: Download artifacts
         if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v7
         with:
           name: linux-latest
           path: artifacts
@@ -163,7 +163,7 @@ jobs:
       - uses: actions/checkout@v6
 
       - name: Download artifacts
-        uses: actions/download-artifact@v6
+        uses: actions/download-artifact@v7
         with:
           path: artifacts
 
@@ -209,10 +209,11 @@ jobs:
         id: create_release
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_REF_NAME: ${{ github.ref_name }}
         run: |
           cd artifacts
           gh release create \
             --verify-tag \
-            --title "Release ${{ github.ref_name }}" \
-            "${{ github.ref_name }}" \
+            --title "Release ${GITHUB_REF_NAME}" \
+            "${GITHUB_REF_NAME}" \
             SHASUM256.txt *-latest/*.zip *-latest/*.tar.gz

.github/workflows/smoke-extra.yml

@@ -18,6 +18,8 @@ jobs:
     if: github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'smoke-test-extra')
     name: Run extra smoke tests
     runs-on: ubuntu-latest
+    env:
+      VAGRANT_DEFAULT_PROVIDER: libvirt
     steps:
 
     - uses: actions/checkout@v6
@@ -30,8 +32,13 @@ jobs:
     - name: add hashicorp source
       run: wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg && echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
 
-    - name: install vagrant
-      run: sudo apt-get update && sudo apt-get install -y vagrant virtualbox
+    - name: install vagrant and libvirt
+      run: |
+        sudo apt-get update && sudo apt-get install -y vagrant libvirt-daemon-system libvirt-dev
+        sudo chmod 666 /dev/kvm
+        sudo usermod -aG libvirt $(whoami)
+        sudo chmod 666 /var/run/libvirt/libvirt-sock
+        vagrant plugin install vagrant-libvirt
 
     - name: freebsd-amd64
       run: make smoke-vagrant/freebsd-amd64
@@ -42,10 +49,19 @@ jobs:
     - name: netbsd-amd64
       run: make smoke-vagrant/netbsd-amd64
 
-    - name: linux-386
-      run: make smoke-vagrant/linux-386
-
     - name: linux-amd64-ipv6disable
       run: make smoke-vagrant/linux-amd64-ipv6disable
 
+    # linux-386 runs last because it requires disabling KVM to use VirtualBox,
+    # which prevents libvirt (used by the other tests) from working after this point.
+    - name: install virtualbox for i386 test
+      run: |
+        sudo apt-get install -y virtualbox
+        sudo rmmod kvm_amd kvm_intel kvm 2>/dev/null || true
+
+    - name: linux-386
+      env:
+        VAGRANT_DEFAULT_PROVIDER: virtualbox
+      run: make smoke-vagrant/linux-386
+
     timeout-minutes: 30

.github/workflows/smoke/build-relay.sh

@@ -16,8 +16,10 @@ relay:
   am_relay: true
 EOF
 
-    export LIGHTHOUSES="192.168.100.1 172.17.0.2:4242"
-    export REMOTE_ALLOW_LIST='{"172.17.0.4/32": false, "172.17.0.5/32": false}'
+    # TEST-NET-3 placeholder IPs; smoke-relay.sh seds them to real container IPs.
+    # Mapping: .2 lighthouse1, .3 host2, .4 host3, .5 host4.
+    export LIGHTHOUSES="192.168.100.1 203.0.113.2:4242"
+    export REMOTE_ALLOW_LIST='{"203.0.113.4/32": false, "203.0.113.5/32": false}'
 
     HOST="host2" ../genconfig.sh >host2.yml <<EOF
 relay:
@@ -25,7 +27,7 @@ relay:
     - 192.168.100.1
 EOF
 
-    export REMOTE_ALLOW_LIST='{"172.17.0.3/32": false}'
+    export REMOTE_ALLOW_LIST='{"203.0.113.3/32": false}'
 
     HOST="host3" ../genconfig.sh >host3.yml
 

.github/workflows/smoke/build.sh

@@ -5,9 +5,15 @@ set -e -x
 rm -rf ./build
 mkdir ./build
 
-# TODO: Assumes your docker bridge network is a /24, and the first container that launches will be .1
-# - We could make this better by launching the lighthouse first and then fetching what IP it is.
-NET="$(docker network inspect bridge -f '{{ range .IPAM.Config }}{{ .Subnet }}{{ end }}' | cut -d. -f1-3)"
+# Smoke containers run on a dedicated docker network whose subnet is allocated
+# at smoke time, not known at build time. Configs are written with TEST-NET-3
+# placeholder IPs (RFC 5737) and smoke.sh / smoke-vagrant.sh / smoke-relay.sh
+# sed the real container IPs in before starting nebula.
+#
+# Placeholder mapping (last octet == fixed container slot):
+#   203.0.113.2 -> lighthouse1, 203.0.113.3 -> host2,
+#   203.0.113.4 -> host3,       203.0.113.5 -> host4.
+LIGHTHOUSE_IP="203.0.113.2"
 
 (
     cd build
@@ -25,16 +31,16 @@ NET="$(docker network inspect bridge -f '{{ range .IPAM.Config }}{{ .Subnet }}{{
         ../genconfig.sh >lighthouse1.yml
 
     HOST="host2" \
-        LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
+        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
         ../genconfig.sh >host2.yml
 
     HOST="host3" \
-        LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
+        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
         INBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
         ../genconfig.sh >host3.yml
 
     HOST="host4" \
-        LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
+        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
         OUTBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
         ../genconfig.sh >host4.yml
 

.github/workflows/smoke/smoke-relay.sh

@@ -6,6 +6,8 @@ set -o pipefail
 
 mkdir -p logs
 
+NETWORK="nebula-smoke-relay"
+
 cleanup() {
     echo
     echo " *** cleanup"
@@ -16,22 +18,53 @@ cleanup() {
     then
         docker kill lighthouse1 host2 host3 host4
     fi
+    docker network rm "$NETWORK" >/dev/null 2>&1
 }
 
 trap cleanup EXIT
 
-docker run --name lighthouse1 --rm nebula:smoke-relay -config lighthouse1.yml -test
-docker run --name host2 --rm nebula:smoke-relay -config host2.yml -test
-docker run --name host3 --rm nebula:smoke-relay -config host3.yml -test
-docker run --name host4 --rm nebula:smoke-relay -config host4.yml -test
+# Create a dedicated smoke network with an explicit subnet (required for --ip
+# below). Probe a short list of candidates so a locally-used range doesn't
+# fail the whole test — we only need one to be free.
+docker network rm "$NETWORK" >/dev/null 2>&1 || true
+for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
+    if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
+        break
+    fi
+done
+if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
+    echo "failed to create $NETWORK: every candidate subnet is in use" >&2
+    exit 1
+fi
 
-docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
+# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
+# .3 host2, .4 host3, .5 host4 — matches the placeholders in build-relay.sh.
+SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
+PREFIX="${SUBNET%/*}"
+PREFIX="${PREFIX%.*}"
+LIGHTHOUSE_IP="$PREFIX.2"
+HOST2_IP="$PREFIX.3"
+HOST3_IP="$PREFIX.4"
+HOST4_IP="$PREFIX.5"
+
+# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
+for f in build/host2.yml build/host3.yml build/host4.yml; do
+    sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
+    mv "$f.tmp" "$f"
+done
+
+docker run --name lighthouse1 --rm nebula:smoke-relay -config lighthouse1.yml -test
+docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" nebula:smoke-relay -config host2.yml -test
+docker run --name host3 --rm -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" nebula:smoke-relay -config host3.yml -test
+docker run --name host4 --rm -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" nebula:smoke-relay -config host4.yml -test
+
+docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
 sleep 1
-docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
+docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
 sleep 1
-docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
+docker run --name host3 --network "$NETWORK" --ip "$HOST3_IP" -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
 sleep 1
-docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
+docker run --name host4 --network "$NETWORK" --ip "$HOST4_IP" -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
 sleep 1
 
 set +x
@@ -76,7 +109,13 @@ docker exec host4 sh -c 'kill 1'
 docker exec host3 sh -c 'kill 1'
 docker exec host2 sh -c 'kill 1'
 docker exec lighthouse1 sh -c 'kill 1'
-sleep 5
+
+# Wait up to 30s for all backgrounded jobs to exit rather than relying on a
+# fixed sleep.
+for _ in $(seq 1 30); do
+    [ -z "$(jobs -r)" ] && break
+    sleep 1
+done
 
 if [ "$(jobs -r)" ]
 then

.github/workflows/smoke/smoke-vagrant.sh

@@ -8,6 +8,8 @@ export VAGRANT_CWD="$PWD/vagrant-$1"
 
 mkdir -p logs
 
+NETWORK="nebula-smoke"
+
 cleanup() {
     echo
     echo " *** cleanup"
@@ -19,21 +21,51 @@ cleanup() {
         docker kill lighthouse1 host2
     fi
     vagrant destroy -f
+    docker network rm "$NETWORK" >/dev/null 2>&1
 }
 
 trap cleanup EXIT
 
+# Create a dedicated smoke network with an explicit subnet (required for --ip
+# below). Probe a short list of candidates so a locally-used range doesn't
+# fail the whole test — we only need one to be free.
+docker network rm "$NETWORK" >/dev/null 2>&1 || true
+for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
+    if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
+        break
+    fi
+done
+if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
+    echo "failed to create $NETWORK: every candidate subnet is in use" >&2
+    exit 1
+fi
+
+# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
+# .3 host2 — matches the placeholders in build.sh.
+SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
+PREFIX="${SUBNET%/*}"
+PREFIX="${PREFIX%.*}"
+LIGHTHOUSE_IP="$PREFIX.2"
+HOST2_IP="$PREFIX.3"
+
+# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
+# This must happen before `vagrant up` rsyncs build/ into the VM for host3.
+for f in build/host2.yml build/host3.yml; do
+    sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
+    mv "$f.tmp" "$f"
+done
+
 CONTAINER="nebula:${NAME:-smoke}"
 
 docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test
-docker run --name host2 --rm "$CONTAINER" -config host2.yml -test
+docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" "$CONTAINER" -config host2.yml -test
 
 vagrant up
 vagrant ssh -c "cd /nebula && /nebula/$1-nebula -config host3.yml -test" -- -T
 
-docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
+docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
 sleep 1
-docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
+docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
 sleep 1
 vagrant ssh -c "cd /nebula && sudo sh -c 'echo \$\$ >/nebula/pid && exec /nebula/$1-nebula -config host3.yml'" 2>&1 -- -T | tee logs/host3 | sed -u 's/^/  [host3]  /' &
 sleep 15
@@ -96,7 +128,14 @@ vagrant ssh -c "ping -c1 192.168.100.2" -- -T
 vagrant ssh -c "sudo xargs kill </nebula/pid" -- -T
 docker exec host2 sh -c 'kill 1'
 docker exec lighthouse1 sh -c 'kill 1'
-sleep 1
+
+# Wait up to 30s for all backgrounded jobs to exit. vagrant ssh in particular
+# takes a beat to tear down after nebula exits on the VM, so a fixed sleep is
+# racy.
+for _ in $(seq 1 30); do
+    [ -z "$(jobs -r)" ] && break
+    sleep 1
+done
 
 if [ "$(jobs -r)" ]
 then

.github/workflows/smoke/smoke.sh

@@ -6,6 +6,8 @@ set -o pipefail
 
 mkdir -p logs
 
+NETWORK="nebula-smoke"
+
 cleanup() {
     echo
     echo " *** cleanup"
@@ -16,38 +18,71 @@ cleanup() {
     then
         docker kill lighthouse1 host2 host3 host4
     fi
+    docker network rm "$NETWORK" >/dev/null 2>&1
 }
 
 trap cleanup EXIT
 
+# Create a dedicated smoke network with an explicit subnet (required for --ip
+# below). Probe a short list of candidates so a locally-used range doesn't
+# fail the whole test — we only need one to be free.
+docker network rm "$NETWORK" >/dev/null 2>&1 || true
+for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
+    if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
+        break
+    fi
+done
+if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
+    echo "failed to create $NETWORK: every candidate subnet is in use" >&2
+    exit 1
+fi
+
+# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
+# .3 host2, .4 host3, .5 host4 — matches the placeholders in build.sh.
+SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
+PREFIX="${SUBNET%/*}"
+PREFIX="${PREFIX%.*}"
+LIGHTHOUSE_IP="$PREFIX.2"
+HOST2_IP="$PREFIX.3"
+HOST3_IP="$PREFIX.4"
+HOST4_IP="$PREFIX.5"
+
+# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
+# build/lighthouse1.yml has no IPs to rewrite so it's skipped.
+for f in build/host2.yml build/host3.yml build/host4.yml; do
+    sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
+    mv "$f.tmp" "$f"
+done
+
 CONTAINER="nebula:${NAME:-smoke}"
 
 docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test
-docker run --name host2 --rm "$CONTAINER" -config host2.yml -test
-docker run --name host3 --rm "$CONTAINER" -config host3.yml -test
-docker run --name host4 --rm "$CONTAINER" -config host4.yml -test
+docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" "$CONTAINER" -config host2.yml -test
+docker run --name host3 --rm -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" "$CONTAINER" -config host3.yml -test
+docker run --name host4 --rm -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" "$CONTAINER" -config host4.yml -test
 
-docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
+docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
 sleep 1
-docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
+docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
 sleep 1
-docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
+docker run --name host3 --network "$NETWORK" --ip "$HOST3_IP" -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
 sleep 1
-docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
+docker run --name host4 --network "$NETWORK" --ip "$HOST4_IP" -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
 sleep 1
 
 # grab tcpdump pcaps for debugging
-docker exec lighthouse1 tcpdump -i nebula1 -q -w - -U 2>logs/lighthouse1.inside.log >logs/lighthouse1.inside.pcap &
+docker exec lighthouse1 tcpdump -i tun0 -q -w - -U 2>logs/lighthouse1.inside.log >logs/lighthouse1.inside.pcap &
 docker exec lighthouse1 tcpdump -i eth0 -q -w - -U 2>logs/lighthouse1.outside.log >logs/lighthouse1.outside.pcap &
-docker exec host2 tcpdump -i nebula1 -q -w - -U 2>logs/host2.inside.log >logs/host2.inside.pcap &
+docker exec host2 tcpdump -i tun0 -q -w - -U 2>logs/host2.inside.log >logs/host2.inside.pcap &
 docker exec host2 tcpdump -i eth0 -q -w - -U 2>logs/host2.outside.log >logs/host2.outside.pcap &
-docker exec host3 tcpdump -i nebula1 -q -w - -U 2>logs/host3.inside.log >logs/host3.inside.pcap &
+docker exec host3 tcpdump -i tun0 -q -w - -U 2>logs/host3.inside.log >logs/host3.inside.pcap &
 docker exec host3 tcpdump -i eth0 -q -w - -U 2>logs/host3.outside.log >logs/host3.outside.pcap &
-docker exec host4 tcpdump -i nebula1 -q -w - -U 2>logs/host4.inside.log >logs/host4.inside.pcap &
+docker exec host4 tcpdump -i tun0 -q -w - -U 2>logs/host4.inside.log >logs/host4.inside.pcap &
 docker exec host4 tcpdump -i eth0 -q -w - -U 2>logs/host4.outside.log >logs/host4.outside.pcap &
 
 docker exec host2 ncat -nklv 0.0.0.0 2000 &
 docker exec host3 ncat -nklv 0.0.0.0 2000 &
+docker exec host4 ncat -nkluv 0.0.0.0 4000 &
 docker exec host2 ncat -e '/usr/bin/echo host2' -nkluv 0.0.0.0 3000 &
 docker exec host3 ncat -e '/usr/bin/echo host3' -nkluv 0.0.0.0 3000 &
 
@@ -119,17 +154,24 @@ echo
 echo " *** Testing conntrack"
 echo
 set -x
-# host2 can ping host3 now that host3 pinged it first
-docker exec host2 ping -c1 192.168.100.3
-# host4 can ping host2 once conntrack established
-docker exec host2 ping -c1 192.168.100.4
-docker exec host4 ping -c1 192.168.100.2
+
+# host2 speaking to host4 on UDP 4000 should allow it to reply, when firewall rules would normally not permit this
+docker exec host2 sh -c "/usr/bin/echo host2 | ncat -nuv 192.168.100.4 4000"
+docker exec host2 ncat -e '/usr/bin/echo helloagainfromhost2' -nkluv 0.0.0.0 4000 &
+sleep 1
+docker exec host4 sh -c "/usr/bin/echo host4 | ncat -nuv 192.168.100.2 4000"
 
 docker exec host4 sh -c 'kill 1'
 docker exec host3 sh -c 'kill 1'
 docker exec host2 sh -c 'kill 1'
 docker exec lighthouse1 sh -c 'kill 1'
-sleep 5
+
+# Wait up to 30s for all backgrounded jobs to exit rather than relying on a
+# fixed sleep.
+for _ in $(seq 1 30); do
+    [ -z "$(jobs -r)" ] && break
+    sleep 1
+done
 
 if [ "$(jobs -r)" ]
 then

.github/workflows/smoke/vagrant-linux-amd64-ipv6disable/Vagrantfile

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 Vagrant.configure("2") do |config|
-  config.vm.box = "ubuntu/jammy64"
+  config.vm.box = "bento/ubuntu-24.04"
 
   config.vm.synced_folder "../build", "/nebula"
 

.github/workflows/smoke/vagrant-openbsd-amd64/Vagrantfile

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 Vagrant.configure("2") do |config|
-  config.vm.box = "generic/openbsd7"
+  config.vm.box = "DefinedNet/openbsd78"
 
   config.vm.synced_folder "../build", "/nebula", type: "rsync"
 end

.github/workflows/test.yml

@@ -45,7 +45,7 @@ jobs:
     - name: Build test mobile
       run: make build-test-mobile
 
-    - uses: actions/upload-artifact@v5
+    - uses: actions/upload-artifact@v6
       with:
         name: e2e packet flow linux-latest
         path: e2e/mermaid/linux-latest
@@ -125,7 +125,7 @@ jobs:
     - name: End 2 end
       run: make e2evv
 
-    - uses: actions/upload-artifact@v5
+    - uses: actions/upload-artifact@v6
       with:
         name: e2e packet flow ${{ matrix.os }}
         path: e2e/mermaid/${{ matrix.os }}

CHANGELOG.md

@@ -7,6 +7,50 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.10.3] - 2026-02-06
+
+### Security
+
+- Fix an issue where blocklist bypass is possible when using curve P256 since the signature can have 2 valid representations.
+  Both fingerprint representations will be tested against the blocklist.
+  Any newly issued P256 based certificates will have their signature clamped to the low-s form.
+  Nebula will assert the low-s signature form when validating certificates in a future version. [GHSA-69x3-g4r3-p962](https://github.com/slackhq/nebula/security/advisories/GHSA-69x3-g4r3-p962)
+
+### Changed
+
+- Improve error reporting if nebula fails to start due to a tun device naming issue. (#1588)
+
+## [1.10.2] - 2026-01-21
+
+### Fixed
+
+- Fix panic when using `use_system_route_table` that was introduced in v1.10.1. (#1580)
+
+### Changed
+
+- Fix some typos in comments. (#1582)
+- Dependency updates. (#1581)
+
+## [1.10.1] - 2026-01-16
+
+See the [v1.10.1](https://github.com/slackhq/nebula/milestone/26?closed=1) milestone for a complete list of changes.
+
+### Fixed
+
+- Fix a bug where an unsafe route derived from the system route table could be lost on a config reload. (#1573)
+- Fix the PEM banner for ECDSA P256 public keys. (#1552)
+- Fix a regression on Windows from 1.9.x where nebula could fall back to a less performant UDP listener if 
+  non-critical ioctls failed. (#1568)
+- Fix a bug in handshake processing when a peer sends an unexpected public key. (#1566)
+
+### Added
+
+- Add a config option to control accepting `recv_error` packets which defaults to `always`. (#1569)
+
+### Changed
+
+- Various dependency updates. (#1541, #1549, #1550, #1557, #1558, #1560, #1561, #1570, #1571)
+
 ## [1.10.0] - 2025-12-04
 
 See the [v1.10.0](https://github.com/slackhq/nebula/milestone/16?closed=1) milestone for a complete list of changes.
@@ -744,7 +788,10 @@ created.)
 
 - Initial public release.
 
-[Unreleased]: https://github.com/slackhq/nebula/compare/v1.10.0...HEAD
+[Unreleased]: https://github.com/slackhq/nebula/compare/v1.10.3...HEAD
+[1.10.3]: https://github.com/slackhq/nebula/releases/tag/v1.10.3
+[1.10.2]: https://github.com/slackhq/nebula/releases/tag/v1.10.2
+[1.10.1]: https://github.com/slackhq/nebula/releases/tag/v1.10.1
 [1.10.0]: https://github.com/slackhq/nebula/releases/tag/v1.10.0
 [1.9.7]: https://github.com/slackhq/nebula/releases/tag/v1.9.7
 [1.9.6]: https://github.com/slackhq/nebula/releases/tag/v1.9.6

CODEOWNERS

@@ -0,0 +1 @@
+#ECCN:Open Source

README.md

@@ -57,7 +57,7 @@ Check the [releases](https://github.com/slackhq/nebula/releases/latest) page for
     docker pull nebulaoss/nebula
     ```
 
-#### Mobile
+#### Mobile ([source code](https://github.com/DefinedNet/mobile_nebula))
 
 - [iOS](https://apps.apple.com/us/app/mobile-nebula/id1509587936?itsct=apps_box&itscg=30200)
 - [Android](https://play.google.com/store/apps/details?id=net.defined.mobile_nebula&pcampaignid=pcampaignidMKT-Other-global-all-co-prtnr-py-PartBadge-Mar2515-1)
@@ -76,6 +76,8 @@ Nebula was created to provide a mechanism for groups of hosts to communicate sec
 
 ## Getting started (quickly)
 
+**Don't want to manage your own PKI and lighthouses?** [Managed Nebula](https://www.defined.net/) from Defined Networking handles all of this for you.
+
 To set up a Nebula network, you'll need:
 
 #### 1. The [Nebula binaries](https://github.com/slackhq/nebula/releases) or [Distribution Packages](https://github.com/slackhq/nebula#distribution-packages) for your specific platform. Specifically you'll need `nebula-cert` and the specific nebula binary for each platform you use.

boring.go

@@ -1,5 +1,4 @@
 //go:build boringcrypto
-// +build boringcrypto
 
 package nebula
 

cert/ca_pool.go

@@ -1,11 +1,14 @@
 package cert
 
 import (
+	"bufio"
+	"bytes"
+	"encoding/pem"
 	"errors"
 	"fmt"
+	"io"
 	"net/netip"
 	"slices"
-	"strings"
 	"time"
 )
 
@@ -29,22 +32,46 @@ func NewCAPool() *CAPool {
 // If the pool contains any expired certificates, an ErrExpired will be
 // returned along with the pool. The caller must handle any such errors.
 func NewCAPoolFromPEM(caPEMs []byte) (*CAPool, error) {
+	return NewCAPoolFromPEMReader(bytes.NewReader(caPEMs))
+}
+
+// NewCAPoolFromPEMReader will create a new CA pool from the provided reader.
+// The reader must contain a PEM-encoded set of nebula certificates.
+func NewCAPoolFromPEMReader(r io.Reader) (*CAPool, error) {
 	pool := NewCAPool()
-	var err error
+
 	var expired bool
-	for {
-		caPEMs, err = pool.AddCAFromPEM(caPEMs)
-		if errors.Is(err, ErrExpired) {
-			expired = true
-			err = nil
+
+	scanner := bufio.NewScanner(r)
+	scanner.Split(SplitPEM)
+
+	for scanner.Scan() {
+		pemBytes := scanner.Bytes()
+
+		block, rest := pem.Decode(pemBytes)
+		if len(bytes.TrimSpace(rest)) > 0 {
+			return nil, ErrInvalidPEMBlock
 		}
+		if block == nil {
+			return nil, ErrInvalidPEMBlock
+		}
+
+		c, err := unmarshalCertificateBlock(block)
 		if err != nil {
 			return nil, err
 		}
-		if len(caPEMs) == 0 || strings.TrimSpace(string(caPEMs)) == "" {
-			break
+
+		err = pool.AddCA(c)
+		if errors.Is(err, ErrExpired) {
+			expired = true
+			continue
+		} else if err != nil {
+			return nil, err
 		}
 	}
+	if err := scanner.Err(); err != nil {
+		return nil, ErrInvalidPEMBlock
+	}
 
 	if expired {
 		return pool, ErrExpired
@@ -141,10 +168,23 @@ func (ncp *CAPool) VerifyCertificate(now time.Time, c Certificate) (*CachedCerti
 		return nil, err
 	}
 
+	// Pre nebula v1.10.3 could generate signatures in either high or low s form and validation
+	// of signatures allowed for either. Nebula v1.10.3 and beyond clamps signature generation to low-s form
+	// but validation still allows for either. Since a change in the signature bytes affects the fingerprint, we
+	// need to test both forms until such a time comes that we enforce low-s form on signature validation.
+	fp2, err := CalculateAlternateFingerprint(c)
+	if err != nil {
+		return nil, fmt.Errorf("could not calculate alternate fingerprint to verify: %w", err)
+	}
+	if fp2 != "" && ncp.IsBlocklisted(fp2) {
+		return nil, ErrBlockListed
+	}
+
 	cc := CachedCertificate{
 		Certificate:       c,
 		InvertedGroups:    make(map[string]struct{}),
 		Fingerprint:       fp,
+		fingerprint2:      fp2,
 		signerFingerprint: signer.Fingerprint,
 	}
 
@@ -158,6 +198,11 @@ func (ncp *CAPool) VerifyCertificate(now time.Time, c Certificate) (*CachedCerti
 // VerifyCachedCertificate is the same as VerifyCertificate other than it operates on a pre-verified structure and
 // is a cheaper operation to perform as a result.
 func (ncp *CAPool) VerifyCachedCertificate(now time.Time, c *CachedCertificate) error {
+	// Check any available alternate fingerprint forms for this certificate, re P256 high-s/low-s
+	if c.fingerprint2 != "" && ncp.IsBlocklisted(c.fingerprint2) {
+		return ErrBlockListed
+	}
+
 	_, err := ncp.verify(c.Certificate, now, c.Fingerprint, c.signerFingerprint)
 	return err
 }

cert/ca_pool_test.go

@@ -1,10 +1,14 @@
 package cert
 
 import (
+	"bytes"
+	"io"
 	"net/netip"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/slackhq/nebula/cert/p256"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -111,6 +115,60 @@ k+coOv04r+zh33ISyhbsafnYduN17p2eD7CmHvHuerguXD9f32gcxo/KsFCKEjMe
 	assert.Len(t, ppppp.CAs, 1)
 }
 
+// oneByteReader wraps a reader to return at most 1 byte per Read call,
+// exercising the streaming accumulation logic in NewCAPoolFromPEMReader.
+type oneByteReader struct {
+	r io.Reader
+}
+
+func (o *oneByteReader) Read(p []byte) (int, error) {
+	if len(p) == 0 {
+		return 0, nil
+	}
+	return o.r.Read(p[:1])
+}
+
+func TestNewCAPoolFromPEMReader_EmptyReader(t *testing.T) {
+	pool, err := NewCAPoolFromPEMReader(bytes.NewReader(nil))
+	require.NoError(t, err)
+	assert.Empty(t, pool.CAs)
+
+	pool, err = NewCAPoolFromPEMReader(strings.NewReader("   \n\t\n  "))
+	require.NoError(t, err)
+	assert.Empty(t, pool.CAs)
+}
+
+func TestNewCAPoolFromPEMReader_OneByteReads(t *testing.T) {
+	ca1, _, _, pem1 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil)
+	ca2, _, _, pem2 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil)
+
+	bundle := append(pem1, pem2...)
+	pool, err := NewCAPoolFromPEMReader(&oneByteReader{r: bytes.NewReader(bundle)})
+	require.NoError(t, err)
+	assert.Len(t, pool.CAs, 2)
+
+	fp1, err := ca1.Fingerprint()
+	require.NoError(t, err)
+	fp2, err := ca2.Fingerprint()
+	require.NoError(t, err)
+
+	assert.Contains(t, pool.CAs, fp1)
+	assert.Contains(t, pool.CAs, fp2)
+}
+
+func TestNewCAPoolFromPEMReader_TruncatedPEM(t *testing.T) {
+	_, err := NewCAPoolFromPEMReader(strings.NewReader("-----BEGIN NEBULA CERTIFICATE-----\npartialdata"))
+	assert.ErrorIs(t, err, ErrInvalidPEMBlock)
+}
+
+func TestNewCAPoolFromPEMReader_TrailingGarbage(t *testing.T) {
+	_, _, _, pem1 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil)
+
+	bundle := append(pem1, []byte("some trailing garbage")...)
+	_, err := NewCAPoolFromPEMReader(bytes.NewReader(bundle))
+	assert.ErrorIs(t, err, ErrInvalidPEMBlock)
+}
+
 func TestCertificateV1_Verify(t *testing.T) {
 	ca, _, caKey, _ := NewTestCaCert(Version1, Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, nil)
 	c, _, _, _ := NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test cert", time.Now(), time.Now().Add(5*time.Minute), nil, nil, nil)
@@ -170,6 +228,15 @@ func TestCertificateV1_VerifyP256(t *testing.T) {
 	_, err = caPool.VerifyCertificate(time.Now(), c)
 	require.EqualError(t, err, "certificate is in the block list")
 
+	// Create a copy of the cert and swap to the alternate form for the signature
+	nc := c.Copy()
+	b, err := p256.Swap(c.Signature())
+	require.NoError(t, err)
+	require.NoError(t, nc.(*certificateV1).setSignature(b))
+
+	_, err = caPool.VerifyCertificate(time.Now(), nc)
+	require.EqualError(t, err, "certificate is in the block list")
+
 	caPool.ResetCertBlocklist()
 	_, err = caPool.VerifyCertificate(time.Now(), c)
 	require.NoError(t, err)
@@ -187,7 +254,7 @@ func TestCertificateV1_VerifyP256(t *testing.T) {
 	require.NoError(t, err)
 
 	caPool = NewCAPool()
-	b, err := caPool.AddCAFromPEM(caPem)
+	b, err = caPool.AddCAFromPEM(caPem)
 	require.NoError(t, err)
 	assert.Empty(t, b)
 
@@ -196,7 +263,17 @@ func TestCertificateV1_VerifyP256(t *testing.T) {
 	})
 
 	c, _, _, _ = NewTestCert(Version1, Curve_P256, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, nil, []string{"test1"})
-	_, err = caPool.VerifyCertificate(time.Now(), c)
+	cc, err := caPool.VerifyCertificate(time.Now(), c)
+	require.NoError(t, err)
+
+	// Reset the blocklist and block the alternate form fingerprint
+	caPool.ResetCertBlocklist()
+	caPool.BlocklistFingerprint(cc.fingerprint2)
+	err = caPool.VerifyCachedCertificate(time.Now(), cc)
+	require.EqualError(t, err, "certificate is in the block list")
+
+	caPool.ResetCertBlocklist()
+	err = caPool.VerifyCachedCertificate(time.Now(), cc)
 	require.NoError(t, err)
 }
 
@@ -394,6 +471,15 @@ func TestCertificateV2_VerifyP256(t *testing.T) {
 	_, err = caPool.VerifyCertificate(time.Now(), c)
 	require.EqualError(t, err, "certificate is in the block list")
 
+	// Create a copy of the cert and swap to the alternate form for the signature
+	nc := c.Copy()
+	b, err := p256.Swap(c.Signature())
+	require.NoError(t, err)
+	require.NoError(t, nc.(*certificateV2).setSignature(b))
+
+	_, err = caPool.VerifyCertificate(time.Now(), nc)
+	require.EqualError(t, err, "certificate is in the block list")
+
 	caPool.ResetCertBlocklist()
 	_, err = caPool.VerifyCertificate(time.Now(), c)
 	require.NoError(t, err)
@@ -411,7 +497,7 @@ func TestCertificateV2_VerifyP256(t *testing.T) {
 	require.NoError(t, err)
 
 	caPool = NewCAPool()
-	b, err := caPool.AddCAFromPEM(caPem)
+	b, err = caPool.AddCAFromPEM(caPem)
 	require.NoError(t, err)
 	assert.Empty(t, b)
 
@@ -420,7 +506,17 @@ func TestCertificateV2_VerifyP256(t *testing.T) {
 	})
 
 	c, _, _, _ = NewTestCert(Version2, Curve_P256, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, nil, []string{"test1"})
-	_, err = caPool.VerifyCertificate(time.Now(), c)
+	cc, err := caPool.VerifyCertificate(time.Now(), c)
+	require.NoError(t, err)
+
+	// Reset the blocklist and block the alternate form fingerprint
+	caPool.ResetCertBlocklist()
+	caPool.BlocklistFingerprint(cc.fingerprint2)
+	err = caPool.VerifyCachedCertificate(time.Now(), cc)
+	require.EqualError(t, err, "certificate is in the block list")
+
+	caPool.ResetCertBlocklist()
+	err = caPool.VerifyCachedCertificate(time.Now(), cc)
 	require.NoError(t, err)
 }
 

cert/cert.go

@@ -4,6 +4,8 @@ import (
 	"fmt"
 	"net/netip"
 	"time"
+
+	"github.com/slackhq/nebula/cert/p256"
 )
 
 type Version uint8
@@ -110,6 +112,9 @@ type CachedCertificate struct {
 	InvertedGroups    map[string]struct{}
 	Fingerprint       string
 	signerFingerprint string
+
+	// A place to store a 2nd fingerprint if the certificate could have one, such as with P256
+	fingerprint2 string
 }
 
 func (cc *CachedCertificate) String() string {
@@ -119,6 +124,7 @@ func (cc *CachedCertificate) String() string {
 // Recombine will attempt to unmarshal a certificate received in a handshake.
 // Handshakes save space by placing the peers public key in a different part of the packet, we have to
 // reassemble the actual certificate structure with that in mind.
+// Implementations MUST assert the public key is not in the raw certificate bytes if the passed in public key is not empty.
 func Recombine(v Version, rawCertBytes, publicKey []byte, curve Curve) (Certificate, error) {
 	if publicKey == nil {
 		return nil, ErrNoPeerStaticKey
@@ -151,3 +157,31 @@ func Recombine(v Version, rawCertBytes, publicKey []byte, curve Curve) (Certific
 
 	return c, nil
 }
+
+// CalculateAlternateFingerprint calculates a 2nd fingerprint representation for P256 certificates
+// CAPool blocklist testing through `VerifyCertificate` and `VerifyCachedCertificate` automatically performs this step.
+func CalculateAlternateFingerprint(c Certificate) (string, error) {
+	if c.Curve() != Curve_P256 {
+		return "", nil
+	}
+
+	nc := c.Copy()
+	b, err := p256.Swap(nc.Signature())
+	if err != nil {
+		return "", err
+	}
+
+	switch v := nc.(type) {
+	case *certificateV1:
+		err = v.setSignature(b)
+	case *certificateV2:
+		err = v.setSignature(b)
+	default:
+		return "", ErrUnknownVersion
+	}
+
+	if err != nil {
+		return "", err
+	}
+	return nc.Fingerprint()
+}

cert/cert_v1.go

@@ -426,7 +426,7 @@ func unmarshalCertificateV1(b []byte, publicKey []byte) (*certificateV1, error)
 			unsafeNetworks: make([]netip.Prefix, len(rc.Details.Subnets)/2),
 			notBefore:      time.Unix(rc.Details.NotBefore, 0),
 			notAfter:       time.Unix(rc.Details.NotAfter, 0),
-			publicKey:      make([]byte, len(rc.Details.PublicKey)),
+			publicKey:      nil,
 			isCA:           rc.Details.IsCA,
 			curve:          rc.Details.Curve,
 		},
@@ -437,12 +437,19 @@ func unmarshalCertificateV1(b []byte, publicKey []byte) (*certificateV1, error)
 	copy(nc.details.groups, rc.Details.Groups)
 	nc.details.issuer = hex.EncodeToString(rc.Details.Issuer)
 
+	// If a public key is passed in as an argument, the certificate pubkey must be empty
+	// and the passed-in pubkey copied into the cert.
 	if len(publicKey) > 0 {
-		nc.details.publicKey = publicKey
+		if len(rc.Details.PublicKey) != 0 {
+			return nil, ErrCertPubkeyPresent
+		}
+		nc.details.publicKey = make([]byte, len(publicKey))
+		copy(nc.details.publicKey, publicKey)
+	} else {
+		nc.details.publicKey = make([]byte, len(rc.Details.PublicKey))
+		copy(nc.details.publicKey, rc.Details.PublicKey)
 	}
 
-	copy(nc.details.publicKey, rc.Details.PublicKey)
-
 	var ip netip.Addr
 	for i, rawIp := range rc.Details.Ips {
 		if i%2 == 0 {

cert/cert_v1_test.go

@@ -62,6 +62,62 @@ func TestCertificateV1_Marshal(t *testing.T) {
 	assert.Equal(t, nc.Groups(), nc2.Groups())
 }
 
+func TestCertificateV1_Unmarshal(t *testing.T) {
+	t.Parallel()
+	before := time.Now().Add(time.Second * -60).Round(time.Second)
+	after := time.Now().Add(time.Second * 60).Round(time.Second)
+	pubKey := []byte("1234567890abcedfghij1234567890ab")
+	invalidPubkey := []byte("00000000000000000000000000000000")
+
+	nc := certificateV1{
+		details: detailsV1{
+			name: "testing",
+			networks: []netip.Prefix{
+				mustParsePrefixUnmapped("10.1.1.1/24"),
+				mustParsePrefixUnmapped("10.1.1.2/16"),
+			},
+			unsafeNetworks: []netip.Prefix{
+				mustParsePrefixUnmapped("9.1.1.2/24"),
+				mustParsePrefixUnmapped("9.1.1.3/16"),
+			},
+			groups:    []string{"test-group1", "test-group2", "test-group3"},
+			notBefore: before,
+			notAfter:  after,
+			publicKey: pubKey,
+			isCA:      false,
+			issuer:    "1234567890abcedfghij1234567890ab",
+		},
+		signature: []byte("1234567890abcedfghij1234567890ab"),
+	}
+
+	// This certificate has a pubkey included
+	certWithPubkey, err := nc.Marshal()
+	require.NoError(t, err)
+
+	// This certificate is missing the pubkey section
+	certWithoutPubkey, err := nc.MarshalForHandshakes()
+	require.NoError(t, err)
+
+	// Cert has no pubkey and no pubkey passed in must fail to validate
+	isNil, err := unmarshalCertificateV1(certWithoutPubkey, nil)
+	require.Error(t, err)
+
+	// Cert has different pubkey than one passed in must fail
+	isNil, err = unmarshalCertificateV1(certWithPubkey, invalidPubkey)
+	require.Nil(t, isNil)
+	require.Error(t, err)
+
+	// Cert has pubkey and no pubkey argument works ok
+	_, err = unmarshalCertificateV1(certWithPubkey, nil)
+	require.NoError(t, err)
+
+	// Cert has no pubkey and valid, correctly signed pubkey passed in
+	nc2, err := unmarshalCertificateV1(certWithoutPubkey, pubKey)
+	require.NoError(t, err)
+
+	assert.Equal(t, pubKey, nc2.PublicKey())
+}
+
 func TestCertificateV1_PublicKeyPem(t *testing.T) {
 	t.Parallel()
 	before := time.Now().Add(time.Second * -60).Round(time.Second)

cert/cert_v2.go

@@ -592,7 +592,13 @@ func unmarshalCertificateV2(b []byte, publicKey []byte, curve Curve) (*certifica
 	// Maybe grab the public key
 	var rawPublicKey cryptobyte.String
 	if len(publicKey) > 0 {
-		rawPublicKey = publicKey
+		// If a public key is passed in, then the handshake certificate must
+		// not have a public key present
+		if input.PeekASN1Tag(TagCertPublicKey) {
+			return nil, ErrCertPubkeyPresent
+		}
+		rawPublicKey = make(cryptobyte.String, len(publicKey))
+		copy(rawPublicKey, publicKey)
 	} else if !input.ReadOptionalASN1(&rawPublicKey, nil, TagCertPublicKey) {
 		return nil, ErrBadFormat
 	}

cert/cert_v2_test.go

@@ -76,6 +76,58 @@ func TestCertificateV2_Marshal(t *testing.T) {
 	assert.Equal(t, nc.Groups(), nc2.Groups())
 }
 
+func TestCertificateV2_Unmarshal(t *testing.T) {
+	t.Parallel()
+	before := time.Now().Add(time.Second * -60).Round(time.Second)
+	after := time.Now().Add(time.Second * 60).Round(time.Second)
+	pubKey := []byte("1234567890abcedfghij1234567890ab")
+
+	nc := certificateV2{
+		details: detailsV2{
+			name: "testing",
+			networks: []netip.Prefix{
+				mustParsePrefixUnmapped("10.1.1.2/16"),
+				mustParsePrefixUnmapped("10.1.1.1/24"),
+			},
+			unsafeNetworks: []netip.Prefix{
+				mustParsePrefixUnmapped("9.1.1.3/16"),
+				mustParsePrefixUnmapped("9.1.1.2/24"),
+			},
+			groups:    []string{"test-group1", "test-group2", "test-group3"},
+			notBefore: before,
+			notAfter:  after,
+			isCA:      false,
+			issuer:    "1234567890abcdef1234567890abcdef",
+		},
+		signature: []byte("1234567890abcdef1234567890abcdef"),
+		publicKey: pubKey,
+	}
+
+	db, err := nc.details.Marshal()
+	require.NoError(t, err)
+	nc.rawDetails = db
+
+	certWithPubkey, err := nc.Marshal()
+	require.NoError(t, err)
+	//t.Log("Cert size:", len(b))
+	certWithoutPubkey, err := nc.MarshalForHandshakes()
+	require.NoError(t, err)
+
+	// Cert must not have a pubkey if one is passed in as an argument
+	_, err = unmarshalCertificateV2(certWithPubkey, pubKey, Curve_CURVE25519)
+	require.ErrorIs(t, err, ErrCertPubkeyPresent)
+
+	// Certs must have pubkeys
+	_, err = unmarshalCertificateV2(certWithoutPubkey, nil, Curve_CURVE25519)
+	require.ErrorIs(t, err, ErrBadFormat)
+
+	// Ensure proper unmarshal if a pubkey is passed in
+	nc2, err := unmarshalCertificateV2(certWithoutPubkey, pubKey, Curve_CURVE25519)
+	require.NoError(t, err)
+
+	assert.Equal(t, nc.PublicKey(), nc2.PublicKey())
+}
+
 func TestCertificateV2_PublicKeyPem(t *testing.T) {
 	t.Parallel()
 	before := time.Now().Add(time.Second * -60).Round(time.Second)

cert/crypto_test.go

@@ -79,7 +79,7 @@ qrlJ69wer3ZUHFXA
 	assert.Nil(t, k)
 	assert.Equal(t, rest, invalidPem)
 
-	// Fail due to ivalid PEM format, because
+	// Fail due to invalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	curve, k, rest, err = DecryptAndUnmarshalSigningPrivateKey(passphrase, rest)
 	require.EqualError(t, err, "input did not contain a valid PEM encoded block")

cert/errors.go

@@ -21,6 +21,7 @@ var (
 	ErrPrivateKeyEncrypted        = errors.New("private key must be decrypted")
 	ErrCaNotFound                 = errors.New("could not find ca for the certificate")
 	ErrUnknownVersion             = errors.New("certificate version unrecognized")
+	ErrCertPubkeyPresent          = errors.New("certificate has unexpected pubkey present")
 
 	ErrInvalidPEMBlock                   = errors.New("input did not contain a valid PEM encoded block")
 	ErrInvalidPEMCertificateBanner       = errors.New("bytes did not contain a proper certificate banner")

cert/p256/p256.go

@@ -0,0 +1,127 @@
+package p256
+
+import (
+	"crypto/elliptic"
+	"errors"
+	"math/big"
+
+	"filippo.io/bigmod"
+
+	"golang.org/x/crypto/cryptobyte"
+	"golang.org/x/crypto/cryptobyte/asn1"
+)
+
+var halfN = new(big.Int).Rsh(elliptic.P256().Params().N, 1)
+var nMod *bigmod.Modulus
+
+func init() {
+	n, err := bigmod.NewModulus(elliptic.P256().Params().N.Bytes())
+	if err != nil {
+		panic(err)
+	}
+	nMod = n
+}
+
+func IsNormalized(sig []byte) (bool, error) {
+	r, s, err := parseSignature(sig)
+	if err != nil {
+		return false, err
+	}
+	return checkLowS(r, s), nil
+}
+
+func checkLowS(_, s []byte) bool {
+	bigS := new(big.Int).SetBytes(s)
+	// Check if S <= (N/2), because we want to include the midpoint in the set of low-s
+	return bigS.Cmp(halfN) <= 0
+}
+
+func swap(r, s []byte) ([]byte, []byte, error) {
+	var err error
+	bigS, err := bigmod.NewNat().SetBytes(s, nMod)
+	if err != nil {
+		return nil, nil, err
+	}
+	sNormalized := nMod.Nat().Sub(bigS, nMod)
+
+	result := sNormalized.Bytes(nMod)
+	for len(result) > 1 && result[0] == 0 {
+		result = result[1:]
+	}
+
+	return r, result, nil
+}
+
+func Normalize(sig []byte) ([]byte, error) {
+	r, s, err := parseSignature(sig)
+	if err != nil {
+		return nil, err
+	}
+
+	if checkLowS(r, s) {
+		return sig, nil
+	}
+
+	newR, newS, err := swap(r, s)
+	if err != nil {
+		return nil, err
+	}
+
+	return encodeSignature(newR, newS)
+}
+
+// Swap will change sig between its current form to the opposite high or low form.
+func Swap(sig []byte) ([]byte, error) {
+	r, s, err := parseSignature(sig)
+	if err != nil {
+		return nil, err
+	}
+
+	newR, newS, err := swap(r, s)
+	if err != nil {
+		return nil, err
+	}
+
+	return encodeSignature(newR, newS)
+}
+
+// parseSignature taken exactly from crypto/ecdsa/ecdsa.go
+func parseSignature(sig []byte) (r, s []byte, err error) {
+	var inner cryptobyte.String
+	input := cryptobyte.String(sig)
+	if !input.ReadASN1(&inner, asn1.SEQUENCE) ||
+		!input.Empty() ||
+		!inner.ReadASN1Integer(&r) ||
+		!inner.ReadASN1Integer(&s) ||
+		!inner.Empty() {
+		return nil, nil, errors.New("invalid ASN.1")
+	}
+	return r, s, nil
+}
+
+func encodeSignature(r, s []byte) ([]byte, error) {
+	var b cryptobyte.Builder
+	b.AddASN1(asn1.SEQUENCE, func(b *cryptobyte.Builder) {
+		addASN1IntBytes(b, r)
+		addASN1IntBytes(b, s)
+	})
+	return b.Bytes()
+}
+
+// addASN1IntBytes encodes in ASN.1 a positive integer represented as
+// a big-endian byte slice with zero or more leading zeroes.
+func addASN1IntBytes(b *cryptobyte.Builder, bytes []byte) {
+	for len(bytes) > 0 && bytes[0] == 0 {
+		bytes = bytes[1:]
+	}
+	if len(bytes) == 0 {
+		b.SetError(errors.New("invalid integer"))
+		return
+	}
+	b.AddASN1(asn1.INTEGER, func(c *cryptobyte.Builder) {
+		if bytes[0]&0x80 != 0 {
+			c.AddUint8(0)
+		}
+		c.AddBytes(bytes)
+	})
+}

cert/p256/p256_test.go

@@ -0,0 +1,28 @@
+package p256
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFlipping(t *testing.T) {
+	priv, err1 := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err1)
+
+	out, err := ecdsa.SignASN1(rand.Reader, priv, []byte("big chungus"))
+	require.NoError(t, err)
+
+	r, s, err := parseSignature(out)
+	require.NoError(t, err)
+
+	r, s1, err := swap(r, s)
+	require.NoError(t, err)
+	r, s2, err := swap(r, s1)
+	require.NoError(t, err)
+	require.Equal(t, s, s2)
+	require.NotEqual(t, s, s1)
+}

cert/pem.go

@@ -1,12 +1,66 @@
 package cert
 
 import (
+	"bytes"
 	"encoding/pem"
+	"errors"
 	"fmt"
 
 	"golang.org/x/crypto/ed25519"
 )
 
+var ErrTruncatedPEMBlock = errors.New("truncated PEM block")
+
+// SplitPEM is a split function for bufio.Scanner that returns each PEM block.
+func SplitPEM(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	// Look for the start of a PEM block
+	start := bytes.Index(data, []byte("-----BEGIN "))
+	if start == -1 {
+		if atEOF && len(bytes.TrimSpace(data)) > 0 {
+			// Non-whitespace content with no PEM block
+			return 0, nil, ErrTruncatedPEMBlock
+		}
+		if atEOF {
+			return len(data), nil, nil
+		}
+		// Request more data
+		return 0, nil, nil
+	}
+
+	// Look for the end marker
+	endMarkerStart := bytes.Index(data[start:], []byte("-----END "))
+	if endMarkerStart == -1 {
+		if atEOF {
+			// Incomplete PEM block at EOF
+			return 0, nil, ErrTruncatedPEMBlock
+		}
+		// Need more data to find the end
+		return 0, nil, nil
+	}
+
+	// Find the actual end of the END line (after the newline)
+	endMarkerStart += start
+	endLineEnd := bytes.IndexByte(data[endMarkerStart:], '\n')
+	var end int
+	if endLineEnd == -1 {
+		if atEOF {
+			// END marker without newline at EOF - take it anyway
+			end = len(data)
+		} else {
+			// Need more data
+			return 0, nil, nil
+		}
+	} else {
+		end = endMarkerStart + endLineEnd + 1
+	}
+
+	// Extract the PEM block
+	pemBlock := data[start:end]
+
+	// Return the valid PEM block
+	return end, pemBlock, nil
+}
+
 const ( //cert banners
 	CertificateBanner   = "NEBULA CERTIFICATE"
 	CertificateV2Banner = "NEBULA CERTIFICATE V2"
@@ -37,19 +91,7 @@ func UnmarshalCertificateFromPEM(b []byte) (Certificate, []byte, error) {
 		return nil, r, ErrInvalidPEMBlock
 	}
 
-	var c Certificate
-	var err error
-
-	switch p.Type {
-	// Implementations must validate the resulting certificate contains valid information
-	case CertificateBanner:
-		c, err = unmarshalCertificateV1(p.Bytes, nil)
-	case CertificateV2Banner:
-		c, err = unmarshalCertificateV2(p.Bytes, nil, Curve_CURVE25519)
-	default:
-		return nil, r, ErrInvalidPEMCertificateBanner
-	}
-
+	c, err := unmarshalCertificateBlock(p)
 	if err != nil {
 		return nil, r, err
 	}
@@ -58,6 +100,20 @@ func UnmarshalCertificateFromPEM(b []byte) (Certificate, []byte, error) {
 
 }
 
+// unmarshalCertificateBlock decodes a single PEM block into a certificate.
+// It expects a Nebula certificate banner and returns ErrInvalidPEMCertificateBanner otherwise.
+func unmarshalCertificateBlock(block *pem.Block) (Certificate, error) {
+	switch block.Type {
+	// Implementations must validate the resulting certificate contains valid information
+	case CertificateBanner:
+		return unmarshalCertificateV1(block.Bytes, nil)
+	case CertificateV2Banner:
+		return unmarshalCertificateV2(block.Bytes, nil, Curve_CURVE25519)
+	default:
+		return nil, ErrInvalidPEMCertificateBanner
+	}
+}
+
 func marshalCertPublicKeyToPEM(c Certificate) []byte {
 	if c.IsCA() {
 		return MarshalSigningPublicKeyToPEM(c.Curve(), c.PublicKey())

cert/pem_test.go

@@ -1,12 +1,88 @@
 package cert
 
 import (
+	"bufio"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+func scanAll(t *testing.T, input string) ([]string, error) {
+	t.Helper()
+	scanner := bufio.NewScanner(strings.NewReader(input))
+	scanner.Split(SplitPEM)
+	var blocks []string
+	for scanner.Scan() {
+		blocks = append(blocks, scanner.Text())
+	}
+	return blocks, scanner.Err()
+}
+
+func TestSplitPEM_Single(t *testing.T) {
+	input := "-----BEGIN TEST-----\ndata\n-----END TEST-----\n"
+	blocks, err := scanAll(t, input)
+	require.NoError(t, err)
+	require.Len(t, blocks, 1)
+	require.Equal(t, input, blocks[0])
+}
+
+func TestSplitPEM_Multiple(t *testing.T) {
+	block1 := "-----BEGIN TEST-----\naaa\n-----END TEST-----\n"
+	block2 := "-----BEGIN TEST-----\nbbb\n-----END TEST-----\n"
+	blocks, err := scanAll(t, block1+block2)
+	require.NoError(t, err)
+	require.Len(t, blocks, 2)
+	require.Equal(t, block1, blocks[0])
+	require.Equal(t, block2, blocks[1])
+}
+
+func TestSplitPEM_CommentsAndWhitespaceBetweenBlocks(t *testing.T) {
+	input := "# comment\n\n-----BEGIN TEST-----\naaa\n-----END TEST-----\n\n# another comment\n\n-----BEGIN TEST-----\nbbb\n-----END TEST-----\n"
+	blocks, err := scanAll(t, input)
+	require.NoError(t, err)
+	require.Len(t, blocks, 2)
+}
+
+func TestSplitPEM_Empty(t *testing.T) {
+	blocks, err := scanAll(t, "")
+	require.NoError(t, err)
+	require.Empty(t, blocks)
+}
+
+func TestSplitPEM_WhitespaceOnly(t *testing.T) {
+	blocks, err := scanAll(t, "  \n\t\n  ")
+	require.NoError(t, err)
+	require.Empty(t, blocks)
+}
+
+func TestSplitPEM_TrailingGarbage(t *testing.T) {
+	input := "-----BEGIN TEST-----\ndata\n-----END TEST-----\ngarbage"
+	blocks, err := scanAll(t, input)
+	require.ErrorIs(t, err, ErrTruncatedPEMBlock)
+	require.Len(t, blocks, 1)
+}
+
+func TestSplitPEM_TruncatedBlock(t *testing.T) {
+	input := "-----BEGIN TEST-----\npartial data with no end"
+	_, err := scanAll(t, input)
+	require.ErrorIs(t, err, ErrTruncatedPEMBlock)
+}
+
+func TestSplitPEM_NoEndNewline(t *testing.T) {
+	input := "-----BEGIN TEST-----\ndata\n-----END TEST-----"
+	blocks, err := scanAll(t, input)
+	require.NoError(t, err)
+	require.Len(t, blocks, 1)
+	require.Equal(t, input, blocks[0])
+}
+
+func TestSplitPEM_GarbageOnly(t *testing.T) {
+	_, err := scanAll(t, "this is not PEM data")
+	require.ErrorIs(t, err, ErrTruncatedPEMBlock)
+}
+
 func TestUnmarshalCertificateFromPEM(t *testing.T) {
 	goodCert := []byte(`
 # A good cert
@@ -44,7 +120,7 @@ bzBEr00kERQxxTzTsH8cpYEgRoipvmExvg8WP8NdAJEYJosB
 	assert.Equal(t, rest, invalidPem)
 	require.EqualError(t, err, "bytes did not contain a proper certificate banner")
 
-	// Fail due to ivalid PEM format, because
+	// Fail due to invalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	cert, rest, err = UnmarshalCertificateFromPEM(rest)
 	assert.Nil(t, cert)
@@ -106,7 +182,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
 	assert.Equal(t, rest, invalidPem)
 	require.EqualError(t, err, "bytes did not contain a proper Ed25519/ECDSA private key banner")
 
-	// Fail due to ivalid PEM format, because
+	// Fail due to invalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	k, rest, curve, err = UnmarshalSigningPrivateKeyFromPEM(rest)
 	assert.Nil(t, k)
@@ -168,7 +244,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 	assert.Equal(t, rest, invalidPem)
 	require.EqualError(t, err, "bytes did not contain a proper private key banner")
 
-	// Fail due to ivalid PEM format, because
+	// Fail due to invalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	k, rest, curve, err = UnmarshalPrivateKeyFromPEM(rest)
 	assert.Nil(t, k)
@@ -221,7 +297,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 	require.EqualError(t, err, "bytes did not contain a proper public key banner")
 	assert.Equal(t, rest, invalidPem)
 
-	// Fail due to ivalid PEM format, because
+	// Fail due to invalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
 	assert.Nil(t, k)
@@ -299,7 +375,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 	require.EqualError(t, err, "bytes did not contain a proper public key banner")
 	assert.Equal(t, rest, invalidPem)
 
-	// Fail due to ivalid PEM format, because
+	// Fail due to invalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
 	assert.Nil(t, k)

cert/sign.go

@@ -9,6 +9,8 @@ import (
 	"fmt"
 	"net/netip"
 	"time"
+
+	"github.com/slackhq/nebula/cert/p256"
 )
 
 // TBSCertificate represents a certificate intended to be signed.
@@ -126,6 +128,13 @@ func (t *TBSCertificate) SignWith(signer Certificate, curve Curve, sp SignerLamb
 		return nil, err
 	}
 
+	if curve == Curve_P256 {
+		sig, err = p256.Normalize(sig)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	err = c.setSignature(sig)
 	if err != nil {
 		return nil, err

cert/sign_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/slackhq/nebula/cert/p256"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -89,3 +90,48 @@ func TestCertificateV1_SignP256(t *testing.T) {
 	require.NoError(t, err)
 	assert.NotNil(t, uc)
 }
+
+func TestCertificate_SignP256_AlwaysNormalized(t *testing.T) {
+	before := time.Now().Add(time.Second * -60).Round(time.Second)
+	after := time.Now().Add(time.Second * 60).Round(time.Second)
+	pubKey := []byte("01234567890abcedfghij1234567890ab1234567890abcedfghij1234567890ab")
+
+	tbs := TBSCertificate{
+		Version: Version1,
+		Name:    "testing",
+		Networks: []netip.Prefix{
+			mustParsePrefixUnmapped("10.1.1.1/24"),
+			mustParsePrefixUnmapped("10.1.1.2/16"),
+		},
+		UnsafeNetworks: []netip.Prefix{
+			mustParsePrefixUnmapped("9.1.1.2/24"),
+			mustParsePrefixUnmapped("9.1.1.3/16"),
+		},
+		Groups:    []string{"test-group1", "test-group2", "test-group3"},
+		NotBefore: before,
+		NotAfter:  after,
+		PublicKey: pubKey,
+		IsCA:      true,
+		Curve:     Curve_P256,
+	}
+
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	require.NoError(t, err)
+	pub := elliptic.Marshal(elliptic.P256(), priv.PublicKey.X, priv.PublicKey.Y)
+	rawPriv := priv.D.FillBytes(make([]byte, 32))
+
+	for i := 0; i < 1000; i++ {
+		if i&1 == 1 {
+			tbs.Version = Version1
+		} else {
+			tbs.Version = Version2
+		}
+		c, err := tbs.Sign(nil, Curve_P256, rawPriv)
+		require.NoError(t, err)
+		assert.NotNil(t, c)
+		assert.True(t, c.CheckSignature(pub))
+		normie, err := p256.IsNormalized(c.Signature())
+		require.NoError(t, err)
+		assert.True(t, normie)
+	}
+}

cmd/nebula-cert/ca_test.go

@@ -200,7 +200,7 @@ func Test_ca(t *testing.T) {
 	assert.Empty(t, b)
 	assert.Len(t, lKey, 64)
 
-	// test when reading passsword results in an error
+	// test when reading password results in an error
 	os.Remove(keyF.Name())
 	os.Remove(crtF.Name())
 	ob.Reset()

cmd/nebula-cert/verify.go

@@ -6,7 +6,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/slackhq/nebula/cert"
@@ -40,21 +39,15 @@ func verify(args []string, out io.Writer, errOut io.Writer) error {
 		return err
 	}
 
-	rawCACert, err := os.ReadFile(*vf.caPath)
+	caFile, err := os.Open(*vf.caPath)
 	if err != nil {
 		return fmt.Errorf("error while reading ca: %w", err)
 	}
+	defer caFile.Close()
 
-	caPool := cert.NewCAPool()
-	for {
-		rawCACert, err = caPool.AddCAFromPEM(rawCACert)
-		if err != nil {
-			return fmt.Errorf("error while adding ca cert to pool: %w", err)
-		}
-
-		if rawCACert == nil || len(rawCACert) == 0 || strings.TrimSpace(string(rawCACert)) == "" {
-			break
-		}
+	caPool, err := cert.NewCAPoolFromPEMReader(caFile)
+	if err != nil && !errors.Is(err, cert.ErrExpired) {
+		return fmt.Errorf("error while adding ca cert to pool: %w", err)
 	}
 
 	rawCert, err := os.ReadFile(*vf.certPath)

cmd/nebula-cert/verify_test.go

@@ -64,7 +64,7 @@ func Test_verify(t *testing.T) {
 	err = verify([]string{"-ca", caFile.Name(), "-crt", "does_not_exist"}, ob, eb)
 	assert.Empty(t, ob.String())
 	assert.Empty(t, eb.String())
-	require.EqualError(t, err, "error while adding ca cert to pool: input did not contain a valid PEM encoded block")
+	require.ErrorIs(t, err, cert.ErrInvalidPEMBlock)
 
 	// make a ca for later
 	caPub, caPriv, _ := ed25519.GenerateKey(rand.Reader)

cmd/nebula-service/main.go

@@ -50,9 +50,15 @@ func main() {
 		os.Exit(0)
 	}
 
+	l := logrus.New()
+	l.Out = os.Stdout
+
 	if *serviceFlag != "" {
-		doService(configPath, configTest, Build, serviceFlag)
-		os.Exit(1)
+		if err := doService(configPath, configTest, Build, serviceFlag); err != nil {
+			l.WithError(err).Error("Service command failed")
+			os.Exit(1)
+		}
+		return
 	}
 
 	if *configPath == "" {
@@ -61,9 +67,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	l := logrus.New()
-	l.Out = os.Stdout
-
 	c := config.NewC(l)
 	err := c.Load(*configPath)
 	if err != nil {
@@ -78,8 +81,20 @@ func main() {
 	}
 
 	if !*configTest {
-		ctrl.Start()
-		ctrl.ShutdownBlock()
+		wait, err := ctrl.Start()
+		if err != nil {
+			util.LogWithContextIfNeeded("Error while running", err, l)
+			os.Exit(1)
+		}
+
+		go ctrl.ShutdownBlock()
+
+		if err := wait(); err != nil {
+			l.WithError(err).Error("Nebula stopped due to fatal error")
+			os.Exit(2)
+		}
+
+		l.Info("Goodbye")
 	}
 
 	os.Exit(0)

cmd/nebula-service/service.go

@@ -57,11 +57,11 @@ func fileExists(filename string) bool {
 	return true
 }
 
-func doService(configPath *string, configTest *bool, build string, serviceFlag *string) {
+func doService(configPath *string, configTest *bool, build string, serviceFlag *string) error {
 	if *configPath == "" {
 		ex, err := os.Executable()
 		if err != nil {
-			panic(err)
+			return err
 		}
 		*configPath = filepath.Dir(ex) + "/config.yaml"
 		if !fileExists(*configPath) {
@@ -88,13 +88,13 @@ func doService(configPath *string, configTest *bool, build string, serviceFlag *
 	// - above, in `Run` we create a `logrus.Logger` which is what nebula expects to use
 	s, err := service.New(prg, svcConfig)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 
 	errs := make(chan error, 5)
 	logger, err = s.Logger(errs)
 	if err != nil {
-		log.Fatal(err)
+		return err
 	}
 
 	go func() {
@@ -109,18 +109,16 @@ func doService(configPath *string, configTest *bool, build string, serviceFlag *
 
 	switch *serviceFlag {
 	case "run":
-		err = s.Run()
-		if err != nil {
+		if err := s.Run(); err != nil {
 			// Route any errors to the system logger
 			logger.Error(err)
 		}
 	default:
-		err := service.Control(s, *serviceFlag)
-		if err != nil {
+		if err := service.Control(s, *serviceFlag); err != nil {
 			log.Printf("Valid actions: %q\n", service.ControlAction)
-			log.Fatal(err)
+			return err
 		}
-		return
 	}
 
+	return nil
 }

cmd/nebula/main.go

@@ -3,9 +3,6 @@ package main
 import (
 	"flag"
 	"fmt"
-	"log"
-	"net/http"
-	_ "net/http/pprof"
 	"os"
 	"runtime/debug"
 	"strings"
@@ -74,14 +71,22 @@ func main() {
 		os.Exit(1)
 	}
 
-	go func() {
-		log.Println(http.ListenAndServe("0.0.0.0:6060", nil))
-	}()
-
 	if !*configTest {
-		ctrl.Start()
+		wait, err := ctrl.Start()
+		if err != nil {
+			util.LogWithContextIfNeeded("Error while running", err, l)
+			os.Exit(1)
+		}
+
+		go ctrl.ShutdownBlock()
 		notifyReady(l)
-		ctrl.ShutdownBlock()
+
+		if err := wait(); err != nil {
+			l.WithError(err).Error("Nebula stopped due to fatal error")
+			os.Exit(2)
+		}
+
+		l.Info("Goodbye")
 	}
 
 	os.Exit(0)

connection_state.go

@@ -13,7 +13,7 @@ import (
 	"github.com/slackhq/nebula/noiseutil"
 )
 
-const ReplayWindow = 4096
+const ReplayWindow = 1024
 
 type ConnectionState struct {
 	eKey           *NebulaCipherState

control.go

@@ -2,17 +2,34 @@ package nebula
 
 import (
 	"context"
+	"errors"
 	"net/netip"
 	"os"
 	"os/signal"
+	"sync"
 	"syscall"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/firewall/events"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/overlay"
 )
 
+type RunState int
+
+const (
+	StateUnknown RunState = iota
+	StateReady
+	StateStarted
+	StateStopping
+	StateStopped
+)
+
+var ErrAlreadyStarted = errors.New("nebula is already started")
+var ErrAlreadyStopped = errors.New("nebula cannot be restarted")
+var ErrUnknownState = errors.New("nebula state is invalid")
+
 // Every interaction here needs to take extra care to copy memory and not return or use arguments "as is" when touching
 // core. This means copying IP objects, slices, de-referencing pointers and taking the actual value, etc
 
@@ -26,6 +43,9 @@ type controlHostLister interface {
 }
 
 type Control struct {
+	stateLock sync.Mutex
+	state     RunState
+
 	f                      *Interface
 	l                      *logrus.Logger
 	ctx                    context.Context
@@ -49,10 +69,31 @@ type ControlHostInfo struct {
 	CurrentRelaysThroughMe []netip.Addr     `json:"currentRelaysThroughMe"`
 }
 
-// Start actually runs nebula, this is a nonblocking call. To block use Control.ShutdownBlock()
-func (c *Control) Start() {
+// Start actually runs nebula, this is a nonblocking call.
+// The returned function blocks until nebula has fully stopped and returns the
+// first fatal reader error (if any). A nil error means nebula shut down
+// gracefully; a non-nil error means a reader hit an unexpected failure that
+// triggered the shutdown.
+func (c *Control) Start() (func() error, error) {
+	c.stateLock.Lock()
+	defer c.stateLock.Unlock()
+	switch c.state {
+	case StateReady:
+		//yay!
+	case StateStopped, StateStopping:
+		return nil, ErrAlreadyStopped
+	case StateStarted:
+		return nil, ErrAlreadyStarted
+	default:
+		return nil, ErrUnknownState
+	}
+
 	// Activate the interface
-	c.f.activate()
+	err := c.f.activate()
+	if err != nil {
+		c.state = StateStopped
+		return nil, err
+	}
 
 	// Call all the delayed funcs that waited patiently for the interface to be created.
 	if c.sshStart != nil {
@@ -71,16 +112,40 @@ func (c *Control) Start() {
 		c.lighthouseStart()
 	}
 
+	c.f.triggerShutdown = c.Stop
+
 	// Start reading packets.
-	c.f.run()
+	out, err := c.f.run()
+	if err != nil {
+		c.state = StateStopped
+		return nil, err
+	}
+	c.state = StateStarted
+	return out, nil
+}
+
+func (c *Control) State() RunState {
+	c.stateLock.Lock()
+	defer c.stateLock.Unlock()
+	return c.state
 }
 
 func (c *Control) Context() context.Context {
 	return c.ctx
 }
 
-// Stop signals nebula to shutdown and close all tunnels, returns after the shutdown is complete
+// Stop is a non-blocking call that signals nebula to close all tunnels and shut down
 func (c *Control) Stop() {
+	c.stateLock.Lock()
+	if c.state != StateStarted {
+		c.stateLock.Unlock()
+		// We are stopping or stopped already
+		return
+	}
+
+	c.state = StateStopping
+	c.stateLock.Unlock()
+
 	// Stop the handshakeManager (and other services), to prevent new tunnels from
 	// being created while we're shutting them all down.
 	c.cancel()
@@ -89,7 +154,9 @@ func (c *Control) Stop() {
 	if err := c.f.Close(); err != nil {
 		c.l.WithError(err).Error("Close interface failed")
 	}
-	c.l.Info("Goodbye")
+	c.stateLock.Lock()
+	c.state = StateStopped
+	c.stateLock.Unlock()
 }
 
 // ShutdownBlock will listen for and block on term and interrupt signals, calling Control.Stop() once signalled
@@ -274,6 +341,36 @@ func (c *Control) Device() overlay.Device {
 	return c.f.inside
 }
 
+// SetFirewallEventReporter installs an event reporter on the current firewall.
+// Passing nil clears any installed reporter. The reporter is carried across
+// firewall rule reloads. Report* methods are invoked while nebula holds
+// internal locks and must be non-blocking; in particular they must not call
+// back into *Control methods that touch the firewall, or deadlock will
+// result.
+//
+// Installation is performed by shallow-copying the current *Firewall,
+// setting the reporter field on the copy, and swapping the pointer under
+// the conntrack lock. Every Firewall the data path sees therefore has an
+// immutable reporter slot, and emit sites can read it without any
+// synchronization of their own.
+func (c *Control) SetFirewallEventReporter(r events.Reporter) {
+	old := c.f.firewall
+	if old == nil {
+		return
+	}
+	old.Conntrack.Lock()
+	defer old.Conntrack.Unlock()
+
+	// Re-read under the lock in case a concurrent reload swapped in a new
+	// Firewall between the unlocked load above and here. Both Firewalls share
+	// the same Conntrack pointer in the normal (non-overflow) reload path,
+	// so the lock we hold is the right one for whichever we see now.
+	current := c.f.firewall
+	fw := *current
+	fw.reporter = r
+	c.f.firewall = &fw
+}
+
 func copyHostInfo(h *HostInfo, preferredRanges []netip.Prefix) ControlHostInfo {
 	chi := ControlHostInfo{
 		VpnAddrs:               make([]netip.Addr, len(h.vpnAddrs)),

control_test.go

@@ -79,6 +79,7 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) {
 	}, &Interface{})
 
 	c := Control{
+		state: StateReady,
 		f: &Interface{
 			hostMap: hm,
 		},

control_tester.go

@@ -1,5 +1,4 @@
 //go:build e2e_testing
-// +build e2e_testing
 
 package nebula
 
@@ -80,8 +79,7 @@ func (c *Control) GetFromTun(block bool) []byte {
 
 // GetFromUDP will pull a udp packet off the udp side of nebula
 func (c *Control) GetFromUDP(block bool) *udp.Packet {
-	out := c.f.outside.(*udp.TesterConn).Get(block)
-	return out
+	return c.f.outside.(*udp.TesterConn).Get(block)
 }
 
 func (c *Control) GetUDPTxChan() <-chan *udp.Packet {

dist/wireshark/nebula.lua

@@ -84,30 +84,24 @@ end
 
 function nebula.prefs_changed()
     if default_settings.all_ports == nebula.prefs.all_ports and default_settings.port == nebula.prefs.port then
-        -- Nothing changed, bail
         return
     end
 
-    -- Remove our old dissector
+    -- Remove all existing registrations
     DissectorTable.get("udp.port"):remove_all(nebula)
 
-    if nebula.prefs.all_ports and default_settings.all_ports ~= nebula.prefs.all_ports then
-        default_settings.all_port = nebula.prefs.all_ports
-
+    if nebula.prefs.all_ports then
+        -- Register on every port for hole punch capture
         for i=0, 65535 do
             DissectorTable.get("udp.port"):add(i, nebula)
         end
-
-        -- no need to establish again on specific ports
-        return
+    else
+        -- Register on the configured port only
+        DissectorTable.get("udp.port"):add(nebula.prefs.port, nebula)
     end
 
-
-    if default_settings.all_ports ~= nebula.prefs.all_ports then
-        -- Add our new port dissector
-        default_settings.port = nebula.prefs.port
-        DissectorTable.get("udp.port"):add(default_settings.port, nebula)
-    end
+    default_settings.all_ports = nebula.prefs.all_ports
+    default_settings.port = nebula.prefs.port
 end
 
 DissectorTable.get("udp.port"):add(default_settings.port, nebula)

dns_server.go

@@ -1,12 +1,14 @@
 package nebula
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"net/netip"
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 
 	"github.com/gaissmai/bart"
 	"github.com/miekg/dns"
@@ -14,32 +16,207 @@ import (
 	"github.com/slackhq/nebula/config"
 )
 
-// This whole thing should be rewritten to use context
-
-var dnsR *dnsRecords
-var dnsServer *dns.Server
-var dnsAddr string
-
-type dnsRecords struct {
+type dnsServer struct {
 	sync.RWMutex
 	l               *logrus.Logger
+	ctx             context.Context
 	dnsMap4         map[string]netip.Addr
 	dnsMap6         map[string]netip.Addr
 	hostMap         *HostMap
 	myVpnAddrsTable *bart.Lite
+
+	mux *dns.ServeMux
+
+	// enabled mirrors `lighthouse.serve_dns && lighthouse.am_lighthouse`.
+	// Start, Add, and reload consult it so callers don't need to know the
+	// gating rules. When it toggles off via reload, accumulated records are
+	// cleared so a later re-enable starts with a fresh map populated from
+	// new handshakes.
+	enabled atomic.Bool
+
+	serverMu sync.Mutex
+	server   *dns.Server
+	// started is closed once `server` has finished binding (or after
+	// ListenAndServe returns on a bind failure). Stop waits on it before
+	// calling Shutdown to avoid the miekg/dns "server not started" race
+	// where a Shutdown that arrives before bind completes is silently
+	// ignored, leaving the listener running forever.
+	started chan struct{}
+	addr    string
 }
 
-func newDnsRecords(l *logrus.Logger, cs *CertState, hostMap *HostMap) *dnsRecords {
-	return &dnsRecords{
+// newDnsServerFromConfig builds a dnsServer, applies the initial config, and
+// registers a reload callback. The reload callback is registered before the
+// initial config is applied, so a SIGHUP can later enable, fix, or disable
+// DNS even if the initial application failed.
+//
+// The dnsServer internally gates on `lighthouse.serve_dns &&
+// lighthouse.am_lighthouse`. Start and Add are safe to call unconditionally,
+// they no-op when DNS isn't enabled. Each Start invocation owns a ctx-cancel
+// watcher that tears the listener down on nebula shutdown. The returned
+// pointer is always non-nil, even on error.
+func newDnsServerFromConfig(ctx context.Context, l *logrus.Logger, cs *CertState, hostMap *HostMap, c *config.C) (*dnsServer, error) {
+	ds := &dnsServer{
 		l:               l,
+		ctx:             ctx,
 		dnsMap4:         make(map[string]netip.Addr),
 		dnsMap6:         make(map[string]netip.Addr),
 		hostMap:         hostMap,
 		myVpnAddrsTable: cs.myVpnAddrsTable,
 	}
+	ds.mux = dns.NewServeMux()
+	ds.mux.HandleFunc(".", ds.handleDnsRequest)
+
+	c.RegisterReloadCallback(func(c *config.C) {
+		if err := ds.reload(c, false); err != nil {
+			l.WithError(err).Error("Failed to reload DNS responder from config")
+		}
+	})
+
+	if err := ds.reload(c, true); err != nil {
+		return ds, err
+	}
+	return ds, nil
 }
 
-func (d *dnsRecords) Query(q uint16, data string) netip.Addr {
+// reload applies the latest config and reconciles the running state with it:
+//   - enabled toggled on  -> spawn a runner
+//   - enabled toggled off -> stop the runner
+//   - listen address changed (while running) -> restart on the new address
+//   - everything else     -> no-op
+//
+// On the initial call it only records configuration; Control.Start is what
+// launches the first runner via dnsStart.
+func (d *dnsServer) reload(c *config.C, initial bool) error {
+	wantsDns := c.GetBool("lighthouse.serve_dns", false)
+	amLighthouse := c.GetBool("lighthouse.am_lighthouse", false)
+	enabled := wantsDns && amLighthouse
+	newAddr := getDnsServerAddr(c)
+
+	d.serverMu.Lock()
+	running := d.server
+	runningStarted := d.started
+	sameAddr := d.addr == newAddr
+	d.addr = newAddr
+	d.enabled.Store(enabled)
+	d.serverMu.Unlock()
+
+	if initial {
+		if wantsDns && !amLighthouse {
+			d.l.Warn("DNS server refusing to run because this host is not a lighthouse.")
+		}
+		return nil
+	}
+
+	if !enabled {
+		if running != nil {
+			d.Stop()
+		}
+		// Drop any records that accumulated while enabled; a later re-enable
+		// will repopulate from fresh handshakes.
+		d.clearRecords()
+		return nil
+	}
+
+	if running == nil {
+		// Was disabled (or never started); bring it up now.
+		go d.Start()
+		return nil
+	}
+
+	if sameAddr {
+		return nil
+	}
+
+	d.shutdownServer(running, runningStarted, "reload")
+	// Old Start goroutine has now exited; bring up a fresh listener on the
+	// new address.
+	go d.Start()
+	return nil
+}
+
+// shutdownServer waits for the server to finish binding (so Shutdown actually
+// stops it rather than no-oping) and then shuts it down.
+func (d *dnsServer) shutdownServer(srv *dns.Server, started chan struct{}, reason string) {
+	if srv == nil {
+		return
+	}
+	if started != nil {
+		<-started
+	}
+	if err := srv.Shutdown(); err != nil {
+		d.l.WithError(err).WithField("reason", reason).Warn("Failed to shut down the DNS responder")
+	}
+}
+
+// Start binds and serves the DNS responder. Blocks until Stop is called or
+// the listener errors. Safe to call when DNS is disabled (returns
+// immediately). This is what Control.dnsStart points at.
+//
+// Must be invoked after the tun device is active so that lighthouse.dns.host
+// may bind to a nebula IP.
+func (d *dnsServer) Start() {
+	if !d.enabled.Load() {
+		return
+	}
+
+	started := make(chan struct{})
+	d.serverMu.Lock()
+	if d.ctx.Err() != nil {
+		d.serverMu.Unlock()
+		return
+	}
+	addr := d.addr
+	server := &dns.Server{
+		Addr:              addr,
+		Net:               "udp",
+		Handler:           d.mux,
+		NotifyStartedFunc: func() { close(started) },
+	}
+	d.server = server
+	d.started = started
+	d.serverMu.Unlock()
+
+	// Per-invocation ctx watcher. Exits when Start does, so we don't leak a
+	// watcher per reload-driven restart.
+	done := make(chan struct{})
+	go func() {
+		select {
+		case <-d.ctx.Done():
+			d.shutdownServer(server, started, "shutdown")
+		case <-done:
+		}
+	}()
+
+	d.l.WithField("dnsListener", addr).Info("Starting DNS responder")
+	err := server.ListenAndServe()
+	close(done)
+
+	// If the listener never bound (bind error) NotifyStartedFunc never fires,
+	// so close started here to release any Stop caller waiting on it.
+	select {
+	case <-started:
+	default:
+		close(started)
+	}
+
+	if err != nil {
+		d.l.WithError(err).Warn("Failed to run the DNS responder")
+	}
+}
+
+// Stop shuts down the active server, if any. Idempotent.
+func (d *dnsServer) Stop() {
+	d.serverMu.Lock()
+	srv := d.server
+	started := d.started
+	d.server = nil
+	d.started = nil
+	d.serverMu.Unlock()
+	d.shutdownServer(srv, started, "stop")
+}
+
+func (d *dnsServer) Query(q uint16, data string) netip.Addr {
 	data = strings.ToLower(data)
 	d.RLock()
 	defer d.RUnlock()
@@ -57,7 +234,7 @@ func (d *dnsRecords) Query(q uint16, data string) netip.Addr {
 	return netip.Addr{}
 }
 
-func (d *dnsRecords) QueryCert(data string) string {
+func (d *dnsServer) QueryCert(data string) string {
 	ip, err := netip.ParseAddr(data[:len(data)-1])
 	if err != nil {
 		return ""
@@ -80,8 +257,19 @@ func (d *dnsRecords) QueryCert(data string) string {
 	return string(b)
 }
 
+// clearRecords drops all DNS records.
+func (d *dnsServer) clearRecords() {
+	d.Lock()
+	defer d.Unlock()
+	clear(d.dnsMap4)
+	clear(d.dnsMap6)
+}
+
 // Add adds the first IPv4 and IPv6 address that appears in `addresses` as the record for `host`
-func (d *dnsRecords) Add(host string, addresses []netip.Addr) {
+func (d *dnsServer) Add(host string, addresses []netip.Addr) {
+	if !d.enabled.Load() {
+		return
+	}
 	host = strings.ToLower(host)
 	d.Lock()
 	defer d.Unlock()
@@ -101,7 +289,7 @@ func (d *dnsRecords) Add(host string, addresses []netip.Addr) {
 	}
 }
 
-func (d *dnsRecords) isSelfNebulaOrLocalhost(addr string) bool {
+func (d *dnsServer) isSelfNebulaOrLocalhost(addr string) bool {
 	a, _, _ := net.SplitHostPort(addr)
 	b, err := netip.ParseAddr(a)
 	if err != nil {
@@ -116,7 +304,7 @@ func (d *dnsRecords) isSelfNebulaOrLocalhost(addr string) bool {
 	return d.myVpnAddrsTable.Contains(b)
 }
 
-func (d *dnsRecords) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
+func (d *dnsServer) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
 	for _, q := range m.Question {
 		switch q.Qtype {
 		case dns.TypeA, dns.TypeAAAA:
@@ -150,7 +338,7 @@ func (d *dnsRecords) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
 	}
 }
 
-func (d *dnsRecords) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
+func (d *dnsServer) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
 	m := new(dns.Msg)
 	m.SetReply(r)
 	m.Compress = false
@@ -163,21 +351,6 @@ func (d *dnsRecords) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
 	w.WriteMsg(m)
 }
 
-func dnsMain(l *logrus.Logger, cs *CertState, hostMap *HostMap, c *config.C) func() {
-	dnsR = newDnsRecords(l, cs, hostMap)
-
-	// attach request handler func
-	dns.HandleFunc(".", dnsR.handleDnsRequest)
-
-	c.RegisterReloadCallback(func(c *config.C) {
-		reloadDns(l, c)
-	})
-
-	return func() {
-		startDns(l, c)
-	}
-}
-
 func getDnsServerAddr(c *config.C) string {
 	dnsHost := strings.TrimSpace(c.GetString("lighthouse.dns.host", ""))
 	// Old guidance was to provide the literal `[::]` in `lighthouse.dns.host` but that won't resolve.
@@ -186,25 +359,3 @@ func getDnsServerAddr(c *config.C) string {
 	}
 	return net.JoinHostPort(dnsHost, strconv.Itoa(c.GetInt("lighthouse.dns.port", 53)))
 }
-
-func startDns(l *logrus.Logger, c *config.C) {
-	dnsAddr = getDnsServerAddr(c)
-	dnsServer = &dns.Server{Addr: dnsAddr, Net: "udp"}
-	l.WithField("dnsListener", dnsAddr).Info("Starting DNS responder")
-	err := dnsServer.ListenAndServe()
-	defer dnsServer.Shutdown()
-	if err != nil {
-		l.Errorf("Failed to start server: %s\n ", err.Error())
-	}
-}
-
-func reloadDns(l *logrus.Logger, c *config.C) {
-	if dnsAddr == getDnsServerAddr(c) {
-		l.Debug("No DNS server config change detected")
-		return
-	}
-
-	l.Debug("Restarting DNS server")
-	dnsServer.Shutdown()
-	go startDns(l, c)
-}

dns_server_test.go

@@ -1,19 +1,31 @@
 package nebula
 
 import (
+	"context"
+	"io"
+	"net"
 	"net/netip"
+	"strconv"
 	"testing"
+	"time"
 
 	"github.com/miekg/dns"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func TestParsequery(t *testing.T) {
 	l := logrus.New()
 	hostMap := &HostMap{}
-	ds := newDnsRecords(l, &CertState{}, hostMap)
+	ds := &dnsServer{
+		l:       l,
+		dnsMap4: make(map[string]netip.Addr),
+		dnsMap6: make(map[string]netip.Addr),
+		hostMap: hostMap,
+	}
+	ds.enabled.Store(true)
 	addrs := []netip.Addr{
 		netip.MustParseAddr("1.2.3.4"),
 		netip.MustParseAddr("1.2.3.5"),
@@ -71,3 +83,209 @@ func Test_getDnsServerAddr(t *testing.T) {
 	}
 	assert.Equal(t, "[::]:1", getDnsServerAddr(c))
 }
+
+func newTestDnsServer(t *testing.T) (*dnsServer, *config.C) {
+	t.Helper()
+	l := logrus.New()
+	l.Out = io.Discard
+	ds := &dnsServer{
+		l:       l,
+		ctx:     context.Background(),
+		dnsMap4: make(map[string]netip.Addr),
+		dnsMap6: make(map[string]netip.Addr),
+		hostMap: &HostMap{},
+	}
+	ds.mux = dns.NewServeMux()
+	ds.mux.HandleFunc(".", ds.handleDnsRequest)
+	return ds, config.NewC(l)
+}
+
+func setDnsConfig(c *config.C, host string, port string, amLighthouse, serveDns bool) {
+	c.Settings["lighthouse"] = map[string]any{
+		"am_lighthouse": amLighthouse,
+		"serve_dns":     serveDns,
+		"dns": map[string]any{
+			"host": host,
+			"port": port,
+		},
+	}
+}
+
+func TestDnsServer_reload_initial_disabled(t *testing.T) {
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", "0", true, false)
+
+	require.NoError(t, ds.reload(c, true))
+	assert.False(t, ds.enabled.Load())
+	assert.Equal(t, "127.0.0.1:0", ds.addr)
+	assert.Nil(t, ds.server)
+}
+
+func TestDnsServer_reload_initial_enabled(t *testing.T) {
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", "0", true, true)
+
+	require.NoError(t, ds.reload(c, true))
+	assert.True(t, ds.enabled.Load())
+	assert.Equal(t, "127.0.0.1:0", ds.addr)
+	// initial never starts a runner; that's Control.Start's job
+	assert.Nil(t, ds.server)
+}
+
+func TestDnsServer_reload_initial_serveDnsWithoutLighthouse(t *testing.T) {
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", "0", false, true)
+
+	require.NoError(t, ds.reload(c, true))
+	// Wants DNS but isn't a lighthouse: gated off, no runner.
+	assert.False(t, ds.enabled.Load())
+}
+
+func TestDnsServer_reload_sameAddr_noOp(t *testing.T) {
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", "0", true, true)
+
+	require.NoError(t, ds.reload(c, true))
+	// No server running yet, no addr change. Reload should not spawn anything.
+	require.NoError(t, ds.reload(c, false))
+	assert.True(t, ds.enabled.Load())
+	assert.Nil(t, ds.server)
+}
+
+func TestDnsServer_StartStop_lifecycle(t *testing.T) {
+	// Bind to a real (random) UDP port so we exercise the actual
+	// ListenAndServe + Shutdown plumbing including the started-chan race fix.
+	port := freeUDPPort(t)
+
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", port, true, true)
+	require.NoError(t, ds.reload(c, true))
+
+	done := make(chan struct{})
+	go func() {
+		ds.Start()
+		close(done)
+	}()
+
+	waitFor(t, func() bool {
+		ds.serverMu.Lock()
+		started := ds.started
+		ds.serverMu.Unlock()
+		if started == nil {
+			return false
+		}
+		select {
+		case <-started:
+			return true
+		default:
+			return false
+		}
+	})
+
+	ds.Stop()
+	select {
+	case <-done:
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not return after Stop")
+	}
+}
+
+func TestDnsServer_Stop_beforeBind_doesNotHang(t *testing.T) {
+	// Stop called immediately after Start should not deadlock even if bind
+	// hasn't completed yet. This exercises the started-chan close-on-bind-fail
+	// path: by binding to an obviously bad port (privileged) we get a fast
+	// bind error before NotifyStartedFunc fires.
+	ds, c := newTestDnsServer(t)
+	// Use a port that should fail to bind (negative would be invalid, use a
+	// host that won't resolve to ensure listenUDP fails quickly).
+	setDnsConfig(c, "256.256.256.256", "53", true, true)
+	require.NoError(t, ds.reload(c, true))
+
+	done := make(chan struct{})
+	go func() {
+		ds.Start()
+		close(done)
+	}()
+
+	// Give Start a moment to attempt the bind and fail.
+	select {
+	case <-done:
+		// Bind failed and Start returned; Stop should be a no-op.
+	case <-time.After(time.Second):
+		t.Fatal("Start did not return after a bad bind")
+	}
+
+	stopped := make(chan struct{})
+	go func() {
+		ds.Stop()
+		close(stopped)
+	}()
+	select {
+	case <-stopped:
+	case <-time.After(time.Second):
+		t.Fatal("Stop hung after a failed bind")
+	}
+}
+
+func TestDnsServer_reload_disable_stopsRunningServer(t *testing.T) {
+	port := freeUDPPort(t)
+	ds, c := newTestDnsServer(t)
+	setDnsConfig(c, "127.0.0.1", port, true, true)
+	require.NoError(t, ds.reload(c, true))
+
+	startReturned := make(chan struct{})
+	go func() {
+		ds.Start()
+		close(startReturned)
+	}()
+	waitForBind(t, ds)
+
+	// Toggle serve_dns off; reload should shut the running server down.
+	setDnsConfig(c, "127.0.0.1", port, true, false)
+	require.NoError(t, ds.reload(c, false))
+	select {
+	case <-startReturned:
+	case <-time.After(5 * time.Second):
+		t.Fatal("Start did not return after reload disabled DNS")
+	}
+	assert.False(t, ds.enabled.Load())
+}
+
+func freeUDPPort(t *testing.T) string {
+	t.Helper()
+	conn, err := net.ListenPacket("udp", "127.0.0.1:0")
+	require.NoError(t, err)
+	port := conn.LocalAddr().(*net.UDPAddr).Port
+	require.NoError(t, conn.Close())
+	return strconv.Itoa(port)
+}
+
+func waitForBind(t *testing.T, ds *dnsServer) {
+	t.Helper()
+	waitFor(t, func() bool {
+		ds.serverMu.Lock()
+		started := ds.started
+		ds.serverMu.Unlock()
+		if started == nil {
+			return false
+		}
+		select {
+		case <-started:
+			return true
+		default:
+			return false
+		}
+	})
+}
+
+func waitFor(t *testing.T, cond func() bool) {
+	t.Helper()
+	deadline := time.Now().Add(5 * time.Second)
+	for time.Now().Before(deadline) {
+		if cond() {
+			return
+		}
+		time.Sleep(5 * time.Millisecond)
+	}
+	t.Fatal("timed out waiting for condition")
+}

e2e/handshake_manager_test.go

@@ -0,0 +1,565 @@
+//go:build e2e_testing
+// +build e2e_testing
+
+package e2e
+
+import (
+	"net/netip"
+	"testing"
+	"time"
+
+	"github.com/slackhq/nebula"
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/cert_test"
+	"github.com/slackhq/nebula/e2e/router"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/udp"
+	"github.com/stretchr/testify/assert"
+)
+
+// makeHandshakePacket creates a handshake packet with the given parameters.
+func makeHandshakePacket(from, to netip.AddrPort, subtype header.MessageSubType, remoteIndex uint32, counter uint64) *udp.Packet {
+	data := make([]byte, 200)
+	header.Encode(data, header.Version, header.Handshake, subtype, remoteIndex, counter)
+	for i := header.Len; i < len(data); i++ {
+		data[i] = byte(i)
+	}
+	return &udp.Packet{To: to, From: from, Data: data}
+}
+
+func TestHandshakeRetransmitDuplicate(t *testing.T) {
+	// Verify the responder correctly handles receiving the same msg1 multiple times
+	// (retransmission). The duplicate goes through CheckAndComplete -> ErrAlreadySeen
+	// and the cached response is resent.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	myControl.Start()
+	theirControl.Start()
+
+	r := router.NewR(t, myControl, theirControl)
+	defer r.RenderFlow()
+
+	t.Log("Trigger handshake from me to them")
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
+
+	t.Log("Grab my msg1")
+	msg1 := myControl.GetFromUDP(true)
+
+	t.Log("Inject msg1 into them, first time")
+	theirControl.InjectUDPPacket(msg1)
+	_ = theirControl.GetFromUDP(true)
+
+	t.Log("Inject the SAME msg1 again, tests ErrAlreadySeen path")
+	theirControl.InjectUDPPacket(msg1)
+	resp2 := theirControl.GetFromUDP(true)
+	assert.NotNil(t, resp2, "should get cached response on duplicate msg1")
+
+	t.Log("Complete handshake with cached response")
+	myControl.InjectUDPPacket(resp2)
+	myControl.WaitForType(1, 0, theirControl)
+
+	t.Log("Drain cached packet and verify tunnel works")
+	cachedPacket := theirControl.GetFromTun(true)
+	assertUdpPacket(t, []byte("Hi"), cachedPacket, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	t.Log("Verify only one tunnel exists on each side")
+	assert.Len(t, myControl.ListHostmapHosts(false), 1)
+	assert.Len(t, theirControl.ListHostmapHosts(false), 1)
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestHandshakeTruncatedPacketRecovery(t *testing.T) {
+	// Verify that a truncated handshake packet is ignored and the real
+	// packet can still complete the handshake.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	myControl.Start()
+	theirControl.Start()
+
+	r := router.NewR(t, myControl, theirControl)
+	defer r.RenderFlow()
+
+	t.Log("Trigger handshake")
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
+
+	t.Log("Get msg1 and deliver to responder")
+	msg1 := myControl.GetFromUDP(true)
+	theirControl.InjectUDPPacket(msg1)
+
+	t.Log("Get the real response")
+	realResp := theirControl.GetFromUDP(true)
+
+	t.Log("Truncate the response and inject, should be ignored")
+	truncResp := realResp.Copy()
+	truncResp.Data = truncResp.Data[:header.Len]
+	myControl.InjectUDPPacket(truncResp)
+
+	t.Log("Verify pending handshake survived the truncated packet")
+	assert.NotEmpty(t, myControl.ListHostmapHosts(true), "pending handshake should still exist")
+
+	t.Log("Inject real response, should complete handshake")
+	myControl.InjectUDPPacket(realResp)
+	myControl.WaitForType(1, 0, theirControl)
+
+	t.Log("Drain and verify tunnel")
+	cachedPacket := theirControl.GetFromTun(true)
+	assertUdpPacket(t, []byte("Hi"), cachedPacket, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestHandshakeOrphanedMsg2Dropped(t *testing.T) {
+	// A msg2 arriving with no matching pending index should be silently dropped
+	// with no response sent and no state changes.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	myControl.Start()
+	theirControl.Start()
+
+	r := router.NewR(t, myControl, theirControl)
+	defer r.RenderFlow()
+
+	t.Log("Complete a normal handshake")
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
+	r.RouteForAllUntilTxTun(theirControl)
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	t.Log("Record hostmap state")
+	myIndexes := len(myControl.ListHostmapIndexes(false))
+
+	t.Log("Inject a fake msg2 with unknown RemoteIndex")
+	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0xDEADBEEF, 2))
+
+	t.Log("Verify no new indexes created")
+	assert.Equal(t, myIndexes, len(myControl.ListHostmapIndexes(false)))
+
+	t.Log("Verify no UDP response was sent")
+	time.Sleep(100 * time.Millisecond)
+	assert.Nil(t, myControl.GetFromUDP(false), "should not send a response to orphaned msg2")
+
+	t.Log("Verify existing tunnel still works")
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestHandshakeUnknownMessageCounter(t *testing.T) {
+	// A handshake packet with an unexpected message counter should be silently
+	// dropped with no side effects and no UDP response.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+
+	myControl.Start()
+	theirControl.Start()
+
+	t.Log("Inject handshake with MessageCounter=3")
+	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 3))
+
+	t.Log("Inject handshake with MessageCounter=99")
+	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 99))
+
+	t.Log("Verify no tunnels or pending handshakes")
+	assert.Empty(t, myControl.ListHostmapHosts(false))
+	assert.Empty(t, myControl.ListHostmapHosts(true))
+
+	t.Log("Verify no UDP response was sent")
+	time.Sleep(100 * time.Millisecond)
+	assert.Nil(t, myControl.GetFromUDP(false))
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestHandshakeUnknownSubtype(t *testing.T) {
+	// A handshake packet with an unknown subtype should be silently dropped.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+	theirControl, _, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.Start()
+	theirControl.Start()
+
+	t.Log("Inject handshake with unknown subtype 99")
+	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.MessageSubType(99), 0, 1))
+
+	t.Log("Verify no tunnels or pending handshakes")
+	assert.Empty(t, myControl.ListHostmapHosts(false))
+	assert.Empty(t, myControl.ListHostmapHosts(true))
+
+	t.Log("Verify no UDP response was sent")
+	time.Sleep(100 * time.Millisecond)
+	assert.Nil(t, myControl.GetFromUDP(false))
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestHandshakeLateResponse(t *testing.T) {
+	// After a handshake times out, a late response should be silently ignored
+	// with no new tunnels created.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{
+		"handshakes": m{
+			"try_interval": "200ms",
+			"retries":      2,
+		},
+	})
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+
+	myControl.Start()
+	theirControl.Start()
+
+	t.Log("Trigger handshake from me")
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
+
+	t.Log("Grab msg1 but don't deliver")
+	msg1 := myControl.GetFromUDP(true)
+
+	t.Log("Wait for handshake to time out")
+	for i := 0; i < 5; i++ {
+		time.Sleep(300 * time.Millisecond)
+		myControl.GetFromUDP(false)
+	}
+
+	t.Log("Confirm no pending handshakes remain")
+	assert.Empty(t, myControl.ListHostmapHosts(true))
+
+	t.Log("Deliver old msg1 to them, they create a tunnel")
+	theirControl.InjectUDPPacket(msg1)
+	resp := theirControl.GetFromUDP(true)
+	assert.NotNil(t, resp)
+
+	t.Log("Inject late response into me, should be ignored")
+	myControl.InjectUDPPacket(resp)
+
+	t.Log("No tunnel should exist on my side")
+	assert.Empty(t, myControl.ListHostmapHosts(false))
+	assert.Empty(t, myControl.ListHostmapHosts(true))
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestHandshakeSelfConnectionRejected(t *testing.T) {
+	// Verify that a node rejects a handshake containing its own VPN IP in the
+	// peer cert. We do this by sending the initiator's own msg1 back to itself.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+
+	// Need a lighthouse entry to trigger a handshake
+	myControl.InjectLightHouseAddr(netip.MustParseAddr("10.128.0.2"), netip.MustParseAddrPort("10.0.0.2:4242"))
+
+	myControl.Start()
+
+	t.Log("Trigger handshake from me")
+	myControl.InjectTunUDPPacket(netip.MustParseAddr("10.128.0.2"), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
+	msg1 := myControl.GetFromUDP(true)
+
+	t.Log("Drain any handshake retransmits before injecting")
+	time.Sleep(100 * time.Millisecond)
+	for myControl.GetFromUDP(false) != nil {
+	}
+
+	t.Log("Feed my own msg1 back to me as if it came from someone else")
+	selfMsg := msg1.Copy()
+	selfMsg.From = netip.MustParseAddrPort("10.0.0.99:4242")
+	selfMsg.To = myUdpAddr
+	myControl.InjectUDPPacket(selfMsg)
+
+	t.Log("Verify no response was sent (self-connection rejected)")
+	time.Sleep(100 * time.Millisecond)
+	// Drain any further retransmits from the original handshake, then check
+	// that none of them are a handshake response (MessageCounter=2)
+	h := &header.H{}
+	for {
+		p := myControl.GetFromUDP(false)
+		if p == nil {
+			break
+		}
+		_ = h.Parse(p.Data)
+		assert.NotEqual(t, uint64(2), h.MessageCounter,
+			"should not send a stage 2 response to self-connection")
+	}
+
+	t.Log("Verify no tunnel to myself was created")
+	assert.Nil(t, myControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false))
+
+	myControl.Stop()
+}
+
+func TestHandshakeMessageCounter0Dropped(t *testing.T) {
+	// MessageCounter=0 is not a valid handshake message and should be dropped.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+	_, _, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.Start()
+
+	t.Log("Inject handshake with MessageCounter=0")
+	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 0))
+
+	time.Sleep(100 * time.Millisecond)
+	assert.Empty(t, myControl.ListHostmapHosts(false))
+	assert.Empty(t, myControl.ListHostmapHosts(true))
+	assert.Nil(t, myControl.GetFromUDP(false))
+
+	myControl.Stop()
+}
+
+func TestHandshakeRemoteAllowList(t *testing.T) {
+	// Verify that a handshake from a blocked underlay IP is dropped with no
+	// response and no state changes. Then verify the same packet from an
+	// allowed IP succeeds.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{
+		"lighthouse": m{
+			"remote_allow_list": m{
+				"10.0.0.0/8": true,
+				"0.0.0.0/0":  false,
+			},
+		},
+	})
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	myControl.Start()
+	theirControl.Start()
+
+	r := router.NewR(t, myControl, theirControl)
+	defer r.RenderFlow()
+
+	t.Log("Trigger handshake from them")
+	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi"))
+	msg1 := theirControl.GetFromUDP(true)
+
+	t.Log("Rewrite the source to a blocked IP and inject")
+	blockedMsg := msg1.Copy()
+	blockedMsg.From = netip.MustParseAddrPort("192.168.1.1:4242")
+	myControl.InjectUDPPacket(blockedMsg)
+
+	t.Log("Verify no tunnel, no pending, no response from blocked source")
+	time.Sleep(100 * time.Millisecond)
+	assert.Empty(t, myControl.ListHostmapHosts(false))
+	assert.Empty(t, myControl.ListHostmapHosts(true))
+	assert.Nil(t, myControl.GetFromUDP(false), "should not respond to blocked source")
+
+	t.Log("Now inject the real packet from the allowed source")
+	myControl.InjectUDPPacket(msg1)
+
+	t.Log("Verify handshake completes from allowed source")
+	resp := myControl.GetFromUDP(true)
+	assert.NotNil(t, resp)
+	theirControl.InjectUDPPacket(resp)
+	theirControl.WaitForType(1, 0, myControl)
+
+	t.Log("Drain cached packet and verify tunnel works")
+	cachedPacket := myControl.GetFromTun(true)
+	assertUdpPacket(t, []byte("Hi"), cachedPacket, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), 80, 80)
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestHandshakeAlreadySeenPreferredRemote(t *testing.T) {
+	// When a duplicate msg1 arrives via ErrAlreadySeen, verify the tunnel
+	// remains functional and hostmap index count is stable.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	myControl.Start()
+	theirControl.Start()
+
+	r := router.NewR(t, myControl, theirControl)
+	defer r.RenderFlow()
+
+	t.Log("Complete a normal handshake via the router")
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
+	r.RouteForAllUntilTxTun(theirControl)
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	t.Log("Record hostmap state")
+	theirIndexes := len(theirControl.ListHostmapIndexes(false))
+	hi := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
+	assert.NotNil(t, hi)
+	originalRemote := hi.CurrentRemote
+
+	t.Log("Re-trigger traffic to cause a new handshake attempt (ErrAlreadySeen)")
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("roam"))
+	r.RouteForAllUntilTxTun(theirControl)
+
+	t.Log("Verify tunnel still works")
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	t.Log("Verify remote is still valid and index count is stable")
+	hi2 := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
+	assert.NotNil(t, hi2)
+	assert.Equal(t, originalRemote, hi2.CurrentRemote)
+	assert.Equal(t, theirIndexes, len(theirControl.ListHostmapIndexes(false)),
+		"no extra indexes should be created from ErrAlreadySeen")
+
+	myControl.Stop()
+	theirControl.Stop()
+}
+
+func TestHandshakeWrongResponderPacketStore(t *testing.T) {
+	// Verify that when the wrong host responds, the cached packets are
+	// transferred to the new handshake, the evil tunnel is closed, evil's
+	// address is blocked, and the correct tunnel is eventually established.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.100/24", nil)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.99/24", nil)
+	evilControl, evilVpnIpNet, evilUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "evil", "10.128.0.2/24", nil)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), evilUdpAddr)
+
+	r := router.NewR(t, myControl, theirControl, evilControl)
+	defer r.RenderFlow()
+
+	myControl.Start()
+	theirControl.Start()
+	evilControl.Start()
+
+	t.Log("Send multiple packets to them (cached during handshake)")
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet1"))
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet2"))
+
+	t.Log("Route until evil tunnel is closed")
+	h := &header.H{}
+	r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType {
+		if err := h.Parse(p.Data); err != nil {
+			panic(err)
+		}
+		if h.Type == header.CloseTunnel && p.To == evilUdpAddr {
+			return router.RouteAndExit
+		}
+		return router.KeepRouting
+	})
+
+	t.Log("Verify evil's address is blocked in the new pending handshake")
+	pendingHI := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), true)
+	if pendingHI != nil {
+		assert.NotContains(t, pendingHI.RemoteAddrs, evilUdpAddr,
+			"evil's address should be blocked")
+	}
+
+	t.Log("Inject correct lighthouse addr for them")
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+
+	t.Log("Route until cached packets arrive at the real them")
+	p := r.RouteForAllUntilTxTun(theirControl)
+	assert.NotNil(t, p, "a cached packet should be delivered to the correct host")
+
+	t.Log("Verify the correct host has a tunnel")
+	assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet, theirVpnIpNet, myControl, theirControl)
+
+	t.Log("Verify no hostinfo artifacts from evil remain")
+	assert.Nil(t, myControl.GetHostInfoByVpnAddr(evilVpnIpNet[0].Addr(), true),
+		"no pending hostinfo for evil")
+	assert.Nil(t, myControl.GetHostInfoByVpnAddr(evilVpnIpNet[0].Addr(), false),
+		"no main hostinfo for evil")
+
+	myControl.Stop()
+	theirControl.Stop()
+	evilControl.Stop()
+}
+
+func TestHandshakeRelayComplete(t *testing.T) {
+	// Verify that a relay handshake completes correctly and relay state is
+	// properly maintained on all three nodes.
+
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
+	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
+
+	myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
+	myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
+	relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+
+	r := router.NewR(t, myControl, relayControl, theirControl)
+	defer r.RenderFlow()
+
+	myControl.Start()
+	relayControl.Start()
+	theirControl.Start()
+
+	t.Log("Trigger handshake via relay")
+	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi via relay"))
+
+	p := r.RouteForAllUntilTxTun(theirControl)
+	assertUdpPacket(t, []byte("Hi via relay"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
+
+	t.Log("Verify bidirectional tunnel via relay")
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	t.Log("Verify relay state on my side shows relay-to-me")
+	myHI := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
+	assert.NotNil(t, myHI)
+	assert.NotEmpty(t, myHI.CurrentRelaysToMe, "should have relay-to-me for them")
+
+	t.Log("Verify relay state on their side shows relay-to-me")
+	theirHI := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
+	assert.NotNil(t, theirHI)
+	assert.NotEmpty(t, theirHI.CurrentRelaysToMe, "should have relay-to-me for me")
+
+	t.Log("Verify relay node shows through-me relays")
+	relayHI := relayControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
+	assert.NotNil(t, relayHI)
+
+	myControl.Stop()
+	relayControl.Stop()
+	theirControl.Stop()
+}
+
+// NOTE: Relay V1 cert + IPv6 rejection is not tested here because
+// InjectTunUDPPacket from a V4 node to a V6 address panics in the test
+// framework. The check is in handshake_manager.go handleOutbound relay
+// logic (lines ~304-313): if the relay host has a V1 cert and either
+// address is IPv6, the relay is skipped.
+
+// NOTE: Relay reestablishment (Disestablished state transition) is covered
+// by the existing TestReestablishRelays in handshakes_test.go.

e2e/tunnels_test.go

@@ -12,6 +12,8 @@ import (
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/cert_test"
 	"github.com/slackhq/nebula/e2e/router"
+	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/udp"
 	"github.com/stretchr/testify/assert"
 	"gopkg.in/yaml.v3"
 )
@@ -365,3 +367,106 @@ func TestCrossStackRelaysWork(t *testing.T) {
 	//theirControl.Stop()
 	//relayControl.Stop()
 }
+
+func TestCloseTunnelAuthenticated(t *testing.T) {
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "5s"}})
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "10m"}})
+
+	// Share our underlay information
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	// Start the servers
+	myControl.Start()
+	theirControl.Start()
+
+	r := router.NewR(t, myControl, theirControl)
+
+	r.Log("Assert the tunnel between me and them works")
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+
+	r.Log("Close the tunnel")
+	myControl.CloseTunnel(theirVpnIpNet[0].Addr(), false)
+	r.FlushAll()
+
+	waitStart := time.Now()
+	for {
+		myIndexes := len(myControl.GetHostmap().Indexes)
+		theirIndexes := len(theirControl.GetHostmap().Indexes)
+		if myIndexes == 0 && theirIndexes == 0 {
+			break
+		}
+
+		since := time.Since(waitStart)
+		r.Logf("my tunnels: %v; their tunnels: %v; duration: %v", myIndexes, theirIndexes, since)
+		if since > time.Second*6 {
+			t.Fatal("Tunnel should have been declared inactive after 2 seconds and before 6 seconds")
+		}
+
+		time.Sleep(1 * time.Second)
+		//r.FlushAll()
+	}
+
+	r.Logf("Happy path success, tunnels were dropped within %v", time.Since(waitStart))
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+	r.Log("Assert another tunnel between me and them works")
+	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
+	hi := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
+	if hi == nil {
+		t.Fatal("There is no hostinfo for this tunnel")
+	}
+	myHi := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
+	if myHi == nil {
+		t.Fatal("There is no hostinfo for my tunnel")
+	}
+	r.Log("It does")
+
+	buf := make([]byte, 1024)
+	hdr := header.H{
+		Version:        1,
+		Type:           header.CloseTunnel,
+		Subtype:        0,
+		Reserved:       0,
+		RemoteIndex:    hi.RemoteIndex,
+		MessageCounter: 5,
+	}
+	out, err := hdr.Encode(buf)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	pkt := &udp.Packet{
+		To:   hi.CurrentRemote,
+		From: myHi.CurrentRemote,
+		Data: out,
+	}
+	r.InjectUDPPacket(myControl, theirControl, pkt)
+	r.Log("Injected bogus close tunnel. Let's see!")
+	waitStart = time.Now()
+	for {
+		myIndexes := len(myControl.GetHostmap().Indexes)
+		theirIndexes := len(theirControl.GetHostmap().Indexes)
+		if myIndexes == 0 {
+			t.Fatal("myIndexes should not be 0")
+		}
+		if theirIndexes == 0 {
+			t.Fatal("theirIndexes should not be 0, they should have rejected this bogus packet")
+		}
+
+		since := time.Since(waitStart)
+		r.Logf("my tunnels: %v; their tunnels: %v; duration: %v", myIndexes, theirIndexes, since)
+		if since > time.Second*4 {
+			t.Log("The tunnel would have been gone by now")
+			break
+		}
+
+		time.Sleep(1 * time.Second)
+		r.FlushAll()
+	}
+
+	myControl.Stop()
+	theirControl.Stop()
+}

examples/config.yml

@@ -144,6 +144,10 @@ listen:
   # valid values: always, never, private
   # This setting is reloadable.
   #send_recv_error: always
+  # Similar to send_recv_error, this option lets you configure if you want to accept "recv_error" packets from remote hosts.
+  # valid values: always, never, private
+  # This setting is reloadable.
+  #accept_recv_error: always
   # The so_sock option is a Linux-specific feature that allows all outgoing Nebula packets to be tagged with a specific identifier.
   # This tagging enables IP rule-based filtering. For example, it supports 0.0.0.0/0 unsafe_routes,
   # allowing for more precise routing decisions based on the packet tags. Default is 0 meaning no mark is set.
@@ -200,6 +204,12 @@ punchy:
   # Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access.
   #trusted_cas:
     #- "ssh public key string"
+  # sandbox_dir restricts file paths for profiling commands (start-cpu-profile, save-heap-profile,
+  # save-mutex-profile) to the specified directory. Relative paths will be resolved within this directory,
+  # and absolute paths outside of it will be rejected. Default is $TMP/nebula-debug.
+  # The directory is NOT automatically created.
+  # Overriding this to "" is the same as "/" and will allow overwriting any path on the host.
+  #sandbox_dir: /var/tmp/nebula-debug
 
 # EXPERIMENTAL: relay support for networks that can't establish direct connections.
 relay:
@@ -378,8 +388,8 @@ firewall:
   # Rules are comprised of a protocol, port, and one or more of host, group, or CIDR
   # Logical evaluation is roughly: port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND (local cidr)
   # - port: Takes `0` or `any` as any, a single number `80`, a range `200-901`, or `fragment` to match second and further fragments of fragmented packets (since there is no port available).
-  #   code: same as port but makes more sense when talking about ICMP, TODO: this is not currently implemented in a way that works, use `any`
   #   proto: `any`, `tcp`, `udp`, or `icmp`
+  #          a port specification is ignored if proto is `icmp`
   #   host: `any` or a literal hostname, ie `test-host`
   #   group: `any` or a literal group name, ie `default-group`
   #   groups: Same as group but accepts a list of values. Multiple values are AND'd together and a certificate would have to contain all groups to pass

firewall.go

@@ -20,6 +20,7 @@ import (
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/firewall"
+	"github.com/slackhq/nebula/firewall/events"
 )
 
 type FirewallInterface interface {
@@ -67,6 +68,14 @@ type Firewall struct {
 	incomingMetrics     firewallMetrics
 	outgoingMetrics     firewallMetrics
 
+	// reporter is the optional embedder-supplied event sink. Immutable for
+	// the lifetime of this Firewall; Control.SetFirewallEventReporter
+	// installs it by shallow-copying the Firewall under the conntrack lock
+	// and swapping the pointer, and reloadFirewall carries it forward.
+	// Read unsynchronized on the data path: the preceding Firewall-pointer
+	// read pins the field's value for the duration of that call.
+	reporter events.Reporter
+
 	l *logrus.Logger
 }
 
@@ -230,7 +239,7 @@ func NewFirewallFromConfig(l *logrus.Logger, cs *CertState, c *config.C) (*Firew
 	case "drop":
 		fw.OutSendReject = false
 	default:
-		l.WithField("action", inboundAction).Warn("invalid firewall.outbound_action, defaulting to `drop`")
+		l.WithField("action", outboundAction).Warn("invalid firewall.outbound_action, defaulting to `drop`")
 		fw.OutSendReject = false
 	}
 
@@ -249,20 +258,6 @@ func NewFirewallFromConfig(l *logrus.Logger, cs *CertState, c *config.C) (*Firew
 
 // AddRule properly creates the in memory rule structure for a firewall table.
 func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error {
-	// We need this rule string because we generate a hash. Removing this will break firewall reload.
-	ruleString := fmt.Sprintf(
-		"incoming: %v, proto: %v, startPort: %v, endPort: %v, groups: %v, host: %v, ip: %v, localIp: %v, caName: %v, caSha: %s",
-		incoming, proto, startPort, endPort, groups, host, cidr, localCidr, caName, caSha,
-	)
-	f.rules += ruleString + "\n"
-
-	direction := "incoming"
-	if !incoming {
-		direction = "outgoing"
-	}
-	f.l.WithField("firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "cidr": cidr, "localCidr": localCidr, "caName": caName, "caSha": caSha}).
-		Info("Firewall rule added")
-
 	var (
 		ft *FirewallTable
 		fp firewallPort
@@ -280,6 +275,12 @@ func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort
 	case firewall.ProtoUDP:
 		fp = ft.UDP
 	case firewall.ProtoICMP, firewall.ProtoICMPv6:
+		//ICMP traffic doesn't have ports, so we always coerce to "any", even if a value is provided
+		if startPort != firewall.PortAny {
+			f.l.WithField("startPort", startPort).Warn("ignoring port specification for ICMP firewall rule")
+		}
+		startPort = firewall.PortAny
+		endPort = firewall.PortAny
 		fp = ft.ICMP
 	case firewall.ProtoAny:
 		fp = ft.AnyProto
@@ -287,6 +288,20 @@ func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort
 		return fmt.Errorf("unknown protocol %v", proto)
 	}
 
+	// We need this rule string because we generate a hash. Removing this will break firewall reload.
+	ruleString := fmt.Sprintf(
+		"incoming: %v, proto: %v, startPort: %v, endPort: %v, groups: %v, host: %v, ip: %v, localIp: %v, caName: %v, caSha: %s",
+		incoming, proto, startPort, endPort, groups, host, cidr, localCidr, caName, caSha,
+	)
+	f.rules += ruleString + "\n"
+
+	direction := "incoming"
+	if !incoming {
+		direction = "outgoing"
+	}
+	f.l.WithField("firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "cidr": cidr, "localCidr": localCidr, "caName": caName, "caSha": caSha}).
+		Info("Firewall rule added")
+
 	return fp.addRule(f, startPort, endPort, groups, host, cidr, localCidr, caName, caSha)
 }
 
@@ -349,24 +364,31 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw
 			sPort = r.Port
 		}
 
-		startPort, endPort, err := parsePort(sPort)
-		if err != nil {
-			return fmt.Errorf("%s rule #%v; %s %s", table, i, errPort, err)
-		}
-
 		var proto uint8
+		var startPort, endPort int32
 		switch r.Proto {
 		case "any":
 			proto = firewall.ProtoAny
+			startPort, endPort, err = parsePort(sPort)
 		case "tcp":
 			proto = firewall.ProtoTCP
+			startPort, endPort, err = parsePort(sPort)
 		case "udp":
 			proto = firewall.ProtoUDP
+			startPort, endPort, err = parsePort(sPort)
 		case "icmp":
 			proto = firewall.ProtoICMP
+			startPort = firewall.PortAny
+			endPort = firewall.PortAny
+			if sPort != "" {
+				l.WithField("port", sPort).Warn("ignoring port specification for ICMP firewall rule")
+			}
 		default:
 			return fmt.Errorf("%s rule #%v; proto was not understood; `%s`", table, i, r.Proto)
 		}
+		if err != nil {
+			return fmt.Errorf("%s rule #%v; %s %s", table, i, errPort, err)
+		}
 
 		if r.Cidr != "" && r.Cidr != "any" {
 			_, err = netip.ParsePrefix(r.Cidr)
@@ -403,23 +425,27 @@ var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
 
 // Drop returns an error if the packet should be dropped, explaining why. It
 // returns nil if the packet should not be dropped.
-func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache, now time.Time) error {
+func (f *Firewall) Drop(fp firewall.Packet, ctx firewall.PacketContext, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error {
 	// Check if we spoke to this tuple, if we did then allow this packet
-	if f.inConns(fp, h, caPool, localCache, now) {
+	if f.inConns(fp, h, caPool, localCache) {
 		return nil
 	}
 
+	peerCert := h.ConnectionState.peerCert
+
 	// Make sure remote address matches nebula certificate, and determine how to treat it
 	if h.networks == nil {
 		// Simple case: Certificate has one address and no unsafe networks
 		if h.vpnAddrs[0] != fp.RemoteAddr {
 			f.metrics(incoming).droppedRemoteAddr.Inc(1)
+			f.reportDrop(incoming, events.DropInvalidRemoteIP, fp, ctx, peerCert)
 			return ErrInvalidRemoteIP
 		}
 	} else {
 		nwType, ok := h.networks.Lookup(fp.RemoteAddr)
 		if !ok {
 			f.metrics(incoming).droppedRemoteAddr.Inc(1)
+			f.reportDrop(incoming, events.DropInvalidRemoteIP, fp, ctx, peerCert)
 			return ErrInvalidRemoteIP
 		}
 		switch nwType {
@@ -427,11 +453,13 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
 			break // nothing special
 		case NetworkTypeVPNPeer:
 			f.metrics(incoming).droppedRemoteAddr.Inc(1)
+			f.reportDrop(incoming, events.DropPeerRejected, fp, ctx, peerCert)
 			return ErrPeerRejected // reject for now, one day this may have different FW rules
 		case NetworkTypeUnsafe:
 			break // nothing special, one day this may have different FW rules
 		default:
 			f.metrics(incoming).droppedRemoteAddr.Inc(1)
+			f.reportDrop(incoming, events.DropUnknownNetwork, fp, ctx, peerCert)
 			return ErrUnknownNetworkType //should never happen
 		}
 	}
@@ -439,6 +467,7 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
 	// Make sure we are supposed to be handling this local ip address
 	if !f.routableNetworks.Contains(fp.LocalAddr) {
 		f.metrics(incoming).droppedLocalAddr.Inc(1)
+		f.reportDrop(incoming, events.DropInvalidLocalIP, fp, ctx, peerCert)
 		return ErrInvalidLocalIP
 	}
 
@@ -448,13 +477,14 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
 	}
 
 	// We now know which firewall table to check against
-	if !table.match(fp, incoming, h.ConnectionState.peerCert, caPool) {
+	if !table.match(fp, incoming, peerCert, caPool) {
 		f.metrics(incoming).droppedNoRule.Inc(1)
+		f.reportDrop(incoming, events.DropNoMatchingRule, fp, ctx, peerCert)
 		return ErrNoMatchingRule
 	}
 
 	// We always want to conntrack since it is a faster operation
-	f.addConn(fp, incoming, now)
+	f.addConn(fp, ctx, incoming, peerCert)
 
 	return nil
 }
@@ -467,12 +497,65 @@ func (f *Firewall) metrics(incoming bool) firewallMetrics {
 	}
 }
 
-// Destroy cleans up any known cyclical references so the object can be free'd my GC. This should be called if a new
+// Destroy cleans up any known cyclical references so the object can be freed by GC. This should be called if a new
 // firewall object is created
 func (f *Firewall) Destroy() {
 	//TODO: clean references if/when needed
 }
 
+func (f *Firewall) reportDrop(incoming bool, reason events.DropReason, fp firewall.Packet, ctx firewall.PacketContext, peerCert *cert.CachedCertificate) {
+	r := f.reporter
+	if r == nil {
+		return
+	}
+	r.ReportDrop(events.DropEvent{
+		Incoming:     incoming,
+		Reason:       reason,
+		Packet:       fp,
+		Context:      ctx,
+		PeerCert:     peerCert,
+		RulesVersion: f.rulesVersion,
+	})
+}
+
+func (f *Firewall) reportFlowCreate(incoming bool, fp firewall.Packet, ctx firewall.PacketContext, peerCert *cert.CachedCertificate) {
+	r := f.reporter
+	if r == nil {
+		return
+	}
+	r.ReportFlowCreate(events.FlowCreateEvent{
+		Incoming:     incoming,
+		Packet:       fp,
+		Context:      ctx,
+		PeerCert:     peerCert,
+		RulesVersion: f.rulesVersion,
+	})
+}
+
+func (f *Firewall) reportFlowEvict(incoming bool, fp firewall.Packet, rulesVersion uint16, expired bool) {
+	r := f.reporter
+	if r == nil {
+		return
+	}
+	r.ReportFlowEvict(events.FlowEvictEvent{
+		Incoming:     incoming,
+		Packet:       fp,
+		RulesVersion: rulesVersion,
+		Expired:      expired,
+	})
+}
+
+func (f *Firewall) reportRulesReload(oldVersion, newVersion uint16) {
+	r := f.reporter
+	if r == nil {
+		return
+	}
+	r.ReportRulesReload(events.RulesReloadEvent{
+		OldVersion: oldVersion,
+		NewVersion: newVersion,
+	})
+}
+
 func (f *Firewall) EmitStats() {
 	conntrack := f.Conntrack
 	conntrack.Lock()
@@ -483,7 +566,7 @@ func (f *Firewall) EmitStats() {
 	metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV()))
 }
 
-func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache, now time.Time) bool {
+func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool {
 	if localCache != nil {
 		if _, ok := localCache[fp]; ok {
 			return true
@@ -495,7 +578,7 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 	// Purge every time we test
 	ep, has := conntrack.TimerWheel.Purge()
 	if has {
-		f.evict(ep, now)
+		f.evict(ep)
 	}
 
 	c, ok := conntrack.Conns[fp]
@@ -523,7 +606,9 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 					WithField("oldRulesVersion", c.rulesVersion).
 					Debugln("dropping old conntrack entry, does not match new ruleset")
 			}
+			oldRulesVersion := c.rulesVersion
 			delete(conntrack.Conns, fp)
+			f.reportFlowEvict(c.incoming, fp, oldRulesVersion, false)
 			conntrack.Unlock()
 			return false
 		}
@@ -542,11 +627,11 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 
 	switch fp.Protocol {
 	case firewall.ProtoTCP:
-		c.Expires = now.Add(f.TCPTimeout)
+		c.Expires = time.Now().Add(f.TCPTimeout)
 	case firewall.ProtoUDP:
-		c.Expires = now.Add(f.UDPTimeout)
+		c.Expires = time.Now().Add(f.UDPTimeout)
 	default:
-		c.Expires = now.Add(f.DefaultTimeout)
+		c.Expires = time.Now().Add(f.DefaultTimeout)
 	}
 
 	conntrack.Unlock()
@@ -558,7 +643,7 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 	return true
 }
 
-func (f *Firewall) addConn(fp firewall.Packet, incoming bool, now time.Time) {
+func (f *Firewall) addConn(fp firewall.Packet, ctx firewall.PacketContext, incoming bool, peerCert *cert.CachedCertificate) {
 	var timeout time.Duration
 	c := &conn{}
 
@@ -573,8 +658,9 @@ func (f *Firewall) addConn(fp firewall.Packet, incoming bool, now time.Time) {
 
 	conntrack := f.Conntrack
 	conntrack.Lock()
-	if _, ok := conntrack.Conns[fp]; !ok {
-		conntrack.TimerWheel.Advance(now)
+	_, existing := conntrack.Conns[fp]
+	if !existing {
+		conntrack.TimerWheel.Advance(time.Now())
 		conntrack.TimerWheel.Add(fp, timeout)
 	}
 
@@ -582,14 +668,21 @@ func (f *Firewall) addConn(fp firewall.Packet, incoming bool, now time.Time) {
 	// firewall reload
 	c.incoming = incoming
 	c.rulesVersion = f.rulesVersion
-	c.Expires = now.Add(timeout)
+	c.Expires = time.Now().Add(timeout)
 	conntrack.Conns[fp] = c
+
+	// Report only when this represents a genuinely new flow. Fires under the
+	// conntrack lock so FlowCreate/FlowEvict events stay ordered relative to
+	// RulesReloadEvent, which also fires under this lock.
+	if !existing {
+		f.reportFlowCreate(incoming, fp, ctx, peerCert)
+	}
 	conntrack.Unlock()
 }
 
 // Evict checks if a conntrack entry has expired, if so it is removed, if not it is re-added to the wheel
 // Caller must own the connMutex lock!
-func (f *Firewall) evict(p firewall.Packet, now time.Time) {
+func (f *Firewall) evict(p firewall.Packet) {
 	// Are we still tracking this conn?
 	conntrack := f.Conntrack
 	t, ok := conntrack.Conns[p]
@@ -597,17 +690,20 @@ func (f *Firewall) evict(p firewall.Packet, now time.Time) {
 		return
 	}
 
-	newT := t.Expires.Sub(now)
+	newT := t.Expires.Sub(time.Now())
 
 	// Timeout is in the future, re-add the timer
 	if newT > 0 {
-		conntrack.TimerWheel.Advance(now)
+		conntrack.TimerWheel.Advance(time.Now())
 		conntrack.TimerWheel.Add(p, newT)
 		return
 	}
 
 	// This conn is done
+	rulesVersion := t.rulesVersion
+	incoming := t.incoming
 	delete(conntrack.Conns, p)
+	f.reportFlowEvict(incoming, p, rulesVersion, true)
 }
 
 func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
@@ -660,6 +756,13 @@ func (fp firewallPort) match(p firewall.Packet, incoming bool, c *cert.CachedCer
 		return false
 	}
 
+	// this branch is here to catch traffic from FirewallTable.Any.match and FirewallTable.ICMP.match
+	if p.Protocol == firewall.ProtoICMP || p.Protocol == firewall.ProtoICMPv6 {
+		// port numbers are re-used for connection tracking of ICMP,
+		// but we don't want to actually filter on them.
+		return fp[firewall.PortAny].match(p, c, caPool)
+	}
+
 	var port int32
 
 	if p.Fragment {
@@ -804,10 +907,8 @@ func (fr *FirewallRule) isAny(groups []string, host string, cidr string) bool {
 		return true
 	}
 
-	for _, group := range groups {
-		if group == "any" {
-			return true
-		}
+	if slices.Contains(groups, "any") {
+		return true
 	}
 
 	if host == "any" {
@@ -1018,54 +1119,56 @@ func (r *rule) sanity() error {
 		}
 	}
 
+	if r.Code != "" {
+		return fmt.Errorf("code specified as [%s]. Support for 'code' will be dropped in a future release, as it has never been functional", r.Code)
+	}
+
 	//todo alert on cidr-any
 
 	return nil
 }
 
-func parsePort(s string) (startPort, endPort int32, err error) {
+func parsePort(s string) (int32, int32, error) {
+	var err error
+	const notAPort int32 = -2
 	if s == "any" {
-		startPort = firewall.PortAny
-		endPort = firewall.PortAny
-
-	} else if s == "fragment" {
-		startPort = firewall.PortFragment
-		endPort = firewall.PortFragment
-
-	} else if strings.Contains(s, `-`) {
-		sPorts := strings.SplitN(s, `-`, 2)
-		sPorts[0] = strings.Trim(sPorts[0], " ")
-		sPorts[1] = strings.Trim(sPorts[1], " ")
-
-		if len(sPorts) != 2 || sPorts[0] == "" || sPorts[1] == "" {
-			return 0, 0, fmt.Errorf("appears to be a range but could not be parsed; `%s`", s)
-		}
-
-		rStartPort, err := strconv.Atoi(sPorts[0])
-		if err != nil {
-			return 0, 0, fmt.Errorf("beginning range was not a number; `%s`", sPorts[0])
-		}
-
-		rEndPort, err := strconv.Atoi(sPorts[1])
-		if err != nil {
-			return 0, 0, fmt.Errorf("ending range was not a number; `%s`", sPorts[1])
-		}
-
-		startPort = int32(rStartPort)
-		endPort = int32(rEndPort)
-
-		if startPort == firewall.PortAny {
-			endPort = firewall.PortAny
-		}
-
-	} else {
+		return firewall.PortAny, firewall.PortAny, nil
+	}
+	if s == "fragment" {
+		return firewall.PortFragment, firewall.PortFragment, nil
+	}
+	if !strings.Contains(s, `-`) {
 		rPort, err := strconv.Atoi(s)
 		if err != nil {
-			return 0, 0, fmt.Errorf("was not a number; `%s`", s)
+			return notAPort, notAPort, fmt.Errorf("was not a number; `%s`", s)
 		}
-		startPort = int32(rPort)
-		endPort = startPort
+		return int32(rPort), int32(rPort), nil
 	}
 
-	return
+	sPorts := strings.SplitN(s, `-`, 2)
+	for i := range sPorts {
+		sPorts[i] = strings.Trim(sPorts[i], " ")
+	}
+	if len(sPorts) != 2 || sPorts[0] == "" || sPorts[1] == "" {
+		return notAPort, notAPort, fmt.Errorf("appears to be a range but could not be parsed; `%s`", s)
+	}
+
+	rStartPort, err := strconv.Atoi(sPorts[0])
+	if err != nil {
+		return notAPort, notAPort, fmt.Errorf("beginning range was not a number; `%s`", sPorts[0])
+	}
+
+	rEndPort, err := strconv.Atoi(sPorts[1])
+	if err != nil {
+		return notAPort, notAPort, fmt.Errorf("ending range was not a number; `%s`", sPorts[1])
+	}
+
+	startPort := int32(rStartPort)
+	endPort := int32(rEndPort)
+
+	if startPort == firewall.PortAny {
+		endPort = firewall.PortAny
+	}
+
+	return startPort, endPort, nil
 }

firewall/cache.go

@@ -1,6 +1,7 @@
 package firewall
 
 import (
+	"context"
 	"sync/atomic"
 	"time"
 
@@ -18,7 +19,7 @@ type ConntrackCacheTicker struct {
 	cache ConntrackCache
 }
 
-func NewConntrackCacheTicker(d time.Duration) *ConntrackCacheTicker {
+func NewConntrackCacheTicker(ctx context.Context, d time.Duration) *ConntrackCacheTicker {
 	if d == 0 {
 		return nil
 	}
@@ -27,15 +28,21 @@ func NewConntrackCacheTicker(d time.Duration) *ConntrackCacheTicker {
 		cache: ConntrackCache{},
 	}
 
-	go c.tick(d)
+	go c.tick(ctx, d)
 
 	return c
 }
 
-func (c *ConntrackCacheTicker) tick(d time.Duration) {
+func (c *ConntrackCacheTicker) tick(ctx context.Context, d time.Duration) {
+	t := time.NewTicker(d)
+	defer t.Stop()
 	for {
-		time.Sleep(d)
-		c.cacheTick.Add(1)
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			c.cacheTick.Add(1)
+		}
 	}
 }
 

firewall/events/events.go

@@ -0,0 +1,102 @@
+// Package events defines the opt-in firewall event reporting interface.
+//
+// Nebula emits raw packet-level events (drops, flow creations, flow evictions,
+// rule reloads) and does no aggregation, counting, batching, rule-description,
+// transport, or timestamping. Embedders correlate events back to yaml rules
+// out of band and capture whatever clock they need themselves. All Report*
+// methods are invoked while nebula holds internal locks and must be
+// non-blocking.
+//
+// Events are passed to Report* methods by value. Implementations must not
+// take the address of a received event: doing so forces Go's escape
+// analysis to move the event to the heap and costs one allocation per call.
+// To forward an event, either copy its fields into the reporter's own
+// pooled record or send it through a value-typed channel (chan DropEvent,
+// not chan *DropEvent).
+package events
+
+import (
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/firewall"
+)
+
+type DropReason uint8
+
+const (
+	DropInvalidLocalIP DropReason = iota
+	DropInvalidRemoteIP
+	DropPeerRejected
+	DropUnknownNetwork
+	DropNoMatchingRule
+)
+
+func (r DropReason) String() string {
+	switch r {
+	case DropInvalidLocalIP:
+		return "invalid_local_ip"
+	case DropInvalidRemoteIP:
+		return "invalid_remote_ip"
+	case DropPeerRejected:
+		return "peer_rejected"
+	case DropUnknownNetwork:
+		return "unknown_network"
+	case DropNoMatchingRule:
+		return "no_matching_rule"
+	default:
+		return "unknown"
+	}
+}
+
+// DropEvent is emitted for every packet that fails the firewall check. Drops
+// are not aggregated; every drop produces one event.
+type DropEvent struct {
+	Incoming     bool
+	Reason       DropReason
+	Packet       firewall.Packet
+	Context      firewall.PacketContext
+	PeerCert     *cert.CachedCertificate
+	RulesVersion uint16
+}
+
+// FlowCreateEvent is emitted when a packet is allowed and a new conntrack
+// entry is created. Subsequent packets in the same flow do not re-emit.
+type FlowCreateEvent struct {
+	Incoming     bool
+	Packet       firewall.Packet
+	Context      firewall.PacketContext
+	PeerCert     *cert.CachedCertificate
+	RulesVersion uint16
+}
+
+// FlowEvictEvent is emitted when a conntrack entry is removed. Context is
+// not carried: timer-wheel eviction has no packet in hand, and reload
+// revalidation evicts the OLD flow rather than the triggering packet.
+// RulesVersion is the version under which the flow was originally allowed,
+// which may differ from the current firewall version.
+type FlowEvictEvent struct {
+	Incoming     bool
+	Packet       firewall.Packet
+	RulesVersion uint16
+	// Expired is true when eviction was due to conntrack timeout; false when
+	// the entry was removed because it failed re-validation after a reload.
+	Expired bool
+}
+
+// RulesReloadEvent is emitted once after each successful firewall reload.
+// Reporters that bucket state by RulesVersion should close the old bucket
+// and open a new one on receipt.
+type RulesReloadEvent struct {
+	OldVersion uint16
+	NewVersion uint16
+}
+
+// Reporter is the embedder-supplied sink for firewall events. Implementations
+// that want a timestamp should call time.Now() themselves at the top of the
+// method; nebula does not provide one. See the package doc for the
+// do-not-take-address rule.
+type Reporter interface {
+	ReportDrop(DropEvent)
+	ReportFlowCreate(FlowCreateEvent)
+	ReportFlowEvict(FlowEvictEvent)
+	ReportRulesReload(RulesReloadEvent)
+}

firewall/packet.go

@@ -22,12 +22,36 @@ const (
 type Packet struct {
 	LocalAddr  netip.Addr
 	RemoteAddr netip.Addr
-	LocalPort  uint16
+	// LocalPort is the destination port for incoming traffic, or the source port for outgoing. Zero for ICMP.
+	LocalPort uint16
+	// RemotePort is the source port for incoming traffic, or the destination port for outgoing.
+	// For ICMP, it's the "identifier". This is only used for connection tracking, actual firewall rules will not filter on ICMP identifier
 	RemotePort uint16
 	Protocol   uint8
 	Fragment   bool
 }
 
+// PacketContext carries additional parsed details about a packet that are
+// useful for event reporting but deliberately kept out of Packet so Packet
+// can keep being used as a conntrack map key. Populated alongside Packet by
+// newPacket.
+//
+// Fields are interpreted based on Packet.Protocol:
+//   - ProtoTCP: TCPFlags is meaningful; ICMPType / ICMPCode are zero
+//   - ProtoICMP, ProtoICMPv6: ICMPType / ICMPCode are meaningful; TCPFlags is zero
+//   - ProtoUDP and others: only Length is meaningful
+type PacketContext struct {
+	// Length is the total IP packet length in bytes, including headers.
+	Length uint16
+	// TCPFlags is the flag byte from the TCP header (bits for FIN, SYN, RST,
+	// PSH, ACK, URG, ECE, CWR).
+	TCPFlags uint8
+	// ICMPType is the type field of the ICMP / ICMPv6 header.
+	ICMPType uint8
+	// ICMPCode is the code field of the ICMP / ICMPv6 header.
+	ICMPCode uint8
+}
+
 func (fp *Packet) Copy() *Packet {
 	return &Packet{
 		LocalAddr:  fp.LocalAddr,
@@ -46,6 +70,8 @@ func (fp Packet) MarshalJSON() ([]byte, error) {
 		proto = "tcp"
 	case ProtoICMP:
 		proto = "icmp"
+	case ProtoICMPv6:
+		proto = "icmpv6"
 	case ProtoUDP:
 		proto = "udp"
 	default:

firewall_events_test.go

@@ -0,0 +1,731 @@
+package nebula
+
+import (
+	"net"
+	"net/netip"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/gaissmai/bart"
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/firewall"
+	"github.com/slackhq/nebula/firewall/events"
+	"github.com/slackhq/nebula/test"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// recordingReporter captures every event fired against it. Its methods take
+// the conntrack lock implicitly (via the firewall code path that invokes
+// them), so we synchronize accumulator mutations with a small mutex to keep
+// the race detector happy across goroutines in case a test introduces any.
+type recordingReporter struct {
+	mu      sync.Mutex
+	drops   []recordedDrop
+	creates []recordedCreate
+	evicts  []recordedEvict
+	reloads []recordedReload
+}
+
+type recordedDrop struct {
+	incoming     bool
+	reason       events.DropReason
+	remote       netip.Addr
+	local        netip.Addr
+	peerName     string
+	rulesVersion uint16
+	ctx          firewall.PacketContext
+}
+
+type recordedCreate struct {
+	incoming     bool
+	remote       netip.Addr
+	local        netip.Addr
+	peerName     string
+	rulesVersion uint16
+	ctx          firewall.PacketContext
+}
+
+type recordedEvict struct {
+	incoming     bool
+	remote       netip.Addr
+	local        netip.Addr
+	rulesVersion uint16
+	expired      bool
+}
+
+type recordedReload struct {
+	oldVersion uint16
+	newVersion uint16
+}
+
+func (r *recordingReporter) ReportDrop(e events.DropEvent) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	name := ""
+	if e.PeerCert != nil && e.PeerCert.Certificate != nil {
+		name = e.PeerCert.Certificate.Name()
+	}
+	r.drops = append(r.drops, recordedDrop{
+		incoming:     e.Incoming,
+		reason:       e.Reason,
+		remote:       e.Packet.RemoteAddr,
+		local:        e.Packet.LocalAddr,
+		peerName:     name,
+		rulesVersion: e.RulesVersion,
+		ctx:          e.Context,
+	})
+}
+
+func (r *recordingReporter) ReportFlowCreate(e events.FlowCreateEvent) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	name := ""
+	if e.PeerCert != nil && e.PeerCert.Certificate != nil {
+		name = e.PeerCert.Certificate.Name()
+	}
+	r.creates = append(r.creates, recordedCreate{
+		incoming:     e.Incoming,
+		remote:       e.Packet.RemoteAddr,
+		local:        e.Packet.LocalAddr,
+		peerName:     name,
+		rulesVersion: e.RulesVersion,
+		ctx:          e.Context,
+	})
+}
+
+func (r *recordingReporter) ReportFlowEvict(e events.FlowEvictEvent) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.evicts = append(r.evicts, recordedEvict{
+		incoming:     e.Incoming,
+		remote:       e.Packet.RemoteAddr,
+		local:        e.Packet.LocalAddr,
+		rulesVersion: e.RulesVersion,
+		expired:      e.Expired,
+	})
+}
+
+func (r *recordingReporter) ReportRulesReload(e events.RulesReloadEvent) {
+	r.mu.Lock()
+	defer r.mu.Unlock()
+	r.reloads = append(r.reloads, recordedReload{
+		oldVersion: e.OldVersion,
+		newVersion: e.NewVersion,
+	})
+}
+
+// eventFixture builds a Firewall wired to a Control plus a packet/hostinfo
+// pair that a test can reuse. By default the ruleset allows the packet;
+// callers mutate fw / p / h as needed before invoking Drop.
+type eventFixture struct {
+	ctl *Control
+	fw  *Firewall
+	p   firewall.Packet
+	h   *HostInfo
+	cp  *cert.CAPool
+}
+
+func newEventFixture(t *testing.T) *eventFixture {
+	t.Helper()
+	l := test.NewLogger()
+
+	// myVpnNetworksTable covers our single peer address so buildNetworks takes
+	// the "simple case" path (h.networks stays nil); tests that want a populated
+	// BART table overwrite h.networks directly.
+	vpnNetworks := new(bart.Lite)
+	vpnNetworks.Insert(netip.MustParsePrefix("1.2.3.0/24"))
+
+	// Use the same cert for "peer" and "local" endpoints, matching the
+	// TestFirewall_Drop fixture style: LocalAddr == RemoteAddr == peer vpn addr.
+	c := &dummyCert{
+		name:     "host1",
+		networks: []netip.Prefix{netip.MustParsePrefix("1.2.3.4/24")},
+		groups:   []string{"default-group"},
+		issuer:   "signer-shasum",
+	}
+	h := &HostInfo{
+		ConnectionState: &ConnectionState{
+			peerCert: &cert.CachedCertificate{
+				Certificate:    c,
+				InvertedGroups: map[string]struct{}{"default-group": {}},
+			},
+		},
+		vpnAddrs: []netip.Addr{netip.MustParseAddr("1.2.3.4")},
+	}
+	h.buildNetworks(vpnNetworks, c)
+
+	fw := NewFirewall(l, time.Minute, time.Minute, time.Minute, c)
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
+	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
+
+	ctl := &Control{
+		f: &Interface{firewall: fw},
+		l: l,
+	}
+
+	return &eventFixture{
+		ctl: ctl,
+		fw:  fw,
+		p: firewall.Packet{
+			LocalAddr:  netip.MustParseAddr("1.2.3.4"),
+			RemoteAddr: netip.MustParseAddr("1.2.3.4"),
+			LocalPort:  10,
+			RemotePort: 90,
+			Protocol:   firewall.ProtoUDP,
+		},
+		h:  h,
+		cp: cert.NewCAPool(),
+	}
+}
+
+// firewall() returns the currently-installed firewall. Needed because
+// SetFirewallEventReporter replaces it via shallow-copy swap.
+func (f *eventFixture) firewall() *Firewall {
+	return f.ctl.f.firewall
+}
+
+func TestEvents_ReportDrop_InvalidRemoteIP(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// Packet to an address not in the cert's networks.
+	f.p.RemoteAddr = netip.MustParseAddr("9.9.9.9")
+	assert.Equal(t, ErrInvalidRemoteIP, f.firewall().Drop(f.p, firewall.PacketContext{}, false, f.h, f.cp, nil))
+
+	require.Len(t, r.drops, 1)
+	assert.Equal(t, events.DropInvalidRemoteIP, r.drops[0].reason)
+	assert.False(t, r.drops[0].incoming)
+	assert.Equal(t, "host1", r.drops[0].peerName)
+	assert.Empty(t, r.creates)
+	assert.Empty(t, r.evicts)
+}
+
+func TestEvents_ReportDrop_InvalidLocalIP(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// LocalAddr outside our routable networks.
+	f.p.LocalAddr = netip.MustParseAddr("9.9.9.9")
+	assert.Equal(t, ErrInvalidLocalIP, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+
+	require.Len(t, r.drops, 1)
+	assert.Equal(t, events.DropInvalidLocalIP, r.drops[0].reason)
+	assert.True(t, r.drops[0].incoming)
+}
+
+func TestEvents_ReportDrop_NoMatchingRule(t *testing.T) {
+	f := newEventFixture(t)
+	// Reset to a firewall with no matching rule.
+	l := test.NewLogger()
+	fw := NewFirewall(l, time.Minute, time.Minute, time.Minute, f.h.ConnectionState.peerCert.Certificate)
+	// Rule that won't match (group not in peer's groups).
+	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", ""))
+	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", ""))
+	f.ctl.f.firewall = fw
+
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	assert.Equal(t, ErrNoMatchingRule, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.drops, 1)
+	assert.Equal(t, events.DropNoMatchingRule, r.drops[0].reason)
+}
+
+func TestEvents_ReportDrop_PeerRejected(t *testing.T) {
+	f := newEventFixture(t)
+	// Re-classify the remote as VPNPeer so it triggers DropPeerRejected.
+	f.h.networks = new(bart.Table[NetworkType])
+	f.h.networks.Insert(netip.MustParsePrefix("1.2.3.0/24"), NetworkTypeVPNPeer)
+
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	assert.Equal(t, ErrPeerRejected, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.drops, 1)
+	assert.Equal(t, events.DropPeerRejected, r.drops[0].reason)
+}
+
+func TestEvents_ReportDrop_UnknownNetwork(t *testing.T) {
+	f := newEventFixture(t)
+	// Insert an unrecognized NetworkType value to hit the default branch.
+	f.h.networks = new(bart.Table[NetworkType])
+	f.h.networks.Insert(netip.MustParsePrefix("1.2.3.0/24"), NetworkTypeUnknown)
+
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	assert.Equal(t, ErrUnknownNetworkType, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.drops, 1)
+	assert.Equal(t, events.DropUnknownNetwork, r.drops[0].reason)
+}
+
+func TestEvents_ReportFlowCreate_OnceOnly(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// First allowed packet creates the conntrack entry.
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	// Second matching packet on the same tuple is short-circuited by conntrack
+	// and must not fire another FlowCreate.
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+
+	require.Len(t, r.creates, 1)
+	assert.True(t, r.creates[0].incoming)
+	assert.Equal(t, f.p.RemoteAddr, r.creates[0].remote)
+	assert.Empty(t, r.drops)
+}
+
+func TestEvents_ReportFlowEvict_OnReloadPurge(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// Create a flow under the current rules.
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.creates, 1)
+
+	// Simulate a reload that produces rules the existing flow no longer
+	// matches. Bump rulesVersion and replace InRules with an empty table so
+	// revalidation fails.
+	fw := f.firewall()
+	fw.Conntrack.Lock()
+	fw.rulesVersion++
+	fw.InRules = newFirewallTable()
+	fw.Conntrack.Unlock()
+
+	// Next packet triggers re-validation, which fails and evicts the entry.
+	err := fw.Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
+	assert.Equal(t, ErrNoMatchingRule, err)
+
+	require.Len(t, r.evicts, 1)
+	assert.False(t, r.evicts[0].expired, "evict from reload purge is not expiration")
+	assert.True(t, r.evicts[0].incoming)
+}
+
+func TestEvents_ReportFlowEvict_OnTimeout(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.creates, 1)
+
+	// Force expiration by rewinding the entry's deadline.
+	fw := f.firewall()
+	fw.Conntrack.Lock()
+	c := fw.Conntrack.Conns[f.p]
+	require.NotNil(t, c)
+	c.Expires = time.Now().Add(-time.Hour)
+	fw.evict(f.p)
+	fw.Conntrack.Unlock()
+
+	require.Len(t, r.evicts, 1)
+	assert.True(t, r.evicts[0].expired)
+}
+
+func TestEvents_SetNil_Clears(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.creates, 1)
+
+	f.ctl.SetFirewallEventReporter(nil)
+	resetConntrack(f.firewall())
+	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	// No second create should be recorded.
+	assert.Len(t, r.creates, 1)
+}
+
+func TestEvents_ReporterSurvivesSwap(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// Simulate a reload by swapping in a fresh Firewall that carries the
+	// reporter forward. Mirrors what reloadFirewall does with the shared
+	// conntrack pointer.
+	l := test.NewLogger()
+	oldFw := f.firewall()
+	newFw := NewFirewall(l, time.Minute, time.Minute, time.Minute, f.h.ConnectionState.peerCert.Certificate)
+	require.NoError(t, newFw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
+	require.NoError(t, newFw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
+	newFw.Conntrack = oldFw.Conntrack
+	newFw.rulesVersion = oldFw.rulesVersion + 1
+	newFw.reporter = oldFw.reporter
+	f.ctl.f.firewall = newFw
+	newFw.reportRulesReload(oldFw.rulesVersion, newFw.rulesVersion)
+
+	require.Len(t, r.reloads, 1)
+	assert.Equal(t, oldFw.rulesVersion, r.reloads[0].oldVersion)
+	assert.Equal(t, newFw.rulesVersion, r.reloads[0].newVersion)
+
+	// Events on the new firewall should still reach the same reporter.
+	require.NoError(t, newFw.Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+	require.Len(t, r.creates, 1)
+	assert.Equal(t, newFw.rulesVersion, r.creates[0].rulesVersion)
+}
+
+func TestEvents_InstallDoesNotMutateOldFirewall(t *testing.T) {
+	f := newEventFixture(t)
+	before := f.firewall()
+
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	after := f.firewall()
+	assert.NotSame(t, before, after, "SetFirewallEventReporter must replace the Firewall pointer")
+	assert.Nil(t, before.reporter, "the pre-install Firewall must remain untouched")
+	assert.NotNil(t, after.reporter)
+}
+
+// --- PacketContext parse tests --------------------------------------------
+
+func mustSerialize(t *testing.T, lrs ...gopacket.SerializableLayer) []byte {
+	t.Helper()
+	buf := gopacket.NewSerializeBuffer()
+	opt := gopacket.SerializeOptions{ComputeChecksums: false, FixLengths: true}
+	require.NoError(t, gopacket.SerializeLayers(buf, opt, lrs...))
+	return buf.Bytes()
+}
+
+func TestPacketContext_IPv4_TCPFlags(t *testing.T) {
+	ip := &layers.IPv4{
+		Version: 4, TTL: 64, Protocol: layers.IPProtocolTCP,
+		SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
+	}
+	tcp := &layers.TCP{SrcPort: 1234, DstPort: 80, SYN: true, ACK: true}
+	require.NoError(t, tcp.SetNetworkLayerForChecksum(ip))
+	data := mustSerialize(t, ip, tcp, gopacket.Payload([]byte("hello")))
+
+	var fp firewall.Packet
+	var ctx firewall.PacketContext
+	require.NoError(t, newPacket(data, true, &fp, &ctx))
+
+	assert.Equal(t, uint8(firewall.ProtoTCP), fp.Protocol)
+	// SYN (0x02) + ACK (0x10) = 0x12
+	assert.Equal(t, uint8(0x12), ctx.TCPFlags)
+	assert.Equal(t, uint16(len(data)), ctx.Length)
+	assert.Equal(t, uint8(0), ctx.ICMPType)
+	assert.Equal(t, uint8(0), ctx.ICMPCode)
+}
+
+func TestPacketContext_IPv4_ICMPTypeCode(t *testing.T) {
+	ip := &layers.IPv4{
+		Version: 4, TTL: 64, Protocol: layers.IPProtocolICMPv4,
+		SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
+	}
+	// Destination Unreachable, code 3 (port unreachable)
+	icmp := &layers.ICMPv4{
+		TypeCode: layers.CreateICMPv4TypeCode(layers.ICMPv4TypeDestinationUnreachable, layers.ICMPv4CodePort),
+	}
+	data := mustSerialize(t, ip, icmp, gopacket.Payload([]byte{0, 0, 0, 0}))
+
+	var fp firewall.Packet
+	var ctx firewall.PacketContext
+	require.NoError(t, newPacket(data, true, &fp, &ctx))
+
+	assert.Equal(t, uint8(firewall.ProtoICMP), fp.Protocol)
+	assert.Equal(t, uint8(layers.ICMPv4TypeDestinationUnreachable), ctx.ICMPType)
+	assert.Equal(t, uint8(layers.ICMPv4CodePort), ctx.ICMPCode)
+	assert.Equal(t, uint16(len(data)), ctx.Length)
+	assert.Equal(t, uint8(0), ctx.TCPFlags)
+}
+
+func TestPacketContext_IPv4_UDPLengthOnly(t *testing.T) {
+	ip := &layers.IPv4{
+		Version: 4, TTL: 64, Protocol: layers.IPProtocolUDP,
+		SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
+	}
+	udp := &layers.UDP{SrcPort: 1234, DstPort: 53}
+	require.NoError(t, udp.SetNetworkLayerForChecksum(ip))
+	data := mustSerialize(t, ip, udp, gopacket.Payload([]byte("query")))
+
+	var fp firewall.Packet
+	var ctx firewall.PacketContext
+	require.NoError(t, newPacket(data, true, &fp, &ctx))
+
+	assert.Equal(t, uint8(firewall.ProtoUDP), fp.Protocol)
+	assert.Equal(t, uint16(len(data)), ctx.Length)
+	assert.Zero(t, ctx.TCPFlags)
+	assert.Zero(t, ctx.ICMPType)
+	assert.Zero(t, ctx.ICMPCode)
+}
+
+func TestPacketContext_IPv6_TCPFlags(t *testing.T) {
+	ip := &layers.IPv6{
+		Version: 6, HopLimit: 64, NextHeader: layers.IPProtocolTCP,
+		SrcIP: net.ParseIP("fd00::1"), DstIP: net.ParseIP("fd00::2"),
+	}
+	tcp := &layers.TCP{SrcPort: 1234, DstPort: 443, FIN: true, ACK: true}
+	require.NoError(t, tcp.SetNetworkLayerForChecksum(ip))
+	data := mustSerialize(t, ip, tcp, gopacket.Payload([]byte("bye")))
+
+	var fp firewall.Packet
+	var ctx firewall.PacketContext
+	require.NoError(t, newPacket(data, true, &fp, &ctx))
+
+	assert.Equal(t, uint8(firewall.ProtoTCP), fp.Protocol)
+	// FIN (0x01) + ACK (0x10) = 0x11
+	assert.Equal(t, uint8(0x11), ctx.TCPFlags)
+	assert.Equal(t, uint16(len(data)), ctx.Length)
+}
+
+func TestPacketContext_IPv6_ICMPv6TypeCode(t *testing.T) {
+	ip := &layers.IPv6{
+		Version: 6, HopLimit: 64, NextHeader: layers.IPProtocolICMPv6,
+		SrcIP: net.ParseIP("fd00::1"), DstIP: net.ParseIP("fd00::2"),
+	}
+	icmp := &layers.ICMPv6{
+		TypeCode: layers.CreateICMPv6TypeCode(layers.ICMPv6TypeDestinationUnreachable, layers.ICMPv6CodePortUnreachable),
+	}
+	require.NoError(t, icmp.SetNetworkLayerForChecksum(ip))
+	data := mustSerialize(t, ip, icmp, gopacket.Payload([]byte{0, 0, 0, 0, 0, 0, 0, 0}))
+
+	var fp firewall.Packet
+	var ctx firewall.PacketContext
+	require.NoError(t, newPacket(data, true, &fp, &ctx))
+
+	assert.Equal(t, uint8(firewall.ProtoICMPv6), fp.Protocol)
+	assert.Equal(t, uint8(layers.ICMPv6TypeDestinationUnreachable), ctx.ICMPType)
+	assert.Equal(t, uint8(layers.ICMPv6CodePortUnreachable), ctx.ICMPCode)
+	assert.Equal(t, uint16(len(data)), ctx.Length)
+}
+
+// TestPacketContext_NilOK confirms a nil context pointer is accepted by
+// newPacket (the hot path may elect not to pass one).
+func TestPacketContext_NilOK(t *testing.T) {
+	ip := &layers.IPv4{
+		Version: 4, TTL: 64, Protocol: layers.IPProtocolUDP,
+		SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
+	}
+	udp := &layers.UDP{SrcPort: 1, DstPort: 2}
+	require.NoError(t, udp.SetNetworkLayerForChecksum(ip))
+	data := mustSerialize(t, ip, udp)
+
+	var fp firewall.Packet
+	require.NoError(t, newPacket(data, true, &fp, nil))
+}
+
+// TestPacketContext_FlowCreateCarriesContext exercises the full Drop -> addConn
+// -> ReportFlowCreate path with a realistic TCP packet and confirms the
+// context makes it into the reporter.
+func TestPacketContext_FlowCreateCarriesContext(t *testing.T) {
+	f := newEventFixture(t)
+	r := &recordingReporter{}
+	f.ctl.SetFirewallEventReporter(r)
+
+	// Hand-construct a matching TCP packet.
+	ctx := firewall.PacketContext{Length: 1500, TCPFlags: 0x12}
+	p := f.p
+	p.Protocol = firewall.ProtoTCP
+	require.NoError(t, f.firewall().Drop(p, ctx, true, f.h, f.cp, nil))
+
+	require.Len(t, r.creates, 1)
+	assert.Equal(t, uint16(1500), r.creates[0].ctx.Length)
+	assert.Equal(t, uint8(0x12), r.creates[0].ctx.TCPFlags)
+}
+
+// --- benchmarks ------------------------------------------------------------
+
+// noopReporter is the cheapest possible reporter. Methods discard the event.
+type noopReporter struct{}
+
+func (noopReporter) ReportDrop(events.DropEvent)             {}
+func (noopReporter) ReportFlowCreate(events.FlowCreateEvent) {}
+func (noopReporter) ReportFlowEvict(events.FlowEvictEvent)   {}
+func (noopReporter) ReportRulesReload(events.RulesReloadEvent) {
+}
+
+// bufferedReporter demonstrates a realistic zero-alloc reporter: each event
+// is forwarded to a value-typed channel. The channel send is a memcpy into
+// the channel's pre-allocated ring buffer -- no heap traffic. A background
+// goroutine would drain these; the bench skips draining to keep the report
+// path pure.
+type bufferedReporter struct {
+	drops  chan events.DropEvent
+	flows  chan events.FlowCreateEvent
+	evicts chan events.FlowEvictEvent
+}
+
+func newBufferedReporter(cap int) *bufferedReporter {
+	return &bufferedReporter{
+		drops:  make(chan events.DropEvent, cap),
+		flows:  make(chan events.FlowCreateEvent, cap),
+		evicts: make(chan events.FlowEvictEvent, cap),
+	}
+}
+
+func (r *bufferedReporter) ReportDrop(e events.DropEvent) {
+	select {
+	case r.drops <- e:
+	default:
+	}
+}
+
+func (r *bufferedReporter) ReportFlowCreate(e events.FlowCreateEvent) {
+	select {
+	case r.flows <- e:
+	default:
+	}
+}
+
+func (r *bufferedReporter) ReportFlowEvict(e events.FlowEvictEvent) {
+	select {
+	case r.evicts <- e:
+	default:
+	}
+}
+
+func (r *bufferedReporter) ReportRulesReload(events.RulesReloadEvent) {}
+
+// pointerReporter is the anti-pattern: it takes the address of the incoming
+// event struct, which forces the callee-side copy onto the heap. Kept for
+// comparison so we can see the alloc cost an unwary reporter would incur.
+type pointerReporter struct {
+	last *events.DropEvent
+}
+
+func (r *pointerReporter) ReportDrop(e events.DropEvent) {
+	r.last = &e
+}
+
+func (r *pointerReporter) ReportFlowCreate(events.FlowCreateEvent) {}
+func (r *pointerReporter) ReportFlowEvict(events.FlowEvictEvent)   {}
+func (r *pointerReporter) ReportRulesReload(events.RulesReloadEvent) {
+}
+
+func newBenchFixture(b *testing.B) *eventFixture {
+	b.Helper()
+	l := test.NewLogger()
+
+	vpnNetworks := new(bart.Lite)
+	vpnNetworks.Insert(netip.MustParsePrefix("1.2.3.0/24"))
+
+	c := &dummyCert{
+		name:     "host1",
+		networks: []netip.Prefix{netip.MustParsePrefix("1.2.3.4/24")},
+		groups:   []string{"default-group"},
+		issuer:   "signer-shasum",
+	}
+	h := &HostInfo{
+		ConnectionState: &ConnectionState{
+			peerCert: &cert.CachedCertificate{
+				Certificate:    c,
+				InvertedGroups: map[string]struct{}{"default-group": {}},
+			},
+		},
+		vpnAddrs: []netip.Addr{netip.MustParseAddr("1.2.3.4")},
+	}
+	h.buildNetworks(vpnNetworks, c)
+
+	fw := NewFirewall(l, time.Minute, time.Minute, time.Minute, c)
+	// Inbound rule that matches our packet; outbound has no match so we can
+	// also benchmark the no-rule drop path.
+	if err := fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""); err != nil {
+		b.Fatal(err)
+	}
+
+	ctl := &Control{f: &Interface{firewall: fw}, l: l}
+	return &eventFixture{
+		ctl: ctl,
+		fw:  fw,
+		p: firewall.Packet{
+			LocalAddr:  netip.MustParseAddr("1.2.3.4"),
+			RemoteAddr: netip.MustParseAddr("1.2.3.4"),
+			LocalPort:  10,
+			RemotePort: 90,
+			Protocol:   firewall.ProtoUDP,
+		},
+		h:  h,
+		cp: cert.NewCAPool(),
+	}
+}
+
+// BenchmarkFirewallDropPath measures the cost of Firewall.Drop on a packet
+// that reaches the no-matching-rule branch (the longest drop path). Compare
+// reporter shapes:
+//
+//	nilReporter      -- no reporter installed (feature cost when off)
+//	noopReporter     -- reporter installed, methods discard args (minimum on-cost)
+//	bufferedReporter -- realistic zero-alloc reporter: value-typed channels
+//	pointerReporter  -- anti-pattern that takes &composite-literal (allocates)
+func BenchmarkFirewallDropPath(b *testing.B) {
+	run := func(b *testing.B, install func(*Control)) {
+		f := newBenchFixture(b)
+		install(f.ctl)
+		b.ReportAllocs()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			_ = f.firewall().Drop(f.p, firewall.PacketContext{}, false, f.h, f.cp, nil)
+		}
+	}
+
+	b.Run("nilReporter", func(b *testing.B) { run(b, func(*Control) {}) })
+	b.Run("noopReporter", func(b *testing.B) {
+		run(b, func(c *Control) { c.SetFirewallEventReporter(noopReporter{}) })
+	})
+	b.Run("bufferedReporter", func(b *testing.B) {
+		run(b, func(c *Control) { c.SetFirewallEventReporter(newBufferedReporter(1024)) })
+	})
+	b.Run("pointerReporter", func(b *testing.B) {
+		run(b, func(c *Control) { c.SetFirewallEventReporter(&pointerReporter{}) })
+	})
+}
+
+// BenchmarkConntrackCreate measures Firewall.Drop for an allowed inbound
+// packet on a fresh conntrack (so addConn fires each iteration).
+func BenchmarkConntrackCreate(b *testing.B) {
+	run := func(b *testing.B, install func(*Control)) {
+		f := newBenchFixture(b)
+		install(f.ctl)
+		b.ReportAllocs()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			resetConntrack(f.firewall())
+			_ = f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
+		}
+	}
+	b.Run("nilReporter", func(b *testing.B) { run(b, func(*Control) {}) })
+	b.Run("noopReporter", func(b *testing.B) {
+		run(b, func(c *Control) { c.SetFirewallEventReporter(noopReporter{}) })
+	})
+	b.Run("bufferedReporter", func(b *testing.B) {
+		run(b, func(c *Control) { c.SetFirewallEventReporter(newBufferedReporter(1024)) })
+	})
+}
+
+// BenchmarkConntrackHit measures the hot path where a flow is already in
+// conntrack and short-circuits rule evaluation. The reporter slot is checked
+// only on create/evict, so this bench should show the reporter having zero
+// impact regardless of install state.
+func BenchmarkConntrackHit(b *testing.B) {
+	b.Run("nilReporter", func(b *testing.B) {
+		f := newBenchFixture(b)
+		// Prime conntrack.
+		require.NoError(b, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+		b.ReportAllocs()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			_ = f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
+		}
+	})
+
+	b.Run("noopReporter", func(b *testing.B) {
+		f := newBenchFixture(b)
+		f.ctl.SetFirewallEventReporter(noopReporter{})
+		require.NoError(b, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
+		b.ReportAllocs()
+		b.ResetTimer()
+		for i := 0; i < b.N; i++ {
+			_ = f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
+		}
+	})
+}

firewall_test.go

@@ -87,9 +87,10 @@ func TestFirewall_AddRule(t *testing.T) {
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoICMP, 1, 1, []string{}, "h1", "", "", "", ""))
-	assert.Nil(t, fw.InRules.ICMP[1].Any.Any)
-	assert.Empty(t, fw.InRules.ICMP[1].Any.Groups)
-	assert.Contains(t, fw.InRules.ICMP[1].Any.Hosts, "h1")
+	//no matter what port is given for icmp, it should end up as "any"
+	assert.Nil(t, fw.InRules.ICMP[firewall.PortAny].Any.Any)
+	assert.Empty(t, fw.InRules.ICMP[firewall.PortAny].Any.Groups)
+	assert.Contains(t, fw.InRules.ICMP[firewall.PortAny].Any.Hosts, "h1")
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
 	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", ti.String(), "", "", ""))
@@ -212,44 +213,44 @@ func TestFirewall_Drop(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// Drop outbound
-	assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil))
+	assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 	// Allow inbound
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 	// Allow outbound because conntrack
-	require.NoError(t, fw.Drop(p, false, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 
 	// test remote mismatch
 	oldRemote := p.RemoteAddr
 	p.RemoteAddr = netip.MustParseAddr("1.2.3.10")
-	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil), ErrInvalidRemoteIP)
 	p.RemoteAddr = oldRemote
 
 	// ensure signer doesn't get in the way of group checks
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum"))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad"))
-	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caSha doesn't drop on match
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad"))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum"))
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 
 	// ensure ca name doesn't get in the way of group checks
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", ""))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", ""))
-	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caName doesn't drop on match
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", ""))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", ""))
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 }
 
 func TestFirewall_DropV6(t *testing.T) {
@@ -291,44 +292,44 @@ func TestFirewall_DropV6(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// Drop outbound
-	assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, false, &h, cp, nil))
+	assert.Equal(t, ErrNoMatchingRule, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 	// Allow inbound
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 	// Allow outbound because conntrack
-	require.NoError(t, fw.Drop(p, false, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 
 	// test remote mismatch
 	oldRemote := p.RemoteAddr
 	p.RemoteAddr = netip.MustParseAddr("fd12::56")
-	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil), ErrInvalidRemoteIP)
 	p.RemoteAddr = oldRemote
 
 	// ensure signer doesn't get in the way of group checks
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum"))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum-bad"))
-	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caSha doesn't drop on match
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", "signer-shasum-bad"))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "", "signer-shasum"))
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 
 	// ensure ca name doesn't get in the way of group checks
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good", ""))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good-bad", ""))
-	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caName doesn't drop on match
 	cp.CAs["signer-shasum"] = &cert.CachedCertificate{Certificate: &dummyCert{name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "ca-good-bad", ""))
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", "", "", "ca-good", ""))
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 }
 
 func BenchmarkFirewallTable_match(b *testing.B) {
@@ -536,10 +537,10 @@ func TestFirewall_Drop2(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// h1/c1 lacks the proper groups
-	require.ErrorIs(t, fw.Drop(p, true, &h1, cp, nil), ErrNoMatchingRule)
+	require.ErrorIs(t, fw.Drop(p, firewall.PacketContext{}, true, &h1, cp, nil), ErrNoMatchingRule)
 	// c has the proper groups
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 }
 
 func TestFirewall_Drop3(t *testing.T) {
@@ -617,18 +618,18 @@ func TestFirewall_Drop3(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// c1 should pass because host match
-	require.NoError(t, fw.Drop(p, true, &h1, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h1, cp, nil))
 	// c2 should pass because ca sha match
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, true, &h2, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h2, cp, nil))
 	// c3 should fail because no match
 	resetConntrack(fw)
-	assert.Equal(t, fw.Drop(p, true, &h3, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h3, cp, nil), ErrNoMatchingRule)
 
 	// Test a remote address match
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "1.2.3.4/24", "", "", ""))
-	require.NoError(t, fw.Drop(p, true, &h1, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h1, cp, nil))
 }
 
 func TestFirewall_Drop3V6(t *testing.T) {
@@ -666,7 +667,7 @@ func TestFirewall_Drop3V6(t *testing.T) {
 	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
 	cp := cert.NewCAPool()
 	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 1, 1, []string{}, "", "fd12::34/120", "", "", ""))
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 }
 
 func TestFirewall_DropConntrackReload(t *testing.T) {
@@ -708,12 +709,12 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// Drop outbound
-	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
 	// Allow inbound
 	resetConntrack(fw)
-	require.NoError(t, fw.Drop(p, true, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, true, &h, cp, nil))
 	// Allow outbound because conntrack
-	require.NoError(t, fw.Drop(p, false, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 
 	oldFw := fw
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
@@ -722,7 +723,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	fw.rulesVersion = oldFw.rulesVersion + 1
 
 	// Allow outbound because conntrack and new rules allow port 10
-	require.NoError(t, fw.Drop(p, false, &h, cp, nil))
+	require.NoError(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil))
 
 	oldFw = fw
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
@@ -731,7 +732,151 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	fw.rulesVersion = oldFw.rulesVersion + 1
 
 	// Drop outbound because conntrack doesn't match new ruleset
-	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
+}
+
+func TestFirewall_ICMPPortBehavior(t *testing.T) {
+	l := test.NewLogger()
+	ob := &bytes.Buffer{}
+	l.SetOutput(ob)
+	myVpnNetworksTable := new(bart.Lite)
+	myVpnNetworksTable.Insert(netip.MustParsePrefix("1.1.1.1/8"))
+
+	network := netip.MustParsePrefix("1.2.3.4/24")
+
+	c := cert.CachedCertificate{
+		Certificate: &dummyCert{
+			name:     "host1",
+			networks: []netip.Prefix{network},
+			groups:   []string{"default-group"},
+			issuer:   "signer-shasum",
+		},
+		InvertedGroups: map[string]struct{}{"default-group": {}},
+	}
+	h := HostInfo{
+		ConnectionState: &ConnectionState{
+			peerCert: &c,
+		},
+		vpnAddrs: []netip.Addr{network.Addr()},
+	}
+	h.buildNetworks(myVpnNetworksTable, c.Certificate)
+
+	cp := cert.NewCAPool()
+
+	templ := firewall.Packet{
+		LocalAddr:  netip.MustParseAddr("1.2.3.4"),
+		RemoteAddr: netip.MustParseAddr("1.2.3.4"),
+		Protocol:   firewall.ProtoICMP,
+		Fragment:   false,
+	}
+
+	t.Run("ICMP allowed", func(t *testing.T) {
+		fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
+		require.NoError(t, fw.AddRule(true, firewall.ProtoICMP, 0, 0, []string{"any"}, "", "", "", "", ""))
+		t.Run("zero ports", func(t *testing.T) {
+			p := templ.Copy()
+			p.LocalPort = 0
+			p.RemotePort = 0
+			// Drop outbound
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
+			// Allow inbound
+			resetConntrack(fw)
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil))
+			//now also allow outbound
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil))
+		})
+
+		t.Run("nonzero ports", func(t *testing.T) {
+			p := templ.Copy()
+			p.LocalPort = 0xabcd
+			p.RemotePort = 0x1234
+			// Drop outbound
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
+			// Allow inbound
+			resetConntrack(fw)
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil))
+			//now also allow outbound
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil))
+		})
+	})
+
+	t.Run("Any proto, some ports allowed", func(t *testing.T) {
+		fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
+		require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 80, 444, []string{"any"}, "", "", "", "", ""))
+		t.Run("zero ports, still blocked", func(t *testing.T) {
+			p := templ.Copy()
+			p.LocalPort = 0
+			p.RemotePort = 0
+			// Drop outbound
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
+			// Allow inbound
+			resetConntrack(fw)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
+			//now also allow outbound
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
+		})
+
+		t.Run("nonzero ports, still blocked", func(t *testing.T) {
+			p := templ.Copy()
+			p.LocalPort = 0xabcd
+			p.RemotePort = 0x1234
+			// Drop outbound
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
+			// Allow inbound
+			resetConntrack(fw)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
+			//now also allow outbound
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
+		})
+
+		t.Run("nonzero, matching ports, still blocked", func(t *testing.T) {
+			p := templ.Copy()
+			p.LocalPort = 80
+			p.RemotePort = 80
+			// Drop outbound
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
+			// Allow inbound
+			resetConntrack(fw)
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil), ErrNoMatchingRule)
+			//now also allow outbound
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
+		})
+	})
+	t.Run("Any proto, any port", func(t *testing.T) {
+		fw := NewFirewall(l, time.Second, time.Minute, time.Hour, c.Certificate)
+		require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
+		t.Run("zero ports, allowed", func(t *testing.T) {
+			resetConntrack(fw)
+			p := templ.Copy()
+			p.LocalPort = 0
+			p.RemotePort = 0
+			// Drop outbound
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
+			// Allow inbound
+			resetConntrack(fw)
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil))
+			//now also allow outbound
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil))
+		})
+
+		t.Run("nonzero ports, allowed", func(t *testing.T) {
+			resetConntrack(fw)
+			p := templ.Copy()
+			p.LocalPort = 0xabcd
+			p.RemotePort = 0x1234
+			// Drop outbound
+			assert.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
+			// Allow inbound
+			resetConntrack(fw)
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, true, &h, cp, nil))
+			//now also allow outbound
+			require.NoError(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil))
+			//different ID is blocked
+			p.RemotePort++
+			require.Equal(t, fw.Drop(*p, firewall.PacketContext{}, false, &h, cp, nil), ErrNoMatchingRule)
+		})
+	})
+
 }
 
 func TestFirewall_DropIPSpoofing(t *testing.T) {
@@ -777,7 +922,7 @@ func TestFirewall_DropIPSpoofing(t *testing.T) {
 		Protocol:   firewall.ProtoUDP,
 		Fragment:   false,
 	}
-	assert.Equal(t, fw.Drop(p, true, &h1, cp, nil), ErrInvalidRemoteIP)
+	assert.Equal(t, fw.Drop(p, firewall.PacketContext{}, true, &h1, cp, nil), ErrInvalidRemoteIP)
 }
 
 func BenchmarkLookup(b *testing.B) {
@@ -919,11 +1064,11 @@ func TestNewFirewallFromConfig(t *testing.T) {
 
 	// Test code/port error
 	conf = config.NewC(l)
-	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"code": "a", "host": "testh"}}}
+	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"code": "a", "host": "testh", "proto": "any"}}}
 	_, err = NewFirewallFromConfig(l, cs, conf)
 	require.EqualError(t, err, "firewall.outbound rule #0; code was not a number; `a`")
 
-	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "a", "host": "testh"}}}
+	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "a", "host": "testh", "proto": "any"}}}
 	_, err = NewFirewallFromConfig(l, cs, conf)
 	require.EqualError(t, err, "firewall.outbound rule #0; port was not a number; `a`")
 
@@ -973,7 +1118,14 @@ func TestAddFirewallRulesFromConfig(t *testing.T) {
 	mf = &mockFirewall{}
 	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"port": "1", "proto": "icmp", "host": "a"}}}
 	require.NoError(t, AddFirewallRulesFromConfig(l, false, conf, mf))
-	assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoICMP, startPort: 1, endPort: 1, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
+	assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoICMP, startPort: firewall.PortAny, endPort: firewall.PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
+
+	// Test adding icmp rule no port
+	conf = config.NewC(l)
+	mf = &mockFirewall{}
+	conf.Settings["firewall"] = map[string]any{"outbound": []any{map[string]any{"proto": "icmp", "host": "a"}}}
+	require.NoError(t, AddFirewallRulesFromConfig(l, false, conf, mf))
+	assert.Equal(t, addRuleCall{incoming: false, proto: firewall.ProtoICMP, startPort: firewall.PortAny, endPort: firewall.PortAny, groups: nil, host: "a", ip: "", localIp: ""}, mf.lastCall)
 
 	// Test adding any rule
 	conf = config.NewC(l)
@@ -1184,7 +1336,7 @@ func (c *testcase) Test(t *testing.T, fw *Firewall) {
 	t.Helper()
 	cp := cert.NewCAPool()
 	resetConntrack(fw)
-	err := fw.Drop(c.p, true, c.h, cp, nil)
+	err := fw.Drop(c.p, firewall.PacketContext{}, true, c.h, cp, nil)
 	if c.err == nil {
 		require.NoError(t, err, "failed to not drop remote address %s", c.p.RemoteAddr)
 	} else {

go.mod

@@ -1,9 +1,10 @@
 module github.com/slackhq/nebula
 
-go 1.25
+go 1.25.0
 
 require (
 	dario.cat/mergo v1.0.2
+	filippo.io/bigmod v0.1.0
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
 	github.com/armon/go-radix v1.0.0
 	github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432
@@ -12,27 +13,27 @@ require (
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/gopacket v1.1.19
 	github.com/kardianos/service v1.2.4
-	github.com/miekg/dns v1.1.68
-	github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b
+	github.com/miekg/dns v1.1.72
+	github.com/miekg/pkcs11 v1.1.2
 	github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f
 	github.com/prometheus/client_golang v1.23.2
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
-	github.com/sirupsen/logrus v1.9.3
+	github.com/sirupsen/logrus v1.9.4
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
 	github.com/stretchr/testify v1.11.1
 	github.com/vishvananda/netlink v1.3.1
 	go.yaml.in/yaml/v3 v3.0.4
-	golang.org/x/crypto v0.45.0
+	golang.org/x/crypto v0.50.0
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
-	golang.org/x/net v0.47.0
-	golang.org/x/sync v0.19.0
-	golang.org/x/sys v0.39.0
-	golang.org/x/term v0.38.0
+	golang.org/x/net v0.52.0
+	golang.org/x/sync v0.20.0
+	golang.org/x/sys v0.43.0
+	golang.org/x/term v0.42.0
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
 	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
-	golang.zx2c4.com/wireguard/windows v0.5.3
-	google.golang.org/protobuf v1.36.10
+	golang.zx2c4.com/wireguard/windows v0.6.1
+	google.golang.org/protobuf v1.36.11
 	gopkg.in/yaml.v3 v3.0.1
 	gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
 )
@@ -49,7 +50,7 @@ require (
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/vishvananda/netns v0.0.5 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
-	golang.org/x/mod v0.24.0 // indirect
-	golang.org/x/time v0.7.0 // indirect
-	golang.org/x/tools v0.33.0 // indirect
+	golang.org/x/mod v0.34.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	golang.org/x/tools v0.43.0 // indirect
 )

go.sum

@@ -1,6 +1,8 @@
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
 dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
+filippo.io/bigmod v0.1.0 h1:UNzDk7y9ADKST+axd9skUpBQeW7fG2KrTZyOE4uGQy8=
+filippo.io/bigmod v0.1.0/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -83,10 +85,10 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/miekg/dns v1.1.68 h1:jsSRkNozw7G/mnmXULynzMNIsgY2dHC8LO6U6Ij2JEA=
-github.com/miekg/dns v1.1.68/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps=
-github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b h1:J/AzCvg5z0Hn1rqZUJjpbzALUmkKX0Zwbc/i4fw7Sfk=
-github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
+github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
+github.com/miekg/pkcs11 v1.1.2 h1:/VxmeAX5qU6Q3EwafypogwWbYryHFmF2RpkJmw3m4MQ=
+github.com/miekg/pkcs11 v1.1.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -131,8 +133,8 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
-github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
+github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
 github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 h1:pnnLyeX7o/5aX8qUQ69P/mLojDqwda8hFOCBTmP/6hw=
@@ -162,16 +164,16 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
-golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
+golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
-golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
+golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
+golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -182,8 +184,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
-golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
+golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -191,8 +193,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4=
-golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -206,26 +208,25 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
-golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
+golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.38.0 h1:PQ5pkm/rLO6HnxFR7N2lJHOZX6Kez5Y1gDSJla6jo7Q=
-golang.org/x/term v0.38.0/go.mod h1:bSEAKrOT1W+VSu9TSCMtoGEOUcKxOKgl3LE5QEF/xVg=
+golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
+golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.7.0 h1:ntUhktv3OPE6TgYxXWv9vKvUSJyIFJlyohwbkEwPrKQ=
-golang.org/x/time v0.7.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
-golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
+golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
+golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -234,8 +235,8 @@ golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeu
 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
 golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo=
 golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
-golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE=
-golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI=
+golang.zx2c4.com/wireguard/windows v0.6.1 h1:XMaKojH1Hs/raMrmnir4n35nTvzvWj7NmSYzHn2F4qU=
+golang.zx2c4.com/wireguard/windows v0.6.1/go.mod h1:04aqInu5GYuTFvMuDw/rKBAF7mHrltW/3rekpfbbZDM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -244,8 +245,8 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE=
-google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

handshake_ix.go

@@ -1,6 +1,7 @@
 package nebula
 
 import (
+	"bytes"
 	"net/netip"
 	"time"
 
@@ -166,6 +167,13 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 		return
 	}
 
+	if !bytes.Equal(remoteCert.Certificate.PublicKey(), ci.H.PeerStatic()) {
+		f.l.WithField("from", via).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+			WithField("cert", remoteCert).Info("public key mismatch between certificate and handshake")
+		return
+	}
+
 	if remoteCert.Certificate.Version() != ci.myCert.Version() {
 		// We started off using the wrong certificate version, lets see if we can match the version that was sent to us
 		myCertOtherVersion := cs.getCertificate(remoteCert.Certificate.Version())
@@ -535,6 +543,12 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 		e.Info("Invalid certificate from host")
 		return true
 	}
+	if !bytes.Equal(remoteCert.Certificate.PublicKey(), ci.H.PeerStatic()) {
+		f.l.WithField("from", via).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+			WithField("cert", remoteCert).Info("public key mismatch between certificate and handshake")
+		return true
+	}
 
 	if len(remoteCert.Certificate.Networks()) == 0 {
 		f.l.WithError(err).WithField("from", via).

handshake_manager.go

@@ -590,7 +590,7 @@ func (hm *HandshakeManager) allocateIndex(hh *HandshakeHostInfo) error {
 	hm.Lock()
 	defer hm.Unlock()
 
-	for i := 0; i < 32; i++ {
+	for range 32 {
 		index, err := generateIndex(hm.l)
 		if err != nil {
 			return err

hostmap.go

@@ -604,9 +604,9 @@ func (hm *HostMap) queryVpnAddr(vpnIp netip.Addr, promoteIfce *Interface) *HostI
 // unlockedAddHostInfo assumes you have a write-lock and will add a hostinfo object to the hostmap Indexes and RemoteIndexes maps.
 // If an entry exists for the Hosts table (vpnIp -> hostinfo) then the provided hostinfo will be made primary
 func (hm *HostMap) unlockedAddHostInfo(hostinfo *HostInfo, f *Interface) {
-	if f.serveDns {
+	if f.dnsServer != nil {
 		remoteCert := hostinfo.ConnectionState.peerCert
-		dnsR.Add(remoteCert.Certificate.Name()+".", hostinfo.vpnAddrs)
+		f.dnsServer.Add(remoteCert.Certificate.Name()+".", hostinfo.vpnAddrs)
 	}
 	for _, addr := range hostinfo.vpnAddrs {
 		hm.unlockedInnerAddHostInfo(addr, hostinfo, f)

hostmap_tester.go

@@ -1,5 +1,4 @@
 //go:build e2e_testing
-// +build e2e_testing
 
 package nebula
 

inside.go

@@ -2,19 +2,17 @@ package nebula
 
 import (
 	"net/netip"
-	"time"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/iputil"
 	"github.com/slackhq/nebula/noiseutil"
-	"github.com/slackhq/nebula/packet"
 	"github.com/slackhq/nebula/routing"
 )
 
-func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb []byte, out *packet.UDPPacket, q int, localCache firewall.ConntrackCache, now time.Time) {
-	err := newPacket(packet, false, fwPacket)
+func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, fwCtx *firewall.PacketContext, nb, out []byte, q int, localCache firewall.ConntrackCache) {
+	err := newPacket(packet, false, fwPacket, fwCtx)
 	if err != nil {
 		if f.l.Level >= logrus.DebugLevel {
 			f.l.WithField("packet", packet).Debugf("Error while validating outbound packet: %s", err)
@@ -55,7 +53,7 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 	})
 
 	if hostinfo == nil {
-		f.rejectInside(packet, out.Payload, q) //todo vector?
+		f.rejectInside(packet, out, q)
 		if f.l.Level >= logrus.DebugLevel {
 			f.l.WithField("vpnAddr", fwPacket.RemoteAddr).
 				WithField("fwPacket", fwPacket).
@@ -68,11 +66,12 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 		return
 	}
 
-	dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache, now)
+	dropReason := f.firewall.Drop(*fwPacket, *fwCtx, false, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason == nil {
-		f.sendNoMetricsDelayed(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
+		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
+
 	} else {
-		f.rejectInside(packet, out.Payload, q) //todo vector?
+		f.rejectInside(packet, out, q)
 		if f.l.Level >= logrus.DebugLevel {
 			hostinfo.logger(f.l).
 				WithField("fwPacket", fwPacket).
@@ -212,14 +211,15 @@ func (f *Interface) getOrHandshakeConsiderRouting(fwPacket *firewall.Packet, cac
 
 func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte) {
 	fp := &firewall.Packet{}
-	err := newPacket(p, false, fp)
+	ctx := &firewall.PacketContext{}
+	err := newPacket(p, false, fp, ctx)
 	if err != nil {
 		f.l.Warnf("error while parsing outgoing packet for firewall check; %v", err)
 		return
 	}
 
 	// check if packet is in outbound fw rules
-	dropReason := f.firewall.Drop(*fp, false, hostinfo, f.pki.GetCAPool(), nil, time.Now())
+	dropReason := f.firewall.Drop(*fp, *ctx, false, hostinfo, f.pki.GetCAPool(), nil)
 	if dropReason != nil {
 		if f.l.Level >= logrus.DebugLevel {
 			f.l.WithField("fwPacket", fp).
@@ -411,81 +411,3 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 		}
 	}
 }
-
-func (f *Interface) sendNoMetricsDelayed(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb []byte, out *packet.UDPPacket, q int) {
-	if ci.eKey == nil {
-		return
-	}
-	useRelay := !remote.IsValid() && !hostinfo.remote.IsValid()
-	fullOut := out.Payload
-
-	if useRelay {
-		if len(out.Payload) < header.Len {
-			// out always has a capacity of mtu, but not always a length greater than the header.Len.
-			// Grow it to make sure the next operation works.
-			out.Payload = out.Payload[:header.Len]
-		}
-		// Save a header's worth of data at the front of the 'out' buffer.
-		out.Payload = out.Payload[header.Len:]
-	}
-
-	if noiseutil.EncryptLockNeeded {
-		// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
-		ci.writeLock.Lock()
-	}
-	c := ci.messageCounter.Add(1)
-
-	//l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p)
-	out.Payload = header.Encode(out.Payload, header.Version, t, st, hostinfo.remoteIndexId, c)
-	f.connectionManager.Out(hostinfo)
-
-	// Query our LH if we haven't since the last time we've been rebound, this will cause the remote to punch against
-	// all our addrs and enable a faster roaming.
-	if t != header.CloseTunnel && hostinfo.lastRebindCount != f.rebindCount {
-		//NOTE: there is an update hole if a tunnel isn't used and exactly 256 rebinds occur before the tunnel is
-		// finally used again. This tunnel would eventually be torn down and recreated if this action didn't help.
-		f.lightHouse.QueryServer(hostinfo.vpnAddrs[0])
-		hostinfo.lastRebindCount = f.rebindCount
-		if f.l.Level >= logrus.DebugLevel {
-			f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).Debug("Lighthouse update triggered for punch due to rebind counter")
-		}
-	}
-
-	var err error
-	out.Payload, err = ci.eKey.EncryptDanger(out.Payload, out.Payload, p, c, nb)
-	if noiseutil.EncryptLockNeeded {
-		ci.writeLock.Unlock()
-	}
-	if err != nil {
-		hostinfo.logger(f.l).WithError(err).
-			WithField("udpAddr", remote).WithField("counter", c).
-			WithField("attemptedCounter", c).
-			Error("Failed to encrypt outgoing packet")
-		return
-	}
-
-	if remote.IsValid() {
-		err = f.writers[q].Prep(out, remote)
-		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", remote).Error("Failed to write outgoing packet")
-		}
-	} else if hostinfo.remote.IsValid() {
-		err = f.writers[q].Prep(out, hostinfo.remote)
-		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", remote).Error("Failed to write outgoing packet")
-		}
-	} else {
-		// Try to send via a relay
-		for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
-			relayHostInfo, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
-			if err != nil {
-				hostinfo.relayState.DeleteRelay(relayIP)
-				hostinfo.logger(f.l).WithField("relay", relayIP).WithError(err).Info("sendNoMetrics failed to find HostInfo")
-				continue
-			}
-			//todo vector!!
-			f.SendVia(relayHostInfo, relay, out.Payload, nb, fullOut[:header.Len+len(out.Payload)], true)
-			break
-		}
-	}
-}

inside_bsd.go

@@ -1,5 +1,4 @@
 //go:build darwin || dragonfly || freebsd || netbsd || openbsd
-// +build darwin dragonfly freebsd netbsd openbsd
 
 package nebula
 

inside_generic.go

@@ -1,5 +1,4 @@
 //go:build !darwin && !dragonfly && !freebsd && !netbsd && !openbsd
-// +build !darwin,!dragonfly,!freebsd,!netbsd,!openbsd
 
 package nebula
 

interface.go

@@ -4,9 +4,9 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"net/netip"
-	"os"
-	"runtime"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -17,12 +17,10 @@ import (
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/overlay"
-	"github.com/slackhq/nebula/packet"
 	"github.com/slackhq/nebula/udp"
 )
 
 const mtu = 9001
-const batch = 1024 //todo config!
 
 type InterfaceConfig struct {
 	HostMap            *HostMap
@@ -31,7 +29,7 @@ type InterfaceConfig struct {
 	pki                *PKI
 	Cipher             string
 	Firewall           *Firewall
-	ServeDns           bool
+	DnsServer          *dnsServer
 	HandshakeManager   *HandshakeManager
 	lightHouse         *LightHouse
 	connectionManager  *connectionManager
@@ -59,7 +57,7 @@ type Interface struct {
 	firewall              *Firewall
 	connectionManager     *connectionManager
 	handshakeManager      *HandshakeManager
-	serveDns              bool
+	dnsServer             *dnsServer
 	createTime            time.Time
 	lightHouse            *LightHouse
 	myBroadcastAddrsTable *bart.Lite
@@ -78,7 +76,8 @@ type Interface struct {
 	reQueryEvery    atomic.Uint32
 	reQueryWait     atomic.Int64
 
-	sendRecvErrorConfig sendRecvErrorConfig
+	sendRecvErrorConfig   recvErrorConfig
+	acceptRecvErrorConfig recvErrorConfig
 
 	// rebindCount is used to decide if an active tunnel should trigger a punch notification through a lighthouse
 	rebindCount int8
@@ -86,19 +85,21 @@ type Interface struct {
 
 	conntrackCacheTimeout time.Duration
 
+	ctx     context.Context
 	writers []udp.Conn
-	readers []overlay.TunDev
+	readers []io.ReadWriteCloser
+	wg      sync.WaitGroup
+
+	// fatalErr holds the first unexpected reader error that caused shutdown.
+	// nil means "no fatal error" (yet)
+	fatalErr atomic.Pointer[error]
+	// triggerShutdown is a function that will be run exactly once, when onFatal swaps something non-nil into fatalErr
+	triggerShutdown func()
 
 	metricHandshakes    metrics.Histogram
 	messageMetrics      *MessageMetrics
 	cachedPacketMetrics *cachedPacketMetrics
 
-	listenInN  int
-	listenOutN int
-
-	listenInMetric  metrics.Histogram
-	listenOutMetric metrics.Histogram
-
 	l *logrus.Logger
 }
 
@@ -117,34 +118,34 @@ type EncWriter interface {
 	GetCertState() *CertState
 }
 
-type sendRecvErrorConfig uint8
+type recvErrorConfig uint8
 
 const (
-	sendRecvErrorAlways sendRecvErrorConfig = iota
-	sendRecvErrorNever
-	sendRecvErrorPrivate
+	recvErrorAlways recvErrorConfig = iota
+	recvErrorNever
+	recvErrorPrivate
 )
 
-func (s sendRecvErrorConfig) ShouldSendRecvError(endpoint netip.AddrPort) bool {
+func (s recvErrorConfig) ShouldRecvError(endpoint netip.AddrPort) bool {
 	switch s {
-	case sendRecvErrorPrivate:
+	case recvErrorPrivate:
 		return endpoint.Addr().IsPrivate()
-	case sendRecvErrorAlways:
+	case recvErrorAlways:
 		return true
-	case sendRecvErrorNever:
+	case recvErrorNever:
 		return false
 	default:
-		panic(fmt.Errorf("invalid sendRecvErrorConfig value: %d", s))
+		panic(fmt.Errorf("invalid recvErrorConfig value: %d", s))
 	}
 }
 
-func (s sendRecvErrorConfig) String() string {
+func (s recvErrorConfig) String() string {
 	switch s {
-	case sendRecvErrorAlways:
+	case recvErrorAlways:
 		return "always"
-	case sendRecvErrorNever:
+	case recvErrorNever:
 		return "never"
-	case sendRecvErrorPrivate:
+	case recvErrorPrivate:
 		return "private"
 	default:
 		return fmt.Sprintf("invalid(%d)", s)
@@ -170,12 +171,13 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 
 	cs := c.pki.getCertState()
 	ifce := &Interface{
+		ctx:                   ctx,
 		pki:                   c.pki,
 		hostMap:               c.HostMap,
 		outside:               c.Outside,
 		inside:                c.Inside,
 		firewall:              c.Firewall,
-		serveDns:              c.ServeDns,
+		dnsServer:             c.DnsServer,
 		handshakeManager:      c.HandshakeManager,
 		createTime:            time.Now(),
 		lightHouse:            c.lightHouse,
@@ -184,7 +186,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 		routines:              c.routines,
 		version:               c.version,
 		writers:               make([]udp.Conn, c.routines),
-		readers:               make([]overlay.TunDev, c.routines),
+		readers:               make([]io.ReadWriteCloser, c.routines),
 		myVpnNetworks:         cs.myVpnNetworks,
 		myVpnNetworksTable:    cs.myVpnNetworksTable,
 		myVpnAddrs:            cs.myVpnAddrs,
@@ -203,8 +205,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 
 		l: c.l,
 	}
-	ifce.listenInMetric = metrics.GetOrRegisterHistogram("vhost.listenIn.n", nil, metrics.NewExpDecaySample(1028, 0.015))
-	ifce.listenOutMetric = metrics.GetOrRegisterHistogram("vhost.listenOut.n", nil, metrics.NewExpDecaySample(1028, 0.015))
 
 	ifce.tryPromoteEvery.Store(c.tryPromoteEvery)
 	ifce.reQueryEvery.Store(c.reQueryEvery)
@@ -218,7 +218,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 // activate creates the interface on the host. After the interface is created, any
 // other services that want to bind listeners to its IP may do so successfully. However,
 // the interface isn't going to process anything until run() is called.
-func (f *Interface) activate() {
+func (f *Interface) activate() error {
 	// actually turn on tun dev
 
 	addr, err := f.outside.LocalAddr()
@@ -241,141 +241,119 @@ func (f *Interface) activate() {
 	metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
 
 	// Prepare n tun queues
-	var reader overlay.TunDev = f.inside
+	var reader io.ReadWriteCloser = f.inside
 	for i := 0; i < f.routines; i++ {
 		if i > 0 {
 			reader, err = f.inside.NewMultiQueueReader()
 			if err != nil {
-				f.l.Fatal(err)
+				return err
 			}
 		}
 		f.readers[i] = reader
 	}
 
-	if err := f.inside.Activate(); err != nil {
+	f.wg.Add(1) // for us to wait on Close() to return
+	if err = f.inside.Activate(); err != nil {
+		f.wg.Done()
 		f.inside.Close()
-		f.l.Fatal(err)
+		return err
 	}
+
+	return nil
 }
 
-func (f *Interface) run() {
+func (f *Interface) run() (func() error, error) {
 	// Launch n queues to read packets from udp
 	for i := 0; i < f.routines; i++ {
-		go f.listenOut(i)
+		f.wg.Go(func() {
+			f.listenOut(i)
+		})
 	}
 
 	// Launch n queues to read packets from tun dev
 	for i := 0; i < f.routines; i++ {
-		go f.listenIn(f.readers[i], i)
+		f.wg.Go(func() {
+			f.listenIn(f.readers[i], i)
+		})
+	}
+
+	return func() error {
+		f.wg.Wait()
+		if e := f.fatalErr.Load(); e != nil {
+			return *e
+		}
+		return nil
+	}, nil
+}
+
+// onFatal stores the first fatal reader error, and calls triggerShutdown if it was the first one
+func (f *Interface) onFatal(err error) {
+	swapped := f.fatalErr.CompareAndSwap(nil, &err)
+	if !swapped {
+		return
+	}
+	if f.triggerShutdown != nil {
+		f.triggerShutdown()
 	}
 }
 
-type Scratches struct {
-	h        *header.H
-	nb       []byte
-	fwPacket *firewall.Packet
-	scratch  []byte
-}
-
-func NewScratches() *Scratches {
-	return &Scratches{
-		h:        &header.H{},
-		fwPacket: &firewall.Packet{},
-		nb:       make([]byte, 12),
-		scratch:  make([]byte, udp.MTU),
-	}
-}
-
-func (f *Interface) listenOut(q int) {
-	runtime.LockOSThread()
-
+func (f *Interface) listenOut(i int) {
 	var li udp.Conn
-	if q > 0 {
-		li = f.writers[q]
+	if i > 0 {
+		li = f.writers[i]
 	} else {
 		li = f.outside
 	}
 
-	ctCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
+	ctCache := firewall.NewConntrackCacheTicker(f.ctx, f.conntrackCacheTimeout)
 	lhh := f.lightHouse.NewRequestHandler()
-
-	outPackets := make([]*packet.OutPacket, batch)
-	for i := 0; i < batch; i++ {
-		outPackets[i] = packet.NewOut()
-	}
-
-	scratches := NewScratches()
-
-	toSend := make([][]byte, batch)
-
-	li.ListenOut(func(pkts []*packet.UDPPacket) {
-		toSend = toSend[:0]
-
-		f.readOutsidePacketsMany(pkts, outPackets, lhh, scratches, q, ctCache.Get(f.l), time.Now())
-		//we opportunistically tx, but try to also send stragglers
-		if _, err := f.readers[q].WriteMany(outPackets, q); err != nil {
-			f.l.WithError(err).Error("Failed to send packets")
-		}
-		//todo I broke this
-		//n := len(toSend)
-		//if f.l.Level == logrus.DebugLevel {
-		//	f.listenOutMetric.Update(int64(n))
-		//}
-		//f.listenOutN = n
-
-	})
-}
-
-func (f *Interface) listenIn(reader overlay.TunDev, queueNum int) {
-	runtime.LockOSThread()
-
+	plaintext := make([]byte, udp.MTU)
+	h := &header.H{}
 	fwPacket := &firewall.Packet{}
+	fwCtx := &firewall.PacketContext{}
 	nb := make([]byte, 12, 12)
 
-	conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
+	err := li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) {
+		f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, plaintext[:0], payload, h, fwPacket, fwCtx, lhh, nb, i, ctCache.Get(f.l))
+	})
 
-	packets := reader.NewPacketArrays(batch)
-
-	outPackets := make([]*packet.UDPPacket, batch)
-	for i := 0; i < batch; i++ {
-		outPackets[i] = packet.New(false) //todo isv4?
+	if err != nil && !f.closed.Load() {
+		f.l.WithError(err).Error("Error while reading inbound packet, closing")
+		f.onFatal(err)
 	}
 
+	f.l.Debugf("underlay reader %v is done", i)
+}
+
+func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
+	packet := make([]byte, mtu)
+	out := make([]byte, mtu)
+	fwPacket := &firewall.Packet{}
+	fwCtx := &firewall.PacketContext{}
+	nb := make([]byte, 12, 12)
+
+	conntrackCache := firewall.NewConntrackCacheTicker(f.ctx, f.conntrackCacheTimeout)
+
 	for {
-		n, err := reader.ReadMany(packets, queueNum)
-
-		//todo!!
+		n, err := reader.Read(packet)
 		if err != nil {
-			if errors.Is(err, os.ErrClosed) && f.closed.Load() {
-				return
+			if !f.closed.Load() {
+				f.l.WithError(err).WithField("reader", i).Error("Error while reading outbound packet, closing")
+				f.onFatal(err)
 			}
-
-			f.l.WithError(err).Error("Error while reading outbound packet")
-			// This only seems to happen when something fatal happens to the fd, so exit.
-			os.Exit(2)
+			break
 		}
 
-		if f.l.Level == logrus.DebugLevel {
-			f.listenInMetric.Update(int64(n))
-		}
-		f.listenInN = n
-
-		now := time.Now()
-		for i, pkt := range packets[:n] {
-			outPackets[i].ReadyToSend = false
-			f.consumeInsidePacket(pkt.GetPayload(), fwPacket, nb, outPackets[i], queueNum, conntrackCache.Get(f.l), now)
-			reader.RecycleRxSeg(pkt, i == (n-1), queueNum) //todo handle err?
-		}
-		_, err = f.writers[queueNum].WriteBatch(outPackets[:n])
-		if err != nil {
-			f.l.WithError(err).Error("Error while writing outbound packets")
-		}
+		f.consumeInsidePacket(packet[:n], fwPacket, fwCtx, nb, out, i, conntrackCache.Get(f.l))
 	}
+
+	f.l.Debugf("overlay reader %v is done", i)
 }
 
 func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
 	c.RegisterReloadCallback(f.reloadFirewall)
 	c.RegisterReloadCallback(f.reloadSendRecvError)
+	c.RegisterReloadCallback(f.reloadAcceptRecvError)
 	c.RegisterReloadCallback(f.reloadDisconnectInvalid)
 	c.RegisterReloadCallback(f.reloadMisc)
 
@@ -424,8 +402,15 @@ func (f *Interface) reloadFirewall(c *config.C) {
 		fw.Conntrack = conntrack
 	}
 
+	fw.reporter = oldFw.reporter
+
 	f.firewall = fw
 
+	// Fire ReportRulesReload under the conntrack lock so the reporter cannot
+	// observe a FlowCreate/FlowEvict for the new rulesVersion before it
+	// observes the reload marker. Report* must be non-blocking.
+	fw.reportRulesReload(oldFw.rulesVersion, fw.rulesVersion)
+
 	oldFw.Destroy()
 	f.l.WithField("firewallHashes", fw.GetRuleHashes()).
 		WithField("oldFirewallHashes", oldFw.GetRuleHashes()).
@@ -439,16 +424,16 @@ func (f *Interface) reloadSendRecvError(c *config.C) {
 
 		switch stringValue {
 		case "always":
-			f.sendRecvErrorConfig = sendRecvErrorAlways
+			f.sendRecvErrorConfig = recvErrorAlways
 		case "never":
-			f.sendRecvErrorConfig = sendRecvErrorNever
+			f.sendRecvErrorConfig = recvErrorNever
 		case "private":
-			f.sendRecvErrorConfig = sendRecvErrorPrivate
+			f.sendRecvErrorConfig = recvErrorPrivate
 		default:
 			if c.GetBool("listen.send_recv_error", true) {
-				f.sendRecvErrorConfig = sendRecvErrorAlways
+				f.sendRecvErrorConfig = recvErrorAlways
 			} else {
-				f.sendRecvErrorConfig = sendRecvErrorNever
+				f.sendRecvErrorConfig = recvErrorNever
 			}
 		}
 
@@ -457,6 +442,30 @@ func (f *Interface) reloadSendRecvError(c *config.C) {
 	}
 }
 
+func (f *Interface) reloadAcceptRecvError(c *config.C) {
+	if c.InitialLoad() || c.HasChanged("listen.accept_recv_error") {
+		stringValue := c.GetString("listen.accept_recv_error", "always")
+
+		switch stringValue {
+		case "always":
+			f.acceptRecvErrorConfig = recvErrorAlways
+		case "never":
+			f.acceptRecvErrorConfig = recvErrorNever
+		case "private":
+			f.acceptRecvErrorConfig = recvErrorPrivate
+		default:
+			if c.GetBool("listen.accept_recv_error", true) {
+				f.acceptRecvErrorConfig = recvErrorAlways
+			} else {
+				f.acceptRecvErrorConfig = recvErrorNever
+			}
+		}
+
+		f.l.WithField("acceptRecvError", f.acceptRecvErrorConfig.String()).
+			Info("Loaded accept_recv_error config")
+	}
+}
+
 func (f *Interface) reloadMisc(c *config.C) {
 	if c.HasChanged("counters.try_promote") {
 		n := c.GetUint32("counters.try_promote", defaultPromoteEvery)
@@ -507,11 +516,6 @@ func (f *Interface) emitStats(ctx context.Context, i time.Duration) {
 			} else {
 				certMaxVersion.Update(int64(certState.v1Cert.Version()))
 			}
-			if f.l.Level != logrus.DebugLevel {
-				f.listenInMetric.Update(int64(f.listenInN))
-				f.listenOutMetric.Update(int64(f.listenOutN))
-			}
-
 		}
 	}
 }
@@ -525,15 +529,23 @@ func (f *Interface) GetCertState() *CertState {
 }
 
 func (f *Interface) Close() error {
+	var errs []error
 	f.closed.Store(true)
 
-	for _, u := range f.writers {
+	// Release the udp readers
+	for i, u := range f.writers {
 		err := u.Close()
 		if err != nil {
-			f.l.WithError(err).Error("Error while closing udp socket")
+			f.l.WithError(err).WithField("writer", i).Error("Error while closing udp socket")
+			errs = append(errs, err)
 		}
 	}
 
-	// Release the tun device
-	return f.inside.Close()
+	// Release the tun device (closing the tun also closes all readers)
+	closeErr := f.inside.Close()
+	if closeErr != nil {
+		errs = append(errs, closeErr)
+	}
+	f.wg.Done()
+	return errors.Join(errs...)
 }

lighthouse.go

@@ -713,21 +713,14 @@ func (lh *LightHouse) unlockedShouldAddV6(vpnAddr netip.Addr, to *V6AddrPort) bo
 
 func (lh *LightHouse) IsLighthouseAddr(vpnAddr netip.Addr) bool {
 	l := lh.GetLighthouses()
-	for i := range l {
-		if l[i] == vpnAddr {
-			return true
-		}
-	}
-	return false
+	return slices.Contains(l, vpnAddr)
 }
 
 func (lh *LightHouse) IsAnyLighthouseAddr(vpnAddrs []netip.Addr) bool {
 	l := lh.GetLighthouses()
 	for i := range vpnAddrs {
-		for j := range l {
-			if l[j] == vpnAddrs[i] {
-				return true
-			}
+		if slices.Contains(l, vpnAddrs[i]) {
+			return true
 		}
 	}
 	return false

lighthouse_test.go

@@ -1,7 +1,6 @@
 package nebula
 
 import (
-	"context"
 	"encoding/binary"
 	"fmt"
 	"net/netip"
@@ -42,14 +41,14 @@ func Test_lhStaticMapping(t *testing.T) {
 	c := config.NewC(l)
 	c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1}}
 	c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}}
-	_, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
+	_, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
 	require.NoError(t, err)
 
 	lh2 := "10.128.0.3"
 	c = config.NewC(l)
 	c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1, lh2}}
 	c.Settings["static_host_map"] = map[string]any{lh1: []any{"100.1.1.1:4242"}}
-	_, err = NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
+	_, err = NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
 	require.EqualError(t, err, "lighthouse 10.128.0.3 does not have a static_host_map entry")
 }
 
@@ -71,7 +70,7 @@ func TestReloadLighthouseInterval(t *testing.T) {
 	}
 
 	c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}}
-	lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
+	lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
 	require.NoError(t, err)
 	lh.ifce = &mockEncWriter{}
 
@@ -99,7 +98,7 @@ func BenchmarkLighthouseHandleRequest(b *testing.B) {
 	}
 
 	c := config.NewC(l)
-	lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
+	lh, err := NewLightHouseFromConfig(b.Context(), l, c, cs, nil, nil)
 	require.NoError(b, err)
 
 	hAddr := netip.MustParseAddrPort("4.5.6.7:12345")
@@ -202,7 +201,7 @@ func TestLighthouse_Memory(t *testing.T) {
 		myVpnNetworks:      []netip.Prefix{myVpnNet},
 		myVpnNetworksTable: nt,
 	}
-	lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
+	lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
 	lh.ifce = &mockEncWriter{}
 	require.NoError(t, err)
 	lhh := lh.NewRequestHandler()
@@ -288,7 +287,7 @@ func TestLighthouse_reload(t *testing.T) {
 		myVpnNetworksTable: nt,
 	}
 
-	lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
+	lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
 	require.NoError(t, err)
 
 	nc := map[string]any{
@@ -523,7 +522,7 @@ func TestLighthouse_Dont_Delete_Static_Hosts(t *testing.T) {
 		myVpnNetworks:      []netip.Prefix{myVpnNet},
 		myVpnNetworksTable: nt,
 	}
-	lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
+	lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
 	require.NoError(t, err)
 	lh.ifce = &mockEncWriter{}
 
@@ -589,7 +588,7 @@ func TestLighthouse_DeletesWork(t *testing.T) {
 		myVpnNetworks:      []netip.Prefix{myVpnNet},
 		myVpnNetworksTable: nt,
 	}
-	lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
+	lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
 	require.NoError(t, err)
 	lh.ifce = &mockEncWriter{}
 

main.go

@@ -105,11 +105,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		// deprecated and undocumented
 		tunQueues := c.GetInt("tun.routines", 1)
 		udpQueues := c.GetInt("listen.routines", 1)
-		if tunQueues > udpQueues {
-			routines = tunQueues
-		} else {
-			routines = udpQueues
-		}
+		routines = max(tunQueues, udpQueues)
 		if routines != 1 {
 			l.WithField("routines", routines).Warn("Setting tun.routines and listen.routines is deprecated. Use `routines` instead")
 		}
@@ -219,13 +215,9 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	handshakeManager := NewHandshakeManager(l, hostMap, lightHouse, udpConns[0], handshakeConfig)
 	lightHouse.handshakeTrigger = handshakeManager.trigger
 
-	serveDns := false
-	if c.GetBool("lighthouse.serve_dns", false) {
-		if c.GetBool("lighthouse.am_lighthouse", false) {
-			serveDns = true
-		} else {
-			l.Warn("DNS server refusing to run because this host is not a lighthouse.")
-		}
+	ds, err := newDnsServerFromConfig(ctx, l, pki.getCertState(), hostMap, c)
+	if err != nil {
+		l.WithError(err).Warn("Failed to start DNS responder")
 	}
 
 	ifConfig := &InterfaceConfig{
@@ -234,7 +226,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		Outside:               udpConns[0],
 		pki:                   pki,
 		Firewall:              fw,
-		ServeDns:              serveDns,
+		DnsServer:             ds,
 		HandshakeManager:      handshakeManager,
 		connectionManager:     connManager,
 		lightHouse:            lightHouse,
@@ -265,6 +257,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		ifce.RegisterConfigChangeCallbacks(c)
 		ifce.reloadDisconnectInvalid(c)
 		ifce.reloadSendRecvError(c)
+		ifce.reloadAcceptRecvError(c)
 
 		handshakeManager.f = ifce
 		go handshakeManager.Run(ctx)
@@ -283,23 +276,17 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 
 	attachCommands(l, c, ssh, ifce)
 
-	// Start DNS server last to allow using the nebula IP as lighthouse.dns.host
-	var dnsStart func()
-	if lightHouse.amLighthouse && serveDns {
-		l.Debugln("Starting dns server")
-		dnsStart = dnsMain(l, pki.getCertState(), hostMap, c)
-	}
-
 	return &Control{
-		ifce,
-		l,
-		ctx,
-		cancel,
-		sshStart,
-		statsStart,
-		dnsStart,
-		lightHouse.StartUpdateWorker,
-		connManager.Start,
+		state:                  StateReady,
+		f:                      ifce,
+		l:                      l,
+		ctx:                    ctx,
+		cancel:                 cancel,
+		sshStart:               sshStart,
+		statsStart:             statsStart,
+		dnsStart:               ds.Start,
+		lighthouseStart:        lightHouse.StartUpdateWorker,
+		connectionManagerStart: connManager.Start,
 	}, nil
 }
 

noiseutil/boring.go

@@ -22,7 +22,7 @@ const EncryptLockNeeded = true
 // NewGCMTLS is no longer exposed in go1.19+, so we need to link it in
 // See: https://github.com/golang/go/issues/56326
 //
-// NewGCMTLS is the internal method used with boringcrypto that provices a
+// NewGCMTLS is the internal method used with boringcrypto that provides a
 // validated mode of AES-GCM which enforces the nonce is strictly
 // monotonically increasing.  This is the TLS 1.2 specification for nonce
 // generation (which also matches the method used by the Noise Protocol)

notboring.go

@@ -1,5 +1,4 @@
 //go:build !boringcrypto
-// +build !boringcrypto
 
 package nebula
 

outside.go

@@ -7,7 +7,6 @@ import (
 	"time"
 
 	"github.com/google/gopacket/layers"
-	"github.com/slackhq/nebula/packet"
 	"golang.org/x/net/ipv6"
 
 	"github.com/sirupsen/logrus"
@@ -20,108 +19,30 @@ const (
 	minFwPacketLen = 4
 )
 
-// handleRelayPackets handles relay packets. Returns false if there's nothing left to do, true for continuing to process an unwrapped TerminalType packet
-// scratch must be large enough to contain a packet to be relayed if needed
-func (f *Interface) handleRelayPackets(via ViaSender, hostinfo *HostInfo, segment []byte, scratch []byte, h *header.H, nb []byte) ([]byte, *ViaSender, bool) {
-	var err error
-	// The entire body is sent as AD, not encrypted.
-	// The packet consists of a 16-byte parsed Nebula header, Associated Data-protected payload, and a trailing 16-byte AEAD signature value.
-	// The packet is guaranteed to be at least 16 bytes at this point, b/c it got past the h.Parse() call above. If it's
-	// otherwise malformed (meaning, there is no trailing 16 byte AEAD value), then this will result in at worst a 0-length slice
-	// which will gracefully fail in the DecryptDanger call.
-	signedPayload := segment[:len(segment)-hostinfo.ConnectionState.dKey.Overhead()]
-	signatureValue := segment[len(segment)-hostinfo.ConnectionState.dKey.Overhead():]
-	scratch, err = hostinfo.ConnectionState.dKey.DecryptDanger(scratch, signedPayload, signatureValue, h.MessageCounter, nb)
+func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, fwCtx *firewall.PacketContext, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
+	err := h.Parse(packet)
 	if err != nil {
-		return nil, nil, false
-	}
-	// Successfully validated the thing. Get rid of the Relay header.
-	signedPayload = signedPayload[header.Len:]
-	// Pull the Roaming parts up here, and return in all call paths.
-	f.handleHostRoaming(hostinfo, via)
-	// Track usage of both the HostInfo and the Relay for the received & authenticated packet
-	f.connectionManager.In(hostinfo)
-	f.connectionManager.RelayUsed(h.RemoteIndex)
-
-	relay, ok := hostinfo.relayState.QueryRelayForByIdx(h.RemoteIndex)
-	if !ok {
-		// The only way this happens is if hostmap has an index to the correct HostInfo, but the HostInfo is missing
-		// its internal mapping. This should never happen.
-		hostinfo.logger(f.l).WithFields(logrus.Fields{"vpnAddrs": hostinfo.vpnAddrs, "remoteIndex": h.RemoteIndex}).Error("HostInfo missing remote relay index")
-		return nil, nil, false
-	}
-
-	switch relay.Type {
-	case TerminalType:
-		// If I am the target of this relay, process the unwrapped packet
-		// We need to re-write our variables to ensure this segment is correctly parsed.
-		// We could set up for a recursive call here, but this makes it easier to prove that we'll never stack-overflow
-
-		//mirrors the top of readOutsideSegment
-		err = h.Parse(signedPayload)
-		if err != nil {
-			// Hole punch packets are 0 or 1 byte big, so let's ignore printing those errors
-			if len(signedPayload) > 1 {
-				f.l.WithField("packet", segment).Infof("Error while parsing inbound packet from %s: %s", via, err)
-			}
-			return nil, nil, false
-		}
-		newVia := &ViaSender{
-			UdpAddr:   via.UdpAddr,
-			relayHI:   hostinfo,
-			remoteIdx: relay.RemoteIndex,
-			relay:     relay,
-			IsRelayed: true,
-		}
-		//continue flowing through readOutsideSegment()
-		return signedPayload, newVia, true
-	case ForwardingType:
-		// Find the target HostInfo relay object
-		targetHI, targetRelay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relay.PeerAddr)
-		if err != nil {
-			hostinfo.logger(f.l).WithField("relayTo", relay.PeerAddr).WithError(err).WithField("hostinfo.vpnAddrs", hostinfo.vpnAddrs).Info("Failed to find target host info by ip")
-			return nil, nil, false
-		}
-
-		// If that relay is Established, forward the payload through it
-		if targetRelay.State == Established {
-			switch targetRelay.Type {
-			case ForwardingType:
-				// Forward this packet through the relay tunnel, and find the target HostInfo
-				f.SendVia(targetHI, targetRelay, signedPayload, nb, scratch[:0], false) //todo it would be nice to queue this up and do it later, or at least avoid a memcpy of signedPayload
-			case TerminalType:
-				hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
-			default:
-				hostinfo.logger(f.l).WithField("targetRelay.Type", targetRelay.Type).Error("Unexpected Relay Type")
-			}
-		} else {
-			hostinfo.logger(f.l).WithFields(logrus.Fields{"relayTo": relay.PeerAddr, "relayFrom": hostinfo.vpnAddrs[0], "targetRelayState": targetRelay.State}).Info("Unexpected target relay state")
-		}
-	}
-	return nil, nil, false
-}
-
-func (f *Interface) readOutsideSegment(via ViaSender, segment []byte, out *packet.OutPacket, lhf *LightHouseHandler, s *Scratches, q int, localCache firewall.ConntrackCache, now time.Time) {
-	h := s.h
-	err := h.Parse(segment)
-	if err != nil {
-		// Hole punch packets are 0 or 1 byte big, so let's ignore printing those errors
-		if len(segment) > 1 {
-			f.l.WithField("packet", segment).Infof("Error while parsing inbound packet from %s: %s", via, err)
+		// Hole punch packets are 0 or 1 byte big, so lets ignore printing those errors
+		if len(packet) > 1 {
+			f.l.WithField("packet", packet).Infof("Error while parsing inbound packet from %s: %s", via, err)
 		}
 		return
 	}
 
+	//l.Error("in packet ", header, packet[HeaderLen:])
+	if !via.IsRelayed {
+		if f.myVpnNetworksTable.Contains(via.UdpAddr.Addr()) {
+			if f.l.Level >= logrus.DebugLevel {
+				f.l.WithField("from", via).Debug("Refusing to process double encrypted packet")
+			}
+			return
+		}
+	}
+
 	var hostinfo *HostInfo
 	// verify if we've seen this index before, otherwise respond to the handshake initiation
 	if h.Type == header.Message && h.Subtype == header.MessageRelay {
 		hostinfo = f.hostMap.QueryRelayIndex(h.RemoteIndex)
-		newSegment, newVia, keepGoing := f.handleRelayPackets(via, hostinfo, segment, s.scratch, h, s.nb)
-		if !keepGoing {
-			return
-		}
-		via = *newVia
-		segment = newSegment
 	} else {
 		hostinfo = f.hostMap.QueryIndex(h.RemoteIndex)
 	}
@@ -139,13 +60,74 @@ func (f *Interface) readOutsideSegment(via ViaSender, segment []byte, out *packe
 
 		switch h.Subtype {
 		case header.MessageNone:
-			if !f.decryptToTunDelayWrite(hostinfo, h.MessageCounter, out, segment, s.fwPacket, s.nb, q, localCache, now) {
-				out.DestroyLastSegment() //prevent a rejected segment from being used
+			if !f.decryptToTun(hostinfo, h.MessageCounter, out, packet, fwPacket, fwCtx, nb, q, localCache) {
 				return
 			}
 		case header.MessageRelay:
-			f.l.Error("relayed messages cannot contain relay messages, dropping packet")
-			return
+			// The entire body is sent as AD, not encrypted.
+			// The packet consists of a 16-byte parsed Nebula header, Associated Data-protected payload, and a trailing 16-byte AEAD signature value.
+			// The packet is guaranteed to be at least 16 bytes at this point, b/c it got past the h.Parse() call above. If it's
+			// otherwise malformed (meaning, there is no trailing 16 byte AEAD value), then this will result in at worst a 0-length slice
+			// which will gracefully fail in the DecryptDanger call.
+			signedPayload := packet[:len(packet)-hostinfo.ConnectionState.dKey.Overhead()]
+			signatureValue := packet[len(packet)-hostinfo.ConnectionState.dKey.Overhead():]
+			out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, signedPayload, signatureValue, h.MessageCounter, nb)
+			if err != nil {
+				return
+			}
+			// Successfully validated the thing. Get rid of the Relay header.
+			signedPayload = signedPayload[header.Len:]
+			// Pull the Roaming parts up here, and return in all call paths.
+			f.handleHostRoaming(hostinfo, via)
+			// Track usage of both the HostInfo and the Relay for the received & authenticated packet
+			f.connectionManager.In(hostinfo)
+			f.connectionManager.RelayUsed(h.RemoteIndex)
+
+			relay, ok := hostinfo.relayState.QueryRelayForByIdx(h.RemoteIndex)
+			if !ok {
+				// The only way this happens is if hostmap has an index to the correct HostInfo, but the HostInfo is missing
+				// its internal mapping. This should never happen.
+				hostinfo.logger(f.l).WithFields(logrus.Fields{"vpnAddrs": hostinfo.vpnAddrs, "remoteIndex": h.RemoteIndex}).Error("HostInfo missing remote relay index")
+				return
+			}
+
+			switch relay.Type {
+			case TerminalType:
+				// If I am the target of this relay, process the unwrapped packet
+				// From this recursive point, all these variables are 'burned'. We shouldn't rely on them again.
+				via = ViaSender{
+					UdpAddr:   via.UdpAddr,
+					relayHI:   hostinfo,
+					remoteIdx: relay.RemoteIndex,
+					relay:     relay,
+					IsRelayed: true,
+				}
+				f.readOutsidePackets(via, out[:0], signedPayload, h, fwPacket, fwCtx, lhf, nb, q, localCache)
+				return
+			case ForwardingType:
+				// Find the target HostInfo relay object
+				targetHI, targetRelay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relay.PeerAddr)
+				if err != nil {
+					hostinfo.logger(f.l).WithField("relayTo", relay.PeerAddr).WithError(err).WithField("hostinfo.vpnAddrs", hostinfo.vpnAddrs).Info("Failed to find target host info by ip")
+					return
+				}
+
+				// If that relay is Established, forward the payload through it
+				if targetRelay.State == Established {
+					switch targetRelay.Type {
+					case ForwardingType:
+						// Forward this packet through the relay tunnel
+						// Find the target HostInfo
+						f.SendVia(targetHI, targetRelay, signedPayload, nb, out, false)
+						return
+					case TerminalType:
+						hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
+					}
+				} else {
+					hostinfo.logger(f.l).WithFields(logrus.Fields{"relayTo": relay.PeerAddr, "relayFrom": hostinfo.vpnAddrs[0], "targetRelayState": targetRelay.State}).Info("Unexpected target relay state")
+					return
+				}
+			}
 		}
 
 	case header.LightHouse:
@@ -154,14 +136,15 @@ func (f *Interface) readOutsideSegment(via ViaSender, segment []byte, out *packe
 			return
 		}
 
-		d, err := f.decrypt(hostinfo, h.MessageCounter, s.scratch, segment, h, s.nb)
+		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", via.UdpAddr).
-				WithField("packet", segment).
+			hostinfo.logger(f.l).WithError(err).WithField("from", via).
+				WithField("packet", packet).
 				Error("Failed to decrypt lighthouse packet")
 			return
 		}
 
+		//TODO: assert via is not relayed
 		lhf.HandleRequest(via.UdpAddr, hostinfo.vpnAddrs, d, f)
 
 		// Fallthrough to the bottom to record incoming traffic
@@ -172,10 +155,10 @@ func (f *Interface) readOutsideSegment(via ViaSender, segment []byte, out *packe
 			return
 		}
 
-		d, err := f.decrypt(hostinfo, h.MessageCounter, s.scratch, segment, h, s.nb)
+		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", via).
-				WithField("packet", segment).
+			hostinfo.logger(f.l).WithError(err).WithField("from", via).
+				WithField("packet", packet).
 				Error("Failed to decrypt test packet")
 			return
 		}
@@ -184,7 +167,7 @@ func (f *Interface) readOutsideSegment(via ViaSender, segment []byte, out *packe
 			// This testRequest might be from TryPromoteBest, so we should roam
 			// to the new IP address before responding
 			f.handleHostRoaming(hostinfo, via)
-			f.send(header.Test, header.TestReply, ci, hostinfo, d, s.nb, s.scratch)
+			f.send(header.Test, header.TestReply, ci, hostinfo, d, nb, out)
 		}
 
 		// Fallthrough to the bottom to record incoming traffic
@@ -194,7 +177,7 @@ func (f *Interface) readOutsideSegment(via ViaSender, segment []byte, out *packe
 
 	case header.Handshake:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		f.handshakeManager.HandleIncoming(via, segment, h)
+		f.handshakeManager.HandleIncoming(via, packet, h)
 		return
 
 	case header.RecvError:
@@ -207,8 +190,15 @@ func (f *Interface) readOutsideSegment(via ViaSender, segment []byte, out *packe
 		if !f.handleEncrypted(ci, via, h) {
 			return
 		}
+		_, err = f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
+		if err != nil {
+			hostinfo.logger(f.l).WithError(err).WithField("from", via).
+				WithField("packet", packet).
+				Error("Failed to decrypt CloseTunnel packet")
+			return
+		}
 
-		hostinfo.logger(f.l).WithField("udpAddr", via).
+		hostinfo.logger(f.l).WithField("from", via).
 			Info("Close tunnel received, tearing down.")
 
 		f.closeTunnel(hostinfo)
@@ -219,10 +209,10 @@ func (f *Interface) readOutsideSegment(via ViaSender, segment []byte, out *packe
 			return
 		}
 
-		d, err := f.decrypt(hostinfo, h.MessageCounter, s.scratch, segment, h, s.nb)
+		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", via).
-				WithField("packet", segment).
+			hostinfo.logger(f.l).WithError(err).WithField("from", via).
+				WithField("packet", packet).
 				Error("Failed to decrypt Control packet")
 			return
 		}
@@ -240,28 +230,6 @@ func (f *Interface) readOutsideSegment(via ViaSender, segment []byte, out *packe
 	f.connectionManager.In(hostinfo)
 }
 
-func (f *Interface) readOutsidePacketsMany(packets []*packet.UDPPacket, out []*packet.OutPacket, lhf *LightHouseHandler, s *Scratches, q int, localCache firewall.ConntrackCache, now time.Time) {
-	for i, pkt := range packets {
-		via := ViaSender{UdpAddr: pkt.AddrPort()}
-
-		//l.Error("in packet ", header, packet[HeaderLen:])
-		if f.myVpnNetworksTable.Contains(via.UdpAddr.Addr()) {
-			if f.l.Level >= logrus.DebugLevel {
-				f.l.WithField("from", via).Debug("Refusing to process double encrypted packet")
-			}
-			return
-		}
-
-		for segment := range pkt.Segments() {
-			f.readOutsideSegment(via, segment, out[i], lhf, s, q, localCache, now)
-		}
-		//_, err := f.readers[q].WriteOne(out[i], false, q)
-		//if err != nil {
-		//	f.l.WithError(err).Error("Failed to write packet")
-		//}
-	}
-}
-
 // closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote
 func (f *Interface) closeTunnel(hostInfo *HostInfo) {
 	final := f.hostMap.DeleteHostInfo(hostInfo)
@@ -327,22 +295,25 @@ var (
 )
 
 // newPacket validates and parses the interesting bits for the firewall out of the ip and sub protocol headers
-func newPacket(data []byte, incoming bool, fp *firewall.Packet) error {
+func newPacket(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.PacketContext) error {
+	if ctx != nil {
+		*ctx = firewall.PacketContext{}
+	}
 	if len(data) < 1 {
 		return ErrPacketTooShort
 	}
 
-	//version := int((data[0] >> 4) & 0x0f)
-	switch data[0] & 0xf0 {
-	case ipv4.Version << 4:
-		return parseV4(data, incoming, fp)
-	case ipv6.Version << 4:
-		return parseV6(data, incoming, fp)
+	version := int((data[0] >> 4) & 0x0f)
+	switch version {
+	case ipv4.Version:
+		return parseV4(data, incoming, fp, ctx)
+	case ipv6.Version:
+		return parseV6(data, incoming, fp, ctx)
 	}
 	return ErrUnknownIPVersion
 }
 
-func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
+func parseV6(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.PacketContext) error {
 	dataLen := len(data)
 	if dataLen < ipv6.HeaderLen {
 		return ErrIPv6PacketTooShort
@@ -366,13 +337,34 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 		proto := layers.IPProtocol(data[protoAt])
 
 		switch proto {
-		case layers.IPProtocolICMPv6, layers.IPProtocolESP, layers.IPProtocolNoNextHeader:
+		case layers.IPProtocolESP, layers.IPProtocolNoNextHeader:
 			fp.Protocol = uint8(proto)
 			fp.RemotePort = 0
 			fp.LocalPort = 0
 			fp.Fragment = false
 			return nil
 
+		case layers.IPProtocolICMPv6:
+			if dataLen < offset+6 {
+				return ErrIPv6PacketTooShort
+			}
+			fp.Protocol = uint8(proto)
+			fp.LocalPort = 0 //incoming vs outgoing doesn't matter for icmpv6
+			icmptype := data[offset+1]
+			switch icmptype {
+			case layers.ICMPv6TypeEchoRequest, layers.ICMPv6TypeEchoReply:
+				fp.RemotePort = binary.BigEndian.Uint16(data[offset+4 : offset+6]) //identifier
+			default:
+				fp.RemotePort = 0
+			}
+			fp.Fragment = false
+			if ctx != nil {
+				ctx.Length = binary.BigEndian.Uint16(data[4:6]) + uint16(ipv6.HeaderLen)
+				ctx.ICMPType = data[offset]
+				ctx.ICMPCode = data[offset+1]
+			}
+			return nil
+
 		case layers.IPProtocolTCP, layers.IPProtocolUDP:
 			if dataLen < offset+4 {
 				return ErrIPv6PacketTooShort
@@ -388,6 +380,12 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 			}
 
 			fp.Fragment = false
+			if ctx != nil {
+				ctx.Length = binary.BigEndian.Uint16(data[4:6]) + uint16(ipv6.HeaderLen)
+				if proto == layers.IPProtocolTCP && dataLen >= offset+14 {
+					ctx.TCPFlags = data[offset+13]
+				}
+			}
 			return nil
 
 		case layers.IPProtocolIPv6Fragment:
@@ -439,7 +437,7 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 	return ErrIPv6CouldNotFindPayload
 }
 
-func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
+func parseV4(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.PacketContext) error {
 	// Do we at least have an ipv4 header worth of data?
 	if len(data) < ipv4.HeaderLen {
 		return ErrIPv4PacketTooShort
@@ -462,33 +460,52 @@ func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
 
 	// Accounting for a variable header length, do we have enough data for our src/dst tuples?
 	minLen := ihl
-	if !fp.Fragment && fp.Protocol != firewall.ProtoICMP {
-		minLen += minFwPacketLen
+	if !fp.Fragment {
+		if fp.Protocol == firewall.ProtoICMP {
+			minLen += minFwPacketLen + 2
+		} else {
+			minLen += minFwPacketLen
+		}
 	}
+
 	if len(data) < minLen {
 		return ErrIPv4InvalidHeaderLength
 	}
 
-	// Firewall packets are locally oriented
-	if incoming {
+	if incoming { // Firewall packets are locally oriented
 		fp.RemoteAddr, _ = netip.AddrFromSlice(data[12:16])
 		fp.LocalAddr, _ = netip.AddrFromSlice(data[16:20])
-		if fp.Fragment || fp.Protocol == firewall.ProtoICMP {
-			fp.RemotePort = 0
-			fp.LocalPort = 0
-		} else {
-			fp.RemotePort = binary.BigEndian.Uint16(data[ihl : ihl+2])
-			fp.LocalPort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4])
-		}
 	} else {
 		fp.LocalAddr, _ = netip.AddrFromSlice(data[12:16])
 		fp.RemoteAddr, _ = netip.AddrFromSlice(data[16:20])
-		if fp.Fragment || fp.Protocol == firewall.ProtoICMP {
-			fp.RemotePort = 0
-			fp.LocalPort = 0
-		} else {
-			fp.LocalPort = binary.BigEndian.Uint16(data[ihl : ihl+2])
-			fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4])
+	}
+
+	if fp.Fragment {
+		fp.RemotePort = 0
+		fp.LocalPort = 0
+	} else if fp.Protocol == firewall.ProtoICMP { //note that orientation doesn't matter on ICMP
+		fp.RemotePort = binary.BigEndian.Uint16(data[ihl+4 : ihl+6]) //identifier
+		fp.LocalPort = 0                                             //code would be uint16(data[ihl+1])
+	} else if incoming {
+		fp.RemotePort = binary.BigEndian.Uint16(data[ihl : ihl+2])  //src port
+		fp.LocalPort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
+	} else {
+		fp.LocalPort = binary.BigEndian.Uint16(data[ihl : ihl+2])    //src port
+		fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
+	}
+
+	if ctx != nil {
+		ctx.Length = binary.BigEndian.Uint16(data[2:4])
+		if !fp.Fragment {
+			switch fp.Protocol {
+			case firewall.ProtoICMP:
+				ctx.ICMPType = data[ihl]
+				ctx.ICMPCode = data[ihl+1]
+			case firewall.ProtoTCP:
+				if len(data) >= ihl+14 {
+					ctx.TCPFlags = data[ihl+13]
+				}
+			}
 		}
 	}
 
@@ -511,23 +528,16 @@ func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, out []byte, packet []
 	return out, nil
 }
 
-func (f *Interface) decryptToTunDelayWrite(hostinfo *HostInfo, messageCounter uint64, out *packet.OutPacket, inSegment []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache, now time.Time) bool {
+func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out []byte, packet []byte, fwPacket *firewall.Packet, fwCtx *firewall.PacketContext, nb []byte, q int, localCache firewall.ConntrackCache) bool {
 	var err error
 
-	seg, err := f.readers[q].AllocSeg(out, q)
-	if err != nil {
-		f.l.WithError(err).Errorln("decryptToTunDelayWrite: failed to allocate segment")
-		return false
-	}
-
-	out.SegmentPayloads[seg] = out.SegmentPayloads[seg][:0]
-	out.SegmentPayloads[seg], err = hostinfo.ConnectionState.dKey.DecryptDanger(out.SegmentPayloads[seg], inSegment[:header.Len], inSegment[header.Len:], messageCounter, nb)
+	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
 	if err != nil {
 		hostinfo.logger(f.l).WithError(err).Error("Failed to decrypt packet")
 		return false
 	}
 
-	err = newPacket(out.SegmentPayloads[seg], true, fwPacket)
+	err = newPacket(out, true, fwPacket, fwCtx)
 	if err != nil {
 		hostinfo.logger(f.l).WithError(err).WithField("packet", out).
 			Warnf("Error while validating inbound packet")
@@ -540,11 +550,11 @@ func (f *Interface) decryptToTunDelayWrite(hostinfo *HostInfo, messageCounter ui
 		return false
 	}
 
-	dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache, now)
+	dropReason := f.firewall.Drop(*fwPacket, *fwCtx, true, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason != nil {
 		// NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore
 		// This gives us a buffer to build the reject packet in
-		f.rejectOutside(out.SegmentPayloads[seg], hostinfo.ConnectionState, hostinfo, nb, inSegment, q)
+		f.rejectOutside(out, hostinfo.ConnectionState, hostinfo, nb, packet, q)
 		if f.l.Level >= logrus.DebugLevel {
 			hostinfo.logger(f.l).WithField("fwPacket", fwPacket).
 				WithField("reason", dropReason).
@@ -554,12 +564,15 @@ func (f *Interface) decryptToTunDelayWrite(hostinfo *HostInfo, messageCounter ui
 	}
 
 	f.connectionManager.In(hostinfo)
-	out.Segments[seg] = out.Segments[seg][:len(out.SegmentHeaders[seg])+len(out.SegmentPayloads[seg])]
+	_, err = f.readers[q].Write(out)
+	if err != nil {
+		f.l.WithError(err).Error("Failed to write to tun")
+	}
 	return true
 }
 
 func (f *Interface) maybeSendRecvError(endpoint netip.AddrPort, index uint32) {
-	if f.sendRecvErrorConfig.ShouldSendRecvError(endpoint) {
+	if f.sendRecvErrorConfig.ShouldRecvError(endpoint) {
 		f.sendRecvError(endpoint, index)
 	}
 }
@@ -577,6 +590,13 @@ func (f *Interface) sendRecvError(endpoint netip.AddrPort, index uint32) {
 }
 
 func (f *Interface) handleRecvError(addr netip.AddrPort, h *header.H) {
+	if !f.acceptRecvErrorConfig.ShouldRecvError(addr) {
+		f.l.WithField("index", h.RemoteIndex).
+			WithField("udpAddr", addr).
+			Debug("Recv error received, ignoring")
+		return
+	}
+
 	if f.l.Level >= logrus.DebugLevel {
 		f.l.WithField("index", h.RemoteIndex).
 			WithField("udpAddr", addr).

outside_test.go

@@ -20,13 +20,13 @@ func Test_newPacket(t *testing.T) {
 	p := &firewall.Packet{}
 
 	// length fails
-	err := newPacket([]byte{}, true, p)
+	err := newPacket([]byte{}, true, p, nil)
 	require.ErrorIs(t, err, ErrPacketTooShort)
 
-	err = newPacket([]byte{0x40}, true, p)
+	err = newPacket([]byte{0x40}, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv4PacketTooShort)
 
-	err = newPacket([]byte{0x60}, true, p)
+	err = newPacket([]byte{0x60}, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 
 	// length fail with ip options
@@ -39,15 +39,15 @@ func Test_newPacket(t *testing.T) {
 	}
 
 	b, _ := h.Marshal()
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv4InvalidHeaderLength)
 
 	// not an ipv4 packet
-	err = newPacket([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p)
+	err = newPacket([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p, nil)
 	require.ErrorIs(t, err, ErrUnknownIPVersion)
 
 	// invalid ihl
-	err = newPacket([]byte{4<<4 | (8 >> 2 & 0x0f), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p)
+	err = newPacket([]byte{4<<4 | (8 >> 2 & 0x0f), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv4InvalidHeaderLength)
 
 	// account for variable ip header length - incoming
@@ -62,7 +62,7 @@ func Test_newPacket(t *testing.T) {
 
 	b, _ = h.Marshal()
 	b = append(b, []byte{0, 3, 0, 4}...)
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol)
@@ -84,7 +84,7 @@ func Test_newPacket(t *testing.T) {
 
 	b, _ = h.Marshal()
 	b = append(b, []byte{0, 5, 0, 6}...)
-	err = newPacket(b, false, p)
+	err = newPacket(b, false, p, nil)
 
 	require.NoError(t, err)
 	assert.Equal(t, uint8(2), p.Protocol)
@@ -114,7 +114,7 @@ func Test_newPacket_v6(t *testing.T) {
 	err := gopacket.SerializeLayers(buffer, opt, &ip)
 	require.NoError(t, err)
 
-	err = newPacket(buffer.Bytes(), true, p)
+	err = newPacket(buffer.Bytes(), true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 
 	// A v6 packet with a hop-by-hop extension
@@ -148,13 +148,14 @@ func Test_newPacket_v6(t *testing.T) {
 
 	// A full IPv6 header and 1 byte in the first extension, but missing
 	// the length byte.
-	err = newPacket(buffer.Bytes()[:41], true, p)
+	err = newPacket(buffer.Bytes()[:41], true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 
 	// A full IPv6 header plus 1 full extension, but only 1 byte of the
 	// next layer, missing length byte
-	err = newPacket(buffer.Bytes()[:49], true, p)
+	err = newPacket(buffer.Bytes()[:49], true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
+	err = nil
 
 	// A good ICMP packet
 	ip = layers.IPv6{
@@ -165,27 +166,33 @@ func Test_newPacket_v6(t *testing.T) {
 		DstIP:      net.IPv6linklocalallnodes,
 	}
 
-	icmp := layers.ICMPv6{}
-
-	buffer.Clear()
-	err = gopacket.SerializeLayers(buffer, opt, &ip, &icmp)
-	if err != nil {
-		panic(err)
+	icmp := layers.ICMPv6{
+		TypeCode: layers.ICMPv6TypeEchoRequest,
+		Checksum: 0x1234,
 	}
 
-	err = newPacket(buffer.Bytes(), true, p)
-	require.NoError(t, err)
+	buffer.Clear()
+	require.NoError(t, gopacket.SerializeLayers(buffer, opt, &ip, &icmp))
+	require.Error(t, newPacket(buffer.Bytes(), true, p, nil))
+
+	buffer.Clear()
+	echo := layers.ICMPv6Echo{
+		Identifier: 0xabcd,
+		SeqNumber:  1234,
+	}
+	require.NoError(t, gopacket.SerializeLayers(buffer, opt, &ip, &icmp, &echo))
+	require.NoError(t, newPacket(buffer.Bytes(), true, p, nil))
 	assert.Equal(t, uint8(layers.IPProtocolICMPv6), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr)
-	assert.Equal(t, uint16(0), p.RemotePort)
+	assert.Equal(t, uint16(0xabcd), p.RemotePort)
 	assert.Equal(t, uint16(0), p.LocalPort)
 	assert.False(t, p.Fragment)
 
 	// A good ESP packet
 	b := buffer.Bytes()
 	b[6] = byte(layers.IPProtocolESP)
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(layers.IPProtocolESP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -197,7 +204,7 @@ func Test_newPacket_v6(t *testing.T) {
 	// A good None packet
 	b = buffer.Bytes()
 	b[6] = byte(layers.IPProtocolNoNextHeader)
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(layers.IPProtocolNoNextHeader), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -209,7 +216,7 @@ func Test_newPacket_v6(t *testing.T) {
 	// An unknown protocol packet
 	b = buffer.Bytes()
 	b[6] = 255 // 255 is a reserved protocol number
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 
 	// A good UDP packet
@@ -236,7 +243,7 @@ func Test_newPacket_v6(t *testing.T) {
 	b = buffer.Bytes()
 
 	// incoming
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -246,7 +253,7 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// outgoing
-	err = newPacket(b, false, p)
+	err = newPacket(b, false, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
@@ -256,14 +263,14 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// Too short UDP packet
-	err = newPacket(b[:len(b)-10], false, p) // pull off the last 10 bytes
+	err = newPacket(b[:len(b)-10], false, p, nil) // pull off the last 10 bytes
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 
 	// A good TCP packet
 	b[6] = byte(layers.IPProtocolTCP)
 
 	// incoming
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -273,7 +280,7 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// outgoing
-	err = newPacket(b, false, p)
+	err = newPacket(b, false, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
@@ -283,7 +290,7 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// Too short TCP packet
-	err = newPacket(b[:len(b)-10], false, p) // pull off the last 10 bytes
+	err = newPacket(b[:len(b)-10], false, p, nil) // pull off the last 10 bytes
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 
 	// A good UDP packet with an AH header
@@ -318,7 +325,7 @@ func Test_newPacket_v6(t *testing.T) {
 	b = append(b, ahb...)
 	b = append(b, udpHeader...)
 
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -328,12 +335,12 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// Ensure buffer bounds checking during processing
-	err = newPacket(b[:41], true, p)
+	err = newPacket(b[:41], true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 
 	// Invalid AH header
 	b = buffer.Bytes()
-	err = newPacket(b, true, p)
+	err = newPacket(b, true, p, nil)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 }
 
@@ -381,7 +388,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	firstFrag = append(firstFrag, []byte{0xde, 0xad, 0xbe, 0xef}...)
 
 	// Test first fragment incoming
-	err = newPacket(firstFrag, true, p)
+	err = newPacket(firstFrag, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr)
@@ -391,7 +398,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// Test first fragment outgoing
-	err = newPacket(firstFrag, false, p)
+	err = newPacket(firstFrag, false, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.RemoteAddr)
@@ -420,7 +427,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	secondFrag = append(secondFrag, []byte{0xde, 0xad, 0xbe, 0xef}...)
 
 	// Test second fragment incoming
-	err = newPacket(secondFrag, true, p)
+	err = newPacket(secondFrag, true, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr)
@@ -430,7 +437,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	assert.True(t, p.Fragment)
 
 	// Test second fragment outgoing
-	err = newPacket(secondFrag, false, p)
+	err = newPacket(secondFrag, false, p, nil)
 	require.NoError(t, err)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.RemoteAddr)
@@ -440,7 +447,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	assert.True(t, p.Fragment)
 
 	// Too short of a fragment packet
-	err = newPacket(secondFrag[:len(secondFrag)-10], false, p)
+	err = newPacket(secondFrag[:len(secondFrag)-10], false, p, nil)
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 }
 
@@ -522,7 +529,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("Normal", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(normalPacket, true, fp); err != nil {
+			if err = parseV6(normalPacket, true, fp, nil); err != nil {
 				b.Fatal(err)
 			}
 		}
@@ -530,7 +537,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("FirstFragment", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(firstFrag, true, fp); err != nil {
+			if err = parseV6(firstFrag, true, fp, nil); err != nil {
 				b.Fatal(err)
 			}
 		}
@@ -538,7 +545,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("SecondFragment", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(secondFrag, true, fp); err != nil {
+			if err = parseV6(secondFrag, true, fp, nil); err != nil {
 				b.Fatal(err)
 			}
 		}
@@ -574,7 +581,7 @@ func BenchmarkParseV6(b *testing.B) {
 	}
 
 	evilBytes := buffer.Bytes()
-	for i := 0; i < 200; i++ {
+	for range 200 {
 		evilBytes = append(evilBytes, hopHeader...)
 	}
 	evilBytes = append(evilBytes, lastHopHeader...)
@@ -583,7 +590,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("200 HopByHop headers", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(evilBytes, false, fp); err != nil {
+			if err = parseV6(evilBytes, false, fp, nil); err != nil {
 				b.Fatal(err)
 			}
 		}

overlay/device.go

@@ -1,17 +1,18 @@
 package overlay
 
 import (
+	"io"
 	"net/netip"
 
 	"github.com/slackhq/nebula/routing"
 )
 
 type Device interface {
-	TunDev
+	io.ReadWriteCloser
 	Activate() error
 	Networks() []netip.Prefix
 	Name() string
 	RoutesFor(netip.Addr) routing.Gateways
 	SupportsMultiqueue() bool
-	NewMultiQueueReader() (TunDev, error)
+	NewMultiQueueReader() (io.ReadWriteCloser, error)
 }

overlay/eventfd/eventfd.go

@@ -1,91 +0,0 @@
-package eventfd
-
-import (
-	"encoding/binary"
-	"syscall"
-
-	"golang.org/x/sys/unix"
-)
-
-type EventFD struct {
-	fd  int
-	buf [8]byte
-}
-
-func New() (EventFD, error) {
-	fd, err := unix.Eventfd(0, unix.EFD_NONBLOCK)
-	if err != nil {
-		return EventFD{}, err
-	}
-	return EventFD{
-		fd:  fd,
-		buf: [8]byte{},
-	}, nil
-}
-
-func (e *EventFD) Kick() error {
-	binary.LittleEndian.PutUint64(e.buf[:], 1) //is this right???
-	_, err := syscall.Write(int(e.fd), e.buf[:])
-	return err
-}
-
-func (e *EventFD) Close() error {
-	if e.fd != 0 {
-		return unix.Close(e.fd)
-	}
-	return nil
-}
-
-func (e *EventFD) FD() int {
-	return e.fd
-}
-
-type Epoll struct {
-	fd     int
-	buf    [8]byte
-	events []syscall.EpollEvent
-}
-
-func NewEpoll() (Epoll, error) {
-	fd, err := unix.EpollCreate1(0)
-	if err != nil {
-		return Epoll{}, err
-	}
-	return Epoll{
-		fd:     fd,
-		buf:    [8]byte{},
-		events: make([]syscall.EpollEvent, 1),
-	}, nil
-}
-
-func (ep *Epoll) AddEvent(fdToAdd int) error {
-	event := syscall.EpollEvent{
-		Events: syscall.EPOLLIN,
-		Fd:     int32(fdToAdd),
-	}
-	return syscall.EpollCtl(ep.fd, syscall.EPOLL_CTL_ADD, fdToAdd, &event)
-}
-
-func (ep *Epoll) Block() (int, error) {
-	n, err := syscall.EpollWait(ep.fd, ep.events, -1)
-	if err != nil {
-		//goland:noinspection GoDirectComparisonOfErrors
-		if err == syscall.EINTR {
-			return 0, nil //??
-		}
-		return -1, err
-	}
-	return n, nil
-}
-
-func (ep *Epoll) Clear() error {
-	_, err := syscall.Read(int(ep.events[0].Fd), ep.buf[:])
-	return err
-}
-
-func (ep *Epoll) Close() error {
-	if ep.fd != 0 {
-		return unix.Close(ep.fd)
-	}
-	return nil
-}

overlay/packets.go

@@ -1,36 +0,0 @@
-package overlay
-
-//import (
-//	"github.com/slackhq/nebula/util/virtio"
-//)
-
-//type VirtIOPacket struct {
-//	Payload   []byte
-//	Header    virtio.NetHdr
-//	Chains    []uint16
-//	ChainRefs [][]byte
-//}
-//
-//func NewVIO() *VirtIOPacket {
-//	out := new(VirtIOPacket)
-//	out.Payload = nil
-//	out.ChainRefs = make([][]byte, 0, 4)
-//	out.Chains = make([]uint16, 0, 8)
-//	return out
-//}
-//
-//func (v *VirtIOPacket) Reset() {
-//	v.Payload = nil
-//	v.ChainRefs = v.ChainRefs[:0]
-//	v.Chains = v.Chains[:0]
-//}
-
-// TunPacket is formerly VirtIOPacket
-type TunPacket interface {
-	SetPayload([]byte)
-	GetPayload() []byte
-}
-type OutPacket interface {
-	SetPayload([]byte)
-	GetPayload() []byte
-}

overlay/tun.go

@@ -2,29 +2,23 @@ package overlay
 
 import (
 	"fmt"
-	"io"
 	"net"
 	"net/netip"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/packet"
 	"github.com/slackhq/nebula/util"
 )
 
 const DefaultMTU = 1300
 
-type TunDev interface {
-	io.WriteCloser
-	NewPacketArrays(batchSize int) []TunPacket
+type NameError struct {
+	Name       string
+	Underlying error
+}
 
-	ReadMany(x []TunPacket, q int) (int, error)
-	RecycleRxSeg(pkt TunPacket, kick bool, q int) error
-
-	//todo this interface sux
-	AllocSeg(pkt *packet.OutPacket, q int) (int, error)
-	WriteOne(x *packet.OutPacket, kick bool, q int) (int, error)
-	WriteMany(x []*packet.OutPacket, q int) (int, error)
+func (e *NameError) Error() string {
+	return fmt.Sprintf("could not set tun device name: %s because %s", e.Name, e.Underlying)
 }
 
 // TODO: We may be able to remove routines
@@ -33,19 +27,19 @@ type DeviceFactory func(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefi
 func NewDeviceFromConfig(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
 	switch {
 	case c.GetBool("tun.disabled", false):
-		t := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l)
-		return t, nil
+		tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l)
+		return tun, nil
 
 	default:
 		return newTun(c, l, vpnNetworks, routines > 1)
 	}
 }
 
-//func NewFdDeviceFromConfig(fd *int) DeviceFactory {
-//	return func(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
-//		return newTunFromFd(c, l, *fd, vpnNetworks)
-//	}
-//}
+func NewFdDeviceFromConfig(fd *int) DeviceFactory {
+	return func(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
+		return newTunFromFd(c, l, *fd, vpnNetworks)
+	}
+}
 
 func getAllRoutesFromConfig(c *config.C, vpnNetworks []netip.Prefix, initial bool) (bool, []Route, error) {
 	if !initial && !c.HasChanged("tun.routes") && !c.HasChanged("tun.unsafe_routes") {

overlay/tun_disabled.go

@@ -9,8 +9,6 @@ import (
 	"github.com/rcrowley/go-metrics"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/iputil"
-	"github.com/slackhq/nebula/overlay/virtqueue"
-	"github.com/slackhq/nebula/packet"
 	"github.com/slackhq/nebula/routing"
 )
 
@@ -24,14 +22,6 @@ type disabledTun struct {
 	l  *logrus.Logger
 }
 
-func (t *disabledTun) NewPacketArrays(batchSize int) []TunPacket {
-	panic("implement me") //TODO
-}
-
-func (*disabledTun) RecycleRxSeg(pkt TunPacket, kick bool, q int) error {
-	return nil
-}
-
 func newDisabledTun(vpnNetworks []netip.Prefix, queueLen int, metricsEnabled bool, l *logrus.Logger) *disabledTun {
 	tun := &disabledTun{
 		vpnNetworks: vpnNetworks,
@@ -50,10 +40,6 @@ func newDisabledTun(vpnNetworks []netip.Prefix, queueLen int, metricsEnabled boo
 	return tun
 }
 
-func (*disabledTun) GetQueues() []*virtqueue.SplitQueue {
-	return nil
-}
-
 func (*disabledTun) Activate() error {
 	return nil
 }
@@ -123,23 +109,7 @@ func (t *disabledTun) SupportsMultiqueue() bool {
 	return true
 }
 
-func (t *disabledTun) AllocSeg(pkt *packet.OutPacket, q int) (int, error) {
-	return 0, fmt.Errorf("tun_disabled: AllocSeg not implemented")
-}
-
-func (t *disabledTun) WriteOne(x *packet.OutPacket, kick bool, q int) (int, error) {
-	return 0, fmt.Errorf("tun_disabled: WriteOne not implemented")
-}
-
-func (t *disabledTun) WriteMany(x []*packet.OutPacket, q int) (int, error) {
-	return 0, fmt.Errorf("tun_disabled: WriteMany not implemented")
-}
-
-func (t *disabledTun) ReadMany(b []TunPacket, _ int) (int, error) {
-	return t.Read(b[0].GetPayload())
-}
-
-func (t *disabledTun) NewMultiQueueReader() (TunDev, error) {
+func (t *disabledTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return t, nil
 }
 

overlay/tun_file_linux_test.go

@@ -0,0 +1,120 @@
+//go:build linux && !android && !e2e_testing
+// +build linux,!android,!e2e_testing
+
+package overlay
+
+import (
+	"errors"
+	"os"
+	"sync"
+	"testing"
+	"time"
+
+	"golang.org/x/sys/unix"
+)
+
+// newReadPipe returns a read fd. The matching write fd is registered for cleanup.
+// The caller takes ownership of the read fd (pass it to newTunFd / newFriend).
+func newReadPipe(t *testing.T) int {
+	t.Helper()
+	var fds [2]int
+	if err := unix.Pipe2(fds[:], unix.O_CLOEXEC); err != nil {
+		t.Fatalf("pipe2: %v", err)
+	}
+	t.Cleanup(func() { _ = unix.Close(fds[1]) })
+	return fds[0]
+}
+
+func TestTunFile_WakeForShutdown_UnblocksRead(t *testing.T) {
+	tf, err := newTunFd(newReadPipe(t))
+	if err != nil {
+		t.Fatalf("newTunFd: %v", err)
+	}
+	t.Cleanup(func() { _ = tf.Close() })
+
+	done := make(chan error, 1)
+	go func() {
+		_, err := tf.Read(make([]byte, 64))
+		done <- err
+	}()
+
+	// Verify Read is actually blocked in poll.
+	select {
+	case err := <-done:
+		t.Fatalf("Read returned before shutdown signal: %v", err)
+	case <-time.After(50 * time.Millisecond):
+	}
+
+	if err := tf.wakeForShutdown(); err != nil {
+		t.Fatalf("wakeForShutdown: %v", err)
+	}
+
+	select {
+	case err := <-done:
+		if !errors.Is(err, os.ErrClosed) {
+			t.Fatalf("expected os.ErrClosed, got %v", err)
+		}
+	case <-time.After(2 * time.Second):
+		t.Fatal("Read did not wake on shutdown")
+	}
+}
+
+func TestTunFile_WakeForShutdown_WakesFriends(t *testing.T) {
+	parent, err := newTunFd(newReadPipe(t))
+	if err != nil {
+		t.Fatalf("newTunFd: %v", err)
+	}
+	friend, err := parent.newFriend(newReadPipe(t))
+	if err != nil {
+		_ = parent.Close()
+		t.Fatalf("newFriend: %v", err)
+	}
+	t.Cleanup(func() {
+		_ = friend.Close()
+		_ = parent.Close()
+	})
+
+	readers := []*tunFile{parent, friend}
+	errs := make([]error, len(readers))
+	var wg sync.WaitGroup
+	for i, r := range readers {
+		wg.Add(1)
+		go func(i int, r *tunFile) {
+			defer wg.Done()
+			_, errs[i] = r.Read(make([]byte, 64))
+		}(i, r)
+	}
+
+	time.Sleep(50 * time.Millisecond)
+
+	if err := parent.wakeForShutdown(); err != nil {
+		t.Fatalf("wakeForShutdown: %v", err)
+	}
+
+	done := make(chan struct{})
+	go func() { wg.Wait(); close(done) }()
+	select {
+	case <-done:
+	case <-time.After(2 * time.Second):
+		t.Fatal("readers did not wake")
+	}
+
+	for i, err := range errs {
+		if !errors.Is(err, os.ErrClosed) {
+			t.Errorf("reader %d: expected os.ErrClosed, got %v", i, err)
+		}
+	}
+}
+
+func TestTunFile_Close_Idempotent(t *testing.T) {
+	tf, err := newTunFd(newReadPipe(t))
+	if err != nil {
+		t.Fatalf("newTunFd: %v", err)
+	}
+	if err := tf.Close(); err != nil {
+		t.Fatalf("first Close: %v", err)
+	}
+	if err := tf.Close(); err != nil {
+		t.Fatalf("second Close should be a no-op, got %v", err)
+	}
+}

overlay/tun_freebsd.go

@@ -10,6 +10,7 @@ import (
 	"io"
 	"io/fs"
 	"net/netip"
+	"os"
 	"sync/atomic"
 	"syscall"
 	"time"
@@ -93,107 +94,184 @@ type tun struct {
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
 	linkAddr    *netroute.LinkAddr
 	l           *logrus.Logger
-	devFd       int
+
+	fd        int
+	shutdownR int // read end of the shutdown pipe; closing the write end wakes blocked polls
+	shutdownW int // write end of the shutdown pipe; closing this signals shutdown to any blocked reader/writer
+	readPoll  [2]unix.PollFd
+	writePoll [2]unix.PollFd
+	closed    atomic.Bool
+}
+
+// blockOnRead waits until the tun fd is readable or shutdown has been signaled.
+// Returns os.ErrClosed if Close was called.
+func (t *tun) blockOnRead() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(t.readPoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	tunEvents := t.readPoll[0].Revents
+	shutdownEvents := t.readPoll[1].Revents
+	t.readPoll[0].Revents = 0
+	t.readPoll[1].Revents = 0
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	}
+	if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
+}
+
+func (t *tun) blockOnWrite() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(t.writePoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	tunEvents := t.writePoll[0].Revents
+	shutdownEvents := t.writePoll[1].Revents
+	t.writePoll[0].Revents = 0
+	t.writePoll[1].Revents = 0
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	}
+	if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
 }
 
 func (t *tun) Read(to []byte) (int, error) {
-	// use readv() to read from the tunnel device, to eliminate the need for copying the buffer
-	if t.devFd < 0 {
-		return -1, syscall.EINVAL
-	}
-
 	// first 4 bytes is protocol family, in network byte order
-	head := make([]byte, 4)
-
-	iovecs := []syscall.Iovec{
+	var head [4]byte
+	iovecs := [2]syscall.Iovec{
 		{&head[0], 4},
 		{&to[0], uint64(len(to))},
 	}
-
-	n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
-
-	var err error
-	if errno != 0 {
-		err = syscall.Errno(errno)
-	} else {
-		err = nil
-	}
-	// fix bytes read number to exclude header
-	bytesRead := int(n)
-	if bytesRead < 0 {
-		return bytesRead, err
-	} else if bytesRead < 4 {
-		return 0, err
-	} else {
-		return bytesRead - 4, err
+	for {
+		n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
+		if errno == 0 {
+			bytesRead := int(n)
+			if bytesRead < 4 {
+				return 0, nil
+			}
+			return bytesRead - 4, nil
+		}
+		switch errno {
+		case unix.EAGAIN:
+			if err := t.blockOnRead(); err != nil {
+				return 0, err
+			}
+		case unix.EINTR:
+			// retry
+		case unix.EBADF:
+			return 0, os.ErrClosed
+		default:
+			return 0, errno
+		}
 	}
 }
 
 // Write is only valid for single threaded use
 func (t *tun) Write(from []byte) (int, error) {
-	// use writev() to write to the tunnel device, to eliminate the need for copying the buffer
-	if t.devFd < 0 {
-		return -1, syscall.EINVAL
-	}
-
 	if len(from) <= 1 {
 		return 0, syscall.EIO
 	}
+
 	ipVer := from[0] >> 4
-	var head []byte
+	var head [4]byte
 	// first 4 bytes is protocol family, in network byte order
-	if ipVer == 4 {
-		head = []byte{0, 0, 0, syscall.AF_INET}
-	} else if ipVer == 6 {
-		head = []byte{0, 0, 0, syscall.AF_INET6}
-	} else {
+	switch ipVer {
+	case 4:
+		head[3] = syscall.AF_INET
+	case 6:
+		head[3] = syscall.AF_INET6
+	default:
 		return 0, fmt.Errorf("unable to determine IP version from packet")
 	}
-	iovecs := []syscall.Iovec{
+
+	iovecs := [2]syscall.Iovec{
 		{&head[0], 4},
 		{&from[0], uint64(len(from))},
 	}
-
-	n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
-
-	var err error
-	if errno != 0 {
-		err = syscall.Errno(errno)
-	} else {
-		err = nil
+	for {
+		n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
+		if errno == 0 {
+			return int(n) - 4, nil
+		}
+		switch errno {
+		case unix.EAGAIN:
+			if err := t.blockOnWrite(); err != nil {
+				return 0, err
+			}
+		case unix.EINTR:
+			// retry
+		case unix.EBADF:
+			return 0, os.ErrClosed
+		default:
+			return 0, errno
+		}
 	}
-
-	return int(n) - 4, err
 }
 
 func (t *tun) Close() error {
-	if t.devFd >= 0 {
-		err := syscall.Close(t.devFd)
-		if err != nil {
+	if t.closed.Swap(true) {
+		return nil
+	}
+
+	// Closing the write end of the shutdown pipe causes any blocked Poll to
+	// return with POLLHUP on the shutdown fd, so readers/writers wake up and
+	// exit with os.ErrClosed.
+	if t.shutdownW >= 0 {
+		_ = unix.Close(t.shutdownW)
+		t.shutdownW = -1
+	}
+
+	if t.fd >= 0 {
+		if err := unix.Close(t.fd); err != nil {
 			t.l.WithError(err).Error("Error closing device")
 		}
-		t.devFd = -1
+		t.fd = -1
+	}
 
-		c := make(chan struct{})
-		go func() {
-			// destroying the interface can block if a read() is still pending. Do this asynchronously.
-			defer close(c)
-			s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
-			if err == nil {
-				defer syscall.Close(s)
-				ifreq := ifreqDestroy{Name: t.deviceBytes()}
-				err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
-			}
-			if err != nil {
-				t.l.WithError(err).Error("Error destroying tunnel")
-			}
-		}()
+	if t.shutdownR >= 0 {
+		_ = unix.Close(t.shutdownR)
+		t.shutdownR = -1
+	}
 
-		// wait up to 1 second so we start blocking at the ioctl
-		select {
-		case <-c:
-		case <-time.After(1 * time.Second):
+	c := make(chan struct{})
+	go func() {
+		// destroying the interface can block if a read() is still pending. Do this asynchronously.
+		defer close(c)
+		s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
+		if err == nil {
+			defer syscall.Close(s)
+			ifreq := ifreqDestroy{Name: t.deviceBytes()}
+			err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
 		}
+		if err != nil {
+			t.l.WithError(err).Error("Error destroying tunnel")
+		}
+	}()
+
+	// wait up to 1 second so we start blocking at the ioctl
+	select {
+	case <-c:
+	case <-time.After(1 * time.Second):
 	}
 
 	return nil
@@ -209,16 +287,38 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 	var err error
 	deviceName := c.GetString("tun.dev", "")
 	if deviceName != "" {
-		fd, err = syscall.Open("/dev/"+deviceName, syscall.O_RDWR, 0)
+		fd, err = unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
 	}
 	if errors.Is(err, fs.ErrNotExist) || deviceName == "" {
 		// If the device doesn't already exist, request a new one and rename it
-		fd, err = syscall.Open("/dev/tun", syscall.O_RDWR, 0)
+		fd, err = unix.Open("/dev/tun", os.O_RDWR, 0)
 	}
 	if err != nil {
 		return nil, err
 	}
 
+	if err = unix.SetNonblock(fd, true); err != nil {
+		_ = unix.Close(fd)
+		return nil, fmt.Errorf("failed to set tun device as nonblocking: %w", err)
+	}
+
+	// Shutdown pipe lets Close wake any reader/writer blocked in Poll.
+	var pipeFds [2]int
+	if err = unix.Pipe2(pipeFds[:], unix.O_CLOEXEC|unix.O_NONBLOCK); err != nil {
+		_ = unix.Close(fd)
+		return nil, fmt.Errorf("failed to create shutdown pipe: %w", err)
+	}
+	shutdownR, shutdownW := pipeFds[0], pipeFds[1]
+
+	closeOnErr := true
+	defer func() {
+		if closeOnErr {
+			_ = unix.Close(fd)
+			_ = unix.Close(shutdownR)
+			_ = unix.Close(shutdownW)
+		}
+	}()
+
 	// Read the name of the interface
 	var name [16]byte
 	arg := fiodgnameArg{length: 16, buf: unsafe.Pointer(&name)}
@@ -237,7 +337,7 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 	}
 
 	if ctrlErr != nil {
-		return nil, err
+		return nil, ctrlErr
 	}
 
 	ifName := string(bytes.TrimRight(name[:], "\x00"))
@@ -253,8 +353,6 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		}
 		defer syscall.Close(s)
 
-		fd := uintptr(s)
-
 		var fromName [16]byte
 		var toName [16]byte
 		copy(fromName[:], ifName)
@@ -266,7 +364,7 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		}
 
 		// Set the device name
-		ioctl(fd, syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr)))
+		_ = ioctl(uintptr(s), syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr)))
 	}
 
 	t := &tun{
@@ -274,13 +372,24 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		vpnNetworks: vpnNetworks,
 		MTU:         c.GetInt("tun.mtu", DefaultMTU),
 		l:           l,
-		devFd:       fd,
+		fd:          fd,
+		shutdownR:   shutdownR,
+		shutdownW:   shutdownW,
+		readPoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLIN},
+			{Fd: int32(shutdownR), Events: unix.POLLIN},
+		},
+		writePoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLOUT},
+			{Fd: int32(shutdownR), Events: unix.POLLIN},
+		},
 	}
 
 	err = t.reload(c, true)
 	if err != nil {
 		return nil, err
 	}
+	closeOnErr = false
 
 	c.RegisterReloadCallback(func(c *config.C) {
 		err := t.reload(c, false)

overlay/tun_linux.go

@@ -4,12 +4,14 @@
 package overlay
 
 import (
-	"context"
+	"encoding/binary"
 	"fmt"
+	"io"
 	"net"
 	"net/netip"
 	"os"
 	"strings"
+	"sync"
 	"sync/atomic"
 	"time"
 	"unsafe"
@@ -17,19 +19,181 @@ import (
 	"github.com/gaissmai/bart"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay/vhostnet"
-	"github.com/slackhq/nebula/packet"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
-	"github.com/slackhq/nebula/util/virtio"
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
 )
 
+// tunFile wraps a TUN file descriptor with poll-based reads. The FD provided will be changed to non-blocking.
+// A shared eventfd allows Close to wake all readers blocked in poll.
+type tunFile struct {
+	fd         int
+	shutdownFd int
+	lastOne    bool
+	readPoll   [2]unix.PollFd
+	writePoll  [2]unix.PollFd
+	closed     bool
+}
+
+// newFriend makes a tunFile for a MultiQueueReader that copies the shutdown eventfd from the parent tun
+func (r *tunFile) newFriend(fd int) (*tunFile, error) {
+	if err := unix.SetNonblock(fd, true); err != nil {
+		return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
+	}
+	return &tunFile{
+		fd:         fd,
+		shutdownFd: r.shutdownFd,
+		readPoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLIN},
+			{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
+		},
+		writePoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLOUT},
+			{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
+		},
+	}, nil
+}
+
+func newTunFd(fd int) (*tunFile, error) {
+	if err := unix.SetNonblock(fd, true); err != nil {
+		return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
+	}
+
+	shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create eventfd: %w", err)
+	}
+
+	out := &tunFile{
+		fd:         fd,
+		shutdownFd: shutdownFd,
+		lastOne:    true,
+		readPoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLIN},
+			{Fd: int32(shutdownFd), Events: unix.POLLIN},
+		},
+		writePoll: [2]unix.PollFd{
+			{Fd: int32(fd), Events: unix.POLLOUT},
+			{Fd: int32(shutdownFd), Events: unix.POLLIN},
+		},
+	}
+
+	return out, nil
+}
+
+func (r *tunFile) blockOnRead() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(r.readPoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	//always reset these!
+	tunEvents := r.readPoll[0].Revents
+	shutdownEvents := r.readPoll[1].Revents
+	r.readPoll[0].Revents = 0
+	r.readPoll[1].Revents = 0
+	//do the err check before trusting the potentially bogus bits we just got
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	} else if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
+}
+
+func (r *tunFile) blockOnWrite() error {
+	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
+	var err error
+	for {
+		_, err = unix.Poll(r.writePoll[:], -1)
+		if err != unix.EINTR {
+			break
+		}
+	}
+	//always reset these!
+	tunEvents := r.writePoll[0].Revents
+	shutdownEvents := r.writePoll[1].Revents
+	r.writePoll[0].Revents = 0
+	r.writePoll[1].Revents = 0
+	//do the err check before trusting the potentially bogus bits we just got
+	if err != nil {
+		return err
+	}
+	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
+		return os.ErrClosed
+	} else if tunEvents&problemFlags != 0 {
+		return os.ErrClosed
+	}
+	return nil
+}
+
+func (r *tunFile) Read(buf []byte) (int, error) {
+	for {
+		if n, err := unix.Read(r.fd, buf); err == nil {
+			return n, nil
+		} else if err == unix.EAGAIN {
+			if err = r.blockOnRead(); err != nil {
+				return 0, err
+			}
+			continue
+		} else if err == unix.EINTR {
+			continue
+		} else if err == unix.EBADF {
+			return 0, os.ErrClosed
+		} else {
+			return 0, err
+		}
+	}
+}
+
+func (r *tunFile) Write(buf []byte) (int, error) {
+	for {
+		if n, err := unix.Write(r.fd, buf); err == nil {
+			return n, nil
+		} else if err == unix.EAGAIN {
+			if err = r.blockOnWrite(); err != nil {
+				return 0, err
+			}
+			continue
+		} else if err == unix.EINTR {
+			continue
+		} else if err == unix.EBADF {
+			return 0, os.ErrClosed
+		} else {
+			return 0, err
+		}
+	}
+}
+
+func (r *tunFile) wakeForShutdown() error {
+	var buf [8]byte
+	binary.NativeEndian.PutUint64(buf[:], 1)
+	_, err := unix.Write(int(r.readPoll[1].Fd), buf[:])
+	return err
+}
+
+func (r *tunFile) Close() error {
+	if r.closed { // avoid closing more than once. Technically a fd could get re-used, which would be a problem
+		return nil
+	}
+	r.closed = true
+	if r.lastOne {
+		_ = unix.Close(r.shutdownFd)
+	}
+	return unix.Close(r.fd)
+}
+
 type tun struct {
-	file        *os.File
-	fd          int
-	vdev        []*vhostnet.Device
+	*tunFile
+	readers     []*tunFile
+	closeLock   sync.Mutex
 	Device      string
 	vpnNetworks []netip.Prefix
 	MaxMTU      int
@@ -44,8 +208,12 @@ type tun struct {
 	useSystemRoutes           bool
 	useSystemRoutesBufferSize int
 
-	isV6 bool
-	l    *logrus.Logger
+	// These are routes learned from `tun.use_system_route_table`
+	// stored here to make it easier to restore them after a reload
+	routesFromSystem     map[netip.Prefix]routing.Gateways
+	routesFromSystemLock sync.Mutex
+
+	l *logrus.Logger
 }
 
 func (t *tun) Networks() []netip.Prefix {
@@ -71,9 +239,7 @@ type ifreqQLEN struct {
 }
 
 func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
-	file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
-
-	t, err := newTunGeneric(c, l, file, vpnNetworks)
+	t, err := newTunGeneric(c, l, deviceFd, vpnNetworks)
 	if err != nil {
 		return nil, err
 	}
@@ -107,70 +273,52 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueu
 	}
 
 	var req ifReq
-	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_TUN_EXCL | unix.IFF_VNET_HDR | unix.IFF_NAPI)
+	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI)
 	if multiqueue {
 		req.Flags |= unix.IFF_MULTI_QUEUE
 	}
-	copy(req.Name[:], c.GetString("tun.dev", ""))
+	nameStr := c.GetString("tun.dev", "")
+	copy(req.Name[:], nameStr)
 	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
-		return nil, err
+		_ = unix.Close(fd)
+		return nil, &NameError{
+			Name:       nameStr,
+			Underlying: err,
+		}
 	}
 	name := strings.Trim(string(req.Name[:]), "\x00")
 
-	if err = unix.SetNonblock(fd, true); err != nil {
-		_ = unix.Close(fd)
-		return nil, fmt.Errorf("make file descriptor non-blocking: %w", err)
-	}
-
-	file := os.NewFile(uintptr(fd), "/dev/net/tun")
-
-	err = unix.IoctlSetPointerInt(fd, unix.TUNSETVNETHDRSZ, virtio.NetHdrSize)
-	if err != nil {
-		return nil, fmt.Errorf("set vnethdr size: %w", err)
-	}
-
-	flags := 0
-	//flags = //unix.TUN_F_CSUM //| unix.TUN_F_TSO4 | unix.TUN_F_USO4 | unix.TUN_F_TSO6 | unix.TUN_F_USO6
-	err = unix.IoctlSetInt(fd, unix.TUNSETOFFLOAD, flags)
-	if err != nil {
-		return nil, fmt.Errorf("set offloads: %w", err)
-	}
-
-	t, err := newTunGeneric(c, l, file, vpnNetworks)
+	t, err := newTunGeneric(c, l, fd, vpnNetworks)
 	if err != nil {
 		return nil, err
 	}
-	t.fd = fd
+
 	t.Device = name
 
-	vdev, err := vhostnet.NewDevice(
-		vhostnet.WithBackendFD(fd),
-		vhostnet.WithQueueSize(8192), //todo config
-	)
-	if err != nil {
-		return nil, err
-	}
-	t.vdev = []*vhostnet.Device{vdev}
-
 	return t, nil
 }
 
-func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, vpnNetworks []netip.Prefix) (*tun, error) {
+// newTunGeneric does all the stuff common to different tun initialization paths. It will close your files on error.
+func newTunGeneric(c *config.C, l *logrus.Logger, fd int, vpnNetworks []netip.Prefix) (*tun, error) {
+	tfd, err := newTunFd(fd)
+	if err != nil {
+		_ = unix.Close(fd)
+		return nil, err
+	}
 	t := &tun{
-		file:                      file,
-		fd:                        int(file.Fd()),
+		tunFile:                   tfd,
+		readers:                   []*tunFile{tfd},
+		closeLock:                 sync.Mutex{},
 		vpnNetworks:               vpnNetworks,
 		TXQueueLen:                c.GetInt("tun.tx_queue", 500),
 		useSystemRoutes:           c.GetBool("tun.use_system_route_table", false),
 		useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0),
+		routesFromSystem:          map[netip.Prefix]routing.Gateways{},
 		l:                         l,
 	}
-	if len(vpnNetworks) != 0 {
-		t.isV6 = vpnNetworks[0].Addr().Is6() //todo what about multi-IP?
-	}
 
-	err := t.reload(c, true)
-	if err != nil {
+	if err = t.reload(c, true); err != nil {
+		_ = t.Close()
 		return nil, err
 	}
 
@@ -184,14 +332,6 @@ func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, vpnNetworks []n
 	return t, nil
 }
 
-func (t *tun) NewPacketArrays(batchSize int) []TunPacket {
-	inPackets := make([]TunPacket, batchSize)
-	for i := 0; i < batchSize; i++ {
-		inPackets[i] = vhostnet.NewVIO()
-	}
-	return inPackets
-}
-
 func (t *tun) reload(c *config.C, initial bool) error {
 	routeChange, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial)
 	if err != nil {
@@ -207,6 +347,13 @@ func (t *tun) reload(c *config.C, initial bool) error {
 		return err
 	}
 
+	// Bring along any routes learned from the system route table on reload
+	t.routesFromSystemLock.Lock()
+	for dst, gw := range t.routesFromSystem {
+		routeTree.Insert(dst, gw)
+	}
+	t.routesFromSystemLock.Unlock()
+
 	oldDefaultMTU := t.DefaultMTU
 	oldMaxMTU := t.MaxMTU
 	newDefaultMTU := c.GetInt("tun.mtu", DefaultMTU)
@@ -263,7 +410,10 @@ func (t *tun) SupportsMultiqueue() bool {
 	return true
 }
 
-func (t *tun) NewMultiQueueReader() (TunDev, error) {
+func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
+	t.closeLock.Lock()
+	defer t.closeLock.Unlock()
+
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
 	if err != nil {
 		return nil, err
@@ -273,20 +423,19 @@ func (t *tun) NewMultiQueueReader() (TunDev, error) {
 	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
 	copy(req.Name[:], t.Device)
 	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
+		_ = unix.Close(fd)
 		return nil, err
 	}
 
-	vdev, err := vhostnet.NewDevice(
-		vhostnet.WithBackendFD(fd),
-		vhostnet.WithQueueSize(8192), //todo config
-	)
+	out, err := t.tunFile.newFriend(fd)
 	if err != nil {
+		_ = unix.Close(fd)
 		return nil, err
 	}
 
-	t.vdev = append(t.vdev, vdev)
+	t.readers = append(t.readers, out)
 
-	return t, nil
+	return out, nil
 }
 
 func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
@@ -701,116 +850,56 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
 
 	newTree := t.routeTree.Load().Clone()
 
+	t.routesFromSystemLock.Lock()
 	if r.Type == unix.RTM_NEWROUTE {
 		t.l.WithField("destination", dst).WithField("via", gateways).Info("Adding route")
+		t.routesFromSystem[dst] = gateways
 		newTree.Insert(dst, gateways)
 
 	} else {
 		t.l.WithField("destination", dst).WithField("via", gateways).Info("Removing route")
+		delete(t.routesFromSystem, dst)
 		newTree.Delete(dst)
 	}
+	t.routesFromSystemLock.Unlock()
 	t.routeTree.Store(newTree)
 }
 
 func (t *tun) Close() error {
+	t.closeLock.Lock()
+	defer t.closeLock.Unlock()
+
 	if t.routeChan != nil {
 		close(t.routeChan)
+		t.routeChan = nil
 	}
 
-	for _, v := range t.vdev {
-		if v != nil {
-			_ = v.Close()
-		}
-	}
-
-	if t.file != nil {
-		_ = t.file.Close()
-	}
+	// Signal all readers blocked in poll to wake up and exit
+	_ = t.tunFile.wakeForShutdown()
 
 	if t.ioctlFd > 0 {
-		_ = os.NewFile(t.ioctlFd, "ioctlFd").Close()
+		_ = unix.Close(int(t.ioctlFd))
+		t.ioctlFd = 0
 	}
 
-	return nil
-}
-
-func (t *tun) ReadMany(p []TunPacket, q int) (int, error) {
-	err := t.vdev[q].ReceiveQueue.WaitForUsedElements(context.TODO())
-	if err != nil {
-		return 0, err
-	}
-	i := 0
-	for i = 0; i < len(p); i++ {
-		item, ok := t.vdev[q].ReceiveQueue.TakeSingleNoBlock()
-		if !ok {
-			break
+	for i := range t.readers {
+		if i == 0 {
+			continue //we want to close the zeroth reader last
 		}
-		pkt := p[i].(*vhostnet.VirtIOPacket) //todo I'm not happy about this but I don't want to change how memory is "owned" rn
-		_, err = t.vdev[q].ProcessRxChain(pkt, item)
+		err := t.readers[i].Close()
 		if err != nil {
-			return i, err
+			t.l.WithField("reader", i).WithError(err).Error("error closing tun reader")
+		} else {
+			t.l.WithField("reader", i).Info("closed tun reader")
 		}
-		i++
 	}
-	return i, nil
-}
 
-func (t *tun) Write(b []byte) (int, error) {
-	maximum := len(b) //we are RXing
-
-	//todo garbagey
-	out := packet.NewOut()
-	x, err := t.AllocSeg(out, 0)
+	//this is t.readers[0] too
+	err := t.tunFile.Close()
 	if err != nil {
-		return 0, err
+		t.l.WithField("reader", 0).WithError(err).Error("error closing tun reader")
+	} else {
+		t.l.WithField("reader", 0).Info("closed tun reader")
 	}
-	copy(out.SegmentPayloads[x], b)
-	err = t.vdev[0].TransmitPacket(out, true)
-
-	if err != nil {
-		t.l.WithError(err).Error("Transmitting packet")
-		return 0, err
-	}
-	return maximum, nil
-}
-
-func (t *tun) AllocSeg(pkt *packet.OutPacket, q int) (int, error) {
-	idx, buf, err := t.vdev[q].GetPacketForTx()
-	if err != nil {
-		return 0, err
-	}
-	x := pkt.UseSegment(idx, buf, t.isV6)
-	return x, nil
-}
-
-func (t *tun) WriteOne(x *packet.OutPacket, kick bool, q int) (int, error) {
-	if err := t.vdev[q].TransmitPacket(x, kick); err != nil {
-		t.l.WithError(err).Error("Transmitting packet")
-		return 0, err
-	}
-	return 1, nil
-}
-
-func (t *tun) WriteMany(x []*packet.OutPacket, q int) (int, error) {
-	maximum := len(x) //we are RXing
-	if maximum == 0 {
-		return 0, nil
-	}
-
-	err := t.vdev[q].TransmitPackets(x)
-	if err != nil {
-		t.l.WithError(err).Error("Transmitting packet")
-		return 0, err
-	}
-	return maximum, nil
-}
-
-func (t *tun) RecycleRxSeg(pkt TunPacket, kick bool, q int) error {
-	if pkt.GetPayload() == nil {
-		return nil
-	}
-	vpkt := pkt.(*vhostnet.VirtIOPacket)
-	err := t.vdev[q].ReceiveQueue.OfferDescriptorChains([]uint16{vpkt.Chain}, kick)
-	vpkt.Reset() //intentionally ignoring err!
 	return err
 }

overlay/tun_tester.go

@@ -13,7 +13,6 @@ import (
 	"github.com/gaissmai/bart"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/packet"
 	"github.com/slackhq/nebula/routing"
 )
 
@@ -27,7 +26,6 @@ type TestTun struct {
 	closed    atomic.Bool
 	rxPackets chan []byte // Packets to receive into nebula
 	TxPackets chan []byte // Packets transmitted outside by nebula
-	buffers   [][]byte
 }
 
 func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (*TestTun, error) {
@@ -106,68 +104,15 @@ func (t *TestTun) Name() string {
 	return t.Device
 }
 
-func (t *TestTun) ReadMany(x []TunPacket, q int) (int, error) {
-	p, ok := <-t.rxPackets
-	if !ok {
-		return 0, os.ErrClosed
-	}
-	x[0].Payload = p
-	return 1, nil
-}
-
-func (t *TestTun) AllocSeg(pkt *packet.OutPacket, q int) (int, error) {
-	buf := make([]byte, 9000)
-	t.buffers = append(t.buffers, buf)
-	idx := len(t.buffers) - 1
-	isV6 := false //todo?
-	x := pkt.UseSegment(uint16(idx), buf, isV6)
-	return x, nil
-}
-
-func (t *TestTun) Write(b []byte) (int, error) {
-	//todo garbagey
-	out := packet.NewOut()
-	x, err := t.AllocSeg(out, 0)
-	if err != nil {
-		return 0, err
-	}
-	copy(out.SegmentPayloads[x], b)
-	return t.WriteOne(out, true, 0)
-}
-
-func (t *TestTun) WriteOne(x *packet.OutPacket, kick bool, q int) (int, error) {
+func (t *TestTun) Write(b []byte) (n int, err error) {
 	if t.closed.Load() {
 		return 0, io.ErrClosedPipe
 	}
-	if len(x.SegmentIDs) == 0 {
-		return 0, nil
-	}
-	for i, _ := range x.SegmentIDs {
-		t.TxPackets <- x.SegmentPayloads[i]
-	}
-	//todo if kick, delete alloced seg
 
-	return 1, nil
-}
-
-func (t *TestTun) WriteMany(x []*packet.OutPacket, q int) (int, error) {
-	if len(x) == 0 {
-		return 0, nil
-	}
-
-	for _, pkt := range x {
-		_, err := t.WriteOne(pkt, true, q)
-		if err != nil {
-			return 0, err
-		}
-	}
-
-	return len(x), nil
-}
-
-func (t *TestTun) RecycleRxSeg(pkt *TunPacket, kick bool, q int) error {
-	//todo this ought to maybe track something
-	return nil
+	packet := make([]byte, len(b), len(b))
+	copy(packet, b)
+	t.TxPackets <- packet
+	return len(b), nil
 }
 
 func (t *TestTun) Close() error {
@@ -178,10 +123,19 @@ func (t *TestTun) Close() error {
 	return nil
 }
 
+func (t *TestTun) Read(b []byte) (int, error) {
+	p, ok := <-t.rxPackets
+	if !ok {
+		return 0, os.ErrClosed
+	}
+	copy(b, p)
+	return len(p), nil
+}
+
 func (t *TestTun) SupportsMultiqueue() bool {
 	return false
 }
 
-func (t *TestTun) NewMultiQueueReader() (TunDev, error) {
+func (t *TestTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented")
 }

overlay/tun_windows.go

@@ -74,7 +74,10 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		l.WithError(err).Debug("Failed to create wintun device, retrying")
 		tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU)
 		if err != nil {
-			return nil, fmt.Errorf("create TUN device failed: %w", err)
+			return nil, &NameError{
+				Name:       deviceName,
+				Underlying: fmt.Errorf("create TUN device failed: %w", err),
+			}
 		}
 	}
 	t.tun = tunDevice.(*wintun.NativeTun)

overlay/user.go

@@ -1,13 +1,11 @@
 package overlay
 
 import (
-	"fmt"
 	"io"
 	"net/netip"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/packet"
 	"github.com/slackhq/nebula/routing"
 )
 
@@ -38,21 +36,6 @@ type UserDevice struct {
 	inboundWriter *io.PipeWriter
 }
 
-func (d *UserDevice) NewPacketArrays(batchSize int) []TunPacket {
-	//inPackets := make([]TunPacket, batchSize)
-	//outPackets := make([]OutPacket, batchSize)
-	panic("not implemented") //todo!
-	//for i := 0; i < batchSize; i++ {
-	//	inPackets[i] = vhostnet.NewVIO()
-	//	outPackets[i] = packet.New(false)
-	//}
-	//return inPackets, outPackets
-}
-
-func (d *UserDevice) RecycleRxSeg(pkt TunPacket, kick bool, q int) error {
-	return nil
-}
-
 func (d *UserDevice) Activate() error {
 	return nil
 }
@@ -67,7 +50,7 @@ func (d *UserDevice) SupportsMultiqueue() bool {
 	return true
 }
 
-func (d *UserDevice) NewMultiQueueReader() (TunDev, error) {
+func (d *UserDevice) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return d, nil
 }
 
@@ -86,23 +69,3 @@ func (d *UserDevice) Close() error {
 	d.outboundWriter.Close()
 	return nil
 }
-
-func (d *UserDevice) ReadMany(b []TunPacket, _ int) (int, error) {
-	_, err := d.Read(b[0].GetPayload())
-	if err != nil {
-		return 0, err
-	}
-	return 1, nil
-}
-
-func (d *UserDevice) AllocSeg(pkt *packet.OutPacket, q int) (int, error) {
-	return 0, fmt.Errorf("user: AllocSeg not implemented")
-}
-
-func (d *UserDevice) WriteOne(x *packet.OutPacket, kick bool, q int) (int, error) {
-	return 0, fmt.Errorf("user: WriteOne not implemented")
-}
-
-func (d *UserDevice) WriteMany(x []*packet.OutPacket, q int) (int, error) {
-	return 0, fmt.Errorf("user: WriteMany not implemented")
-}

overlay/vhost/README.md

@@ -1,23 +0,0 @@
-Significant portions of this code are derived from https://pkg.go.dev/github.com/hetznercloud/virtio-go
-
-MIT License
-
-Copyright (c) 2025 Hetzner Cloud GmbH
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

overlay/vhost/doc.go

@@ -1,4 +0,0 @@
-// Package vhost implements the basic ioctl requests needed to interact with the
-// kernel-level virtio server that provides accelerated virtio devices for
-// networking and more.
-package vhost

overlay/vhost/ioctl.go

@@ -1,218 +0,0 @@
-package vhost
-
-import (
-	"fmt"
-	"unsafe"
-
-	"github.com/slackhq/nebula/overlay/virtqueue"
-	"github.com/slackhq/nebula/util/virtio"
-	"golang.org/x/sys/unix"
-)
-
-const (
-	// vhostIoctlGetFeatures can be used to retrieve the features supported by
-	// the vhost implementation in the kernel.
-	//
-	// Response payload: [virtio.Feature]
-	// Kernel name: VHOST_GET_FEATURES
-	vhostIoctlGetFeatures = 0x8008af00
-
-	// vhostIoctlSetFeatures can be used to communicate the features supported
-	// by this virtio implementation to the kernel.
-	//
-	// Request payload: [virtio.Feature]
-	// Kernel name: VHOST_SET_FEATURES
-	vhostIoctlSetFeatures = 0x4008af00
-
-	// vhostIoctlSetOwner can be used to set the current process as the
-	// exclusive owner of a control file descriptor.
-	//
-	// Request payload: none
-	// Kernel name: VHOST_SET_OWNER
-	vhostIoctlSetOwner = 0x0000af01
-
-	// vhostIoctlSetMemoryLayout can be used to set up or modify the memory
-	// layout which describes the IOTLB mappings in the kernel.
-	//
-	// Request payload: [MemoryLayout] with custom serialization
-	// Kernel name: VHOST_SET_MEM_TABLE
-	vhostIoctlSetMemoryLayout = 0x4008af03
-
-	// vhostIoctlSetQueueSize can be used to set the size of the virtqueue.
-	//
-	// Request payload: [QueueState]
-	// Kernel name: VHOST_SET_VRING_NUM
-	vhostIoctlSetQueueSize = 0x4008af10
-
-	// vhostIoctlSetQueueAddress can be used to set the addresses of the
-	// different parts of the virtqueue.
-	//
-	// Request payload: [QueueAddresses]
-	// Kernel name: VHOST_SET_VRING_ADDR
-	vhostIoctlSetQueueAddress = 0x4028af11
-
-	// vhostIoctlSetAvailableRingBase can be used to set the index of the next
-	// available ring entry the device will process.
-	//
-	// Request payload: [QueueState]
-	// Kernel name: VHOST_SET_VRING_BASE
-	vhostIoctlSetAvailableRingBase = 0x4008af12
-
-	// vhostIoctlSetQueueKickEventFD can be used to set the event file
-	// descriptor to signal the device when descriptor chains were added to the
-	// available ring.
-	//
-	// Request payload: [QueueFile]
-	// Kernel name: VHOST_SET_VRING_KICK
-	vhostIoctlSetQueueKickEventFD = 0x4008af20
-
-	// vhostIoctlSetQueueCallEventFD can be used to set the event file
-	// descriptor that gets signaled by the device when descriptor chains have
-	// been used by it.
-	//
-	// Request payload: [QueueFile]
-	// Kernel name: VHOST_SET_VRING_CALL
-	vhostIoctlSetQueueCallEventFD = 0x4008af21
-)
-
-// QueueState is an ioctl request payload that can hold a queue index and any
-// 32-bit number.
-//
-// Kernel name: vhost_vring_state
-type QueueState struct {
-	// QueueIndex is the index of the virtqueue.
-	QueueIndex uint32
-	// Num is any 32-bit number, depending on the request.
-	Num uint32
-}
-
-// QueueAddresses is an ioctl request payload that can hold the addresses of the
-// different parts of a virtqueue.
-//
-// Kernel name: vhost_vring_addr
-type QueueAddresses struct {
-	// QueueIndex is the index of the virtqueue.
-	QueueIndex uint32
-	// Flags that are not used in this implementation.
-	Flags uint32
-	// DescriptorTableAddress is the address of the descriptor table in user
-	// space memory. It must be 16-byte aligned.
-	DescriptorTableAddress uintptr
-	// UsedRingAddress is the address of the used ring in user space memory. It
-	// must be 4-byte aligned.
-	UsedRingAddress uintptr
-	// AvailableRingAddress is the address of the available ring in user space
-	// memory. It must be 2-byte aligned.
-	AvailableRingAddress uintptr
-	// LogAddress is used for an optional logging support, not supported by this
-	// implementation.
-	LogAddress uintptr
-}
-
-// QueueFile is an ioctl request payload that can hold a queue index and a file
-// descriptor.
-//
-// Kernel name: vhost_vring_file
-type QueueFile struct {
-	// QueueIndex is the index of the virtqueue.
-	QueueIndex uint32
-	// FD is the file descriptor of the file. Pass -1 to unbind from a file.
-	FD int32
-}
-
-// IoctlPtr is a copy of the similarly named unexported function from the Go
-// unix package. This is needed to do custom ioctl requests not supported by the
-// standard library.
-func IoctlPtr(fd int, req uint, arg unsafe.Pointer) error {
-	_, _, err := unix.Syscall(unix.SYS_IOCTL, uintptr(fd), uintptr(req), uintptr(arg))
-	if err != 0 {
-		return fmt.Errorf("ioctl request %d: %w", req, err)
-	}
-	return nil
-}
-
-// GetFeatures requests the supported feature bits from the virtio device
-// associated with the given control file descriptor.
-func GetFeatures(controlFD int) (virtio.Feature, error) {
-	var features virtio.Feature
-	if err := IoctlPtr(controlFD, vhostIoctlGetFeatures, unsafe.Pointer(&features)); err != nil {
-		return 0, fmt.Errorf("get features: %w", err)
-	}
-	return features, nil
-}
-
-// SetFeatures communicates the feature bits supported by this implementation
-// to the virtio device associated with the given control file descriptor.
-func SetFeatures(controlFD int, features virtio.Feature) error {
-	if err := IoctlPtr(controlFD, vhostIoctlSetFeatures, unsafe.Pointer(&features)); err != nil {
-		return fmt.Errorf("set features: %w", err)
-	}
-	return nil
-}
-
-// OwnControlFD sets the current process as the exclusive owner for the
-// given control file descriptor. This must be called before interacting with
-// the control file descriptor in any other way.
-func OwnControlFD(controlFD int) error {
-	if err := IoctlPtr(controlFD, vhostIoctlSetOwner, unsafe.Pointer(nil)); err != nil {
-		return fmt.Errorf("set control file descriptor owner: %w", err)
-	}
-	return nil
-}
-
-// SetMemoryLayout sets up or modifies the memory layout for the kernel-level
-// virtio device associated with the given control file descriptor.
-func SetMemoryLayout(controlFD int, layout MemoryLayout) error {
-	payload := layout.serializePayload()
-	if err := IoctlPtr(controlFD, vhostIoctlSetMemoryLayout, unsafe.Pointer(&payload[0])); err != nil {
-		return fmt.Errorf("set memory layout: %w", err)
-	}
-	return nil
-}
-
-// RegisterQueue registers a virtio queue with the kernel-level virtio server.
-// The virtqueue will be linked to the given control file descriptor and will
-// have the given index. The kernel will use this queue until the control file
-// descriptor is closed.
-func RegisterQueue(controlFD int, queueIndex uint32, queue *virtqueue.SplitQueue) error {
-	if err := IoctlPtr(controlFD, vhostIoctlSetQueueSize, unsafe.Pointer(&QueueState{
-		QueueIndex: queueIndex,
-		Num:        uint32(queue.Size()),
-	})); err != nil {
-		return fmt.Errorf("set queue size: %w", err)
-	}
-
-	if err := IoctlPtr(controlFD, vhostIoctlSetQueueAddress, unsafe.Pointer(&QueueAddresses{
-		QueueIndex:             queueIndex,
-		Flags:                  0,
-		DescriptorTableAddress: queue.DescriptorTable().Address(),
-		UsedRingAddress:        queue.UsedRing().Address(),
-		AvailableRingAddress:   queue.AvailableRing().Address(),
-		LogAddress:             0,
-	})); err != nil {
-		return fmt.Errorf("set queue addresses: %w", err)
-	}
-
-	if err := IoctlPtr(controlFD, vhostIoctlSetAvailableRingBase, unsafe.Pointer(&QueueState{
-		QueueIndex: queueIndex,
-		Num:        0,
-	})); err != nil {
-		return fmt.Errorf("set available ring base: %w", err)
-	}
-
-	if err := IoctlPtr(controlFD, vhostIoctlSetQueueKickEventFD, unsafe.Pointer(&QueueFile{
-		QueueIndex: queueIndex,
-		FD:         int32(queue.KickEventFD()),
-	})); err != nil {
-		return fmt.Errorf("set kick event file descriptor: %w", err)
-	}
-
-	if err := IoctlPtr(controlFD, vhostIoctlSetQueueCallEventFD, unsafe.Pointer(&QueueFile{
-		QueueIndex: queueIndex,
-		FD:         int32(queue.CallEventFD()),
-	})); err != nil {
-		return fmt.Errorf("set call event file descriptor: %w", err)
-	}
-
-	return nil
-}

overlay/vhost/ioctl_test.go

@@ -1,21 +0,0 @@
-package vhost_test
-
-import (
-	"testing"
-	"unsafe"
-
-	"github.com/slackhq/nebula/overlay/vhost"
-	"github.com/stretchr/testify/assert"
-)
-
-func TestQueueState_Size(t *testing.T) {
-	assert.EqualValues(t, 8, unsafe.Sizeof(vhost.QueueState{}))
-}
-
-func TestQueueAddresses_Size(t *testing.T) {
-	assert.EqualValues(t, 40, unsafe.Sizeof(vhost.QueueAddresses{}))
-}
-
-func TestQueueFile_Size(t *testing.T) {
-	assert.EqualValues(t, 8, unsafe.Sizeof(vhost.QueueFile{}))
-}

overlay/vhost/memory.go

@@ -1,73 +0,0 @@
-package vhost
-
-import (
-	"encoding/binary"
-	"fmt"
-	"unsafe"
-
-	"github.com/slackhq/nebula/overlay/virtqueue"
-)
-
-// MemoryRegion describes a region of userspace memory which is being made
-// accessible to a vhost device.
-//
-// Kernel name: vhost_memory_region
-type MemoryRegion struct {
-	// GuestPhysicalAddress is the physical address of the memory region within
-	// the guest, when virtualization is used. When no virtualization is used,
-	// this should be the same as UserspaceAddress.
-	GuestPhysicalAddress uintptr
-	// Size is the size of the memory region.
-	Size uint64
-	// UserspaceAddress is the virtual address in the userspace of the host
-	// where the memory region can be found.
-	UserspaceAddress uintptr
-	// Padding and room for flags. Currently unused.
-	_ uint64
-}
-
-// MemoryLayout is a list of [MemoryRegion]s.
-type MemoryLayout []MemoryRegion
-
-// NewMemoryLayoutForQueues returns a new [MemoryLayout] that describes the
-// memory pages used by the descriptor tables of the given queues.
-func NewMemoryLayoutForQueues(queues []*virtqueue.SplitQueue) MemoryLayout {
-	regions := make([]MemoryRegion, 0)
-	for _, queue := range queues {
-		for address, size := range queue.DescriptorTable().BufferAddresses() {
-			regions = append(regions, MemoryRegion{
-				// There is no virtualization in play here, so the guest address
-				// is the same as in the host's userspace.
-				GuestPhysicalAddress: address,
-				Size:                 uint64(size),
-				UserspaceAddress:     address,
-			})
-		}
-	}
-	return regions
-}
-
-// serializePayload serializes the list of memory regions into a format that is
-// compatible to the vhost_memory kernel struct. The returned byte slice can be
-// used as a payload for the vhostIoctlSetMemoryLayout ioctl.
-func (regions MemoryLayout) serializePayload() []byte {
-	regionCount := len(regions)
-	regionSize := int(unsafe.Sizeof(MemoryRegion{}))
-	payload := make([]byte, 8+regionCount*regionSize)
-
-	// The first 32 bits contain the number of memory regions. The following 32
-	// bits are padding.
-	binary.LittleEndian.PutUint32(payload[0:4], uint32(regionCount))
-
-	if regionCount > 0 {
-		// The underlying byte array of the slice should already have the correct
-		// format, so just copy that.
-		copied := copy(payload[8:], unsafe.Slice((*byte)(unsafe.Pointer(&regions[0])), regionCount*regionSize))
-		if copied != regionCount*regionSize {
-			panic(fmt.Sprintf("copied only %d bytes of the memory regions, but expected %d",
-				copied, regionCount*regionSize))
-		}
-	}
-
-	return payload
-}

overlay/vhost/memory_internal_test.go

@@ -1,42 +0,0 @@
-package vhost
-
-import (
-	"testing"
-	"unsafe"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestMemoryRegion_Size(t *testing.T) {
-	assert.EqualValues(t, 32, unsafe.Sizeof(MemoryRegion{}))
-}
-
-func TestMemoryLayout_SerializePayload(t *testing.T) {
-	layout := MemoryLayout([]MemoryRegion{
-		{
-			GuestPhysicalAddress: 42,
-			Size:                 100,
-			UserspaceAddress:     142,
-		}, {
-			GuestPhysicalAddress: 99,
-			Size:                 100,
-			UserspaceAddress:     99,
-		},
-	})
-	payload := layout.serializePayload()
-
-	assert.Equal(t, []byte{
-		0x02, 0x00, 0x00, 0x00, // nregions
-		0x00, 0x00, 0x00, 0x00, // padding
-		// region 0
-		0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // guest_phys_addr
-		0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // memory_size
-		0x8e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // userspace_addr
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // flags_padding
-		// region 1
-		0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // guest_phys_addr
-		0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // memory_size
-		0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // userspace_addr
-		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // flags_padding
-	}, payload)
-}

overlay/vhostnet/README.md

@@ -1,23 +0,0 @@
-Significant portions of this code are derived from https://pkg.go.dev/github.com/hetznercloud/virtio-go
-
-MIT License
-
-Copyright (c) 2025 Hetzner Cloud GmbH
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

overlay/vhostnet/device.go

@@ -1,339 +0,0 @@
-package vhostnet
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"os"
-	"runtime"
-
-	"github.com/slackhq/nebula/overlay/vhost"
-	"github.com/slackhq/nebula/overlay/virtqueue"
-	"github.com/slackhq/nebula/packet"
-	"github.com/slackhq/nebula/util/virtio"
-	"golang.org/x/sys/unix"
-)
-
-// ErrDeviceClosed is returned when the [Device] is closed while operations are
-// still running.
-var ErrDeviceClosed = errors.New("device was closed")
-
-// The indexes for the receive and transmit queues.
-const (
-	receiveQueueIndex  = 0
-	transmitQueueIndex = 1
-)
-
-// Device represents a vhost networking device within the kernel-level virtio
-// implementation and provides methods to interact with it.
-type Device struct {
-	controlFD int
-
-	ReceiveQueue  *virtqueue.SplitQueue
-	TransmitQueue *virtqueue.SplitQueue
-}
-
-// NewDevice initializes a new vhost networking device within the
-// kernel-level virtio implementation, sets up the virtqueues and returns a
-// [Device] instance that can be used to communicate with that vhost device.
-//
-// There are multiple options that can be passed to this constructor to
-// influence device creation:
-//   - [WithQueueSize]
-//   - [WithBackendFD]
-//   - [WithBackendDevice]
-//
-// Remember to call [Device.Close] after use to free up resources.
-func NewDevice(options ...Option) (*Device, error) {
-	var err error
-	opts := optionDefaults
-	opts.apply(options)
-	if err = opts.validate(); err != nil {
-		return nil, fmt.Errorf("invalid options: %w", err)
-	}
-
-	dev := Device{
-		controlFD: -1,
-	}
-
-	// Clean up a partially initialized device when something fails.
-	defer func() {
-		if err != nil {
-			_ = dev.Close()
-		}
-	}()
-
-	// Retrieve a new control file descriptor. This will be used to configure
-	// the vhost networking device in the kernel.
-	dev.controlFD, err = unix.Open("/dev/vhost-net", os.O_RDWR, 0666)
-	if err != nil {
-		return nil, fmt.Errorf("get control file descriptor: %w", err)
-	}
-	if err = vhost.OwnControlFD(dev.controlFD); err != nil {
-		return nil, fmt.Errorf("own control file descriptor: %w", err)
-	}
-
-	// Advertise the supported features. This isn't much for now.
-	// TODO: Add feature options and implement proper feature negotiation.
-	getFeatures, err := vhost.GetFeatures(dev.controlFD) //0x1033D008000 but why
-	if err != nil {
-		return nil, fmt.Errorf("get features: %w", err)
-	}
-	if getFeatures == 0 {
-
-	}
-	//const funky = virtio.Feature(1 << 27)
-	//features := virtio.FeatureVersion1 | funky // | todo virtio.FeatureNetMergeRXBuffers
-	features := virtio.FeatureVersion1 | virtio.FeatureNetMergeRXBuffers
-	if err = vhost.SetFeatures(dev.controlFD, features); err != nil {
-		return nil, fmt.Errorf("set features: %w", err)
-	}
-
-	itemSize := os.Getpagesize() * 4 //todo config
-
-	// Initialize and register the queues needed for the networking device.
-	if dev.ReceiveQueue, err = createQueue(dev.controlFD, receiveQueueIndex, opts.queueSize, itemSize); err != nil {
-		return nil, fmt.Errorf("create receive queue: %w", err)
-	}
-	if dev.TransmitQueue, err = createQueue(dev.controlFD, transmitQueueIndex, opts.queueSize, itemSize); err != nil {
-		return nil, fmt.Errorf("create transmit queue: %w", err)
-	}
-
-	// Set up memory mappings for all buffers used by the queues. This has to
-	// happen before a backend for the queues can be registered.
-	memoryLayout := vhost.NewMemoryLayoutForQueues(
-		[]*virtqueue.SplitQueue{dev.ReceiveQueue, dev.TransmitQueue},
-	)
-	if err = vhost.SetMemoryLayout(dev.controlFD, memoryLayout); err != nil {
-		return nil, fmt.Errorf("setup memory layout: %w", err)
-	}
-
-	// Set the queue backends. This activates the queues within the kernel.
-	if err = SetQueueBackend(dev.controlFD, receiveQueueIndex, opts.backendFD); err != nil {
-		return nil, fmt.Errorf("set receive queue backend: %w", err)
-	}
-	if err = SetQueueBackend(dev.controlFD, transmitQueueIndex, opts.backendFD); err != nil {
-		return nil, fmt.Errorf("set transmit queue backend: %w", err)
-	}
-
-	// Fully populate the rx queue with available buffers which the device
-	// can write new packets into.
-	if err = dev.refillReceiveQueue(); err != nil {
-		return nil, fmt.Errorf("refill receive queue: %w", err)
-	}
-	if err = dev.prefillTxQueue(); err != nil {
-		return nil, fmt.Errorf("prefill tx queue: %w", err)
-	}
-
-	// Make sure to clean up even when the device gets garbage collected without
-	// Close being called first.
-	devPtr := &dev
-	runtime.SetFinalizer(devPtr, (*Device).Close)
-
-	return devPtr, nil
-}
-
-// refillReceiveQueue offers as many new device-writable buffers to the device
-// as the queue can fit. The device will then use these to write received
-// packets.
-func (dev *Device) refillReceiveQueue() error {
-	for {
-		_, err := dev.ReceiveQueue.OfferInDescriptorChains()
-		if err != nil {
-			if errors.Is(err, virtqueue.ErrNotEnoughFreeDescriptors) {
-				// Queue is full, job is done.
-				return nil
-			}
-			return fmt.Errorf("offer descriptor chain: %w", err)
-		}
-	}
-}
-
-func (dev *Device) prefillTxQueue() error {
-	for {
-		dt := dev.TransmitQueue.DescriptorTable()
-		for {
-			x, _, err := dt.CreateDescriptorForOutputs()
-			if err != nil {
-				if errors.Is(err, virtqueue.ErrNotEnoughFreeDescriptors) {
-					// Queue is full, job is done.
-					return nil
-				}
-				return err
-			}
-			err = dev.TransmitQueue.OfferDescriptorChains([]uint16{x}, false)
-			if err != nil {
-				return err
-			}
-		}
-
-	}
-}
-
-// Close cleans up the vhost networking device within the kernel and releases
-// all resources used for it.
-// The implementation will try to release as many resources as possible and
-// collect potential errors before returning them.
-func (dev *Device) Close() error {
-	// Closing the control file descriptor will unregister all queues from the
-	// kernel.
-	if dev.controlFD >= 0 {
-		if err := unix.Close(dev.controlFD); err != nil {
-			// Return an error and do not continue, because the memory used for
-			// the queues should not be released before they were unregistered
-			// from the kernel.
-			return fmt.Errorf("close control file descriptor: %w", err)
-		}
-		dev.controlFD = -1
-	}
-
-	var errs []error
-
-	if dev.ReceiveQueue != nil {
-		if err := dev.ReceiveQueue.Close(); err == nil {
-			dev.ReceiveQueue = nil
-		} else {
-			errs = append(errs, fmt.Errorf("close receive queue: %w", err))
-		}
-	}
-
-	if dev.TransmitQueue != nil {
-		if err := dev.TransmitQueue.Close(); err == nil {
-			dev.TransmitQueue = nil
-		} else {
-			errs = append(errs, fmt.Errorf("close transmit queue: %w", err))
-		}
-	}
-
-	if len(errs) == 0 {
-		// Everything was cleaned up. No need to run the finalizer anymore.
-		runtime.SetFinalizer(dev, nil)
-	}
-
-	return errors.Join(errs...)
-}
-
-// createQueue creates a new virtqueue and registers it with the vhost device
-// using the given index.
-func createQueue(controlFD int, queueIndex int, queueSize int, itemSize int) (*virtqueue.SplitQueue, error) {
-	queue, err := virtqueue.NewSplitQueue(queueSize, itemSize)
-	if err != nil {
-		return nil, fmt.Errorf("create virtqueue: %w", err)
-	}
-	if err = vhost.RegisterQueue(controlFD, uint32(queueIndex), queue); err != nil {
-		return nil, fmt.Errorf("register virtqueue with index %d: %w", queueIndex, err)
-	}
-	return queue, nil
-}
-
-func (dev *Device) GetPacketForTx() (uint16, []byte, error) {
-	idx, err := dev.TransmitQueue.TakeSingleIndex(context.TODO())
-	if err != nil {
-		return 0, nil, fmt.Errorf("transmit queue: %w", err)
-	}
-	buf := dev.TransmitQueue.GetDescriptorItem(idx)
-	return idx, buf, nil
-}
-
-func (dev *Device) TransmitPacket(pkt *packet.OutPacket, kick bool) error {
-	if len(pkt.SegmentIDs) == 0 {
-		return nil
-	}
-	for idx := range pkt.SegmentIDs {
-		segmentID := pkt.SegmentIDs[idx]
-		dev.TransmitQueue.SetDescSize(segmentID, len(pkt.Segments[idx]))
-	}
-	err := dev.TransmitQueue.OfferDescriptorChains(pkt.SegmentIDs, false)
-	if err != nil {
-		return fmt.Errorf("offer descriptor chains: %w", err)
-	}
-	pkt.Reset()
-	if kick {
-		return dev.TransmitQueue.Kick()
-	}
-
-	return nil
-}
-
-func (dev *Device) TransmitPackets(pkts []*packet.OutPacket) error {
-	if len(pkts) == 0 {
-		return nil
-	}
-
-	for i := range pkts {
-		if err := dev.TransmitPacket(pkts[i], false); err != nil {
-			return err
-		}
-	}
-	if err := dev.TransmitQueue.Kick(); err != nil {
-		return err
-	}
-	return nil
-}
-
-// ProcessRxChain processes a single chain to create one packet. The number of processed chains is returned.
-func (dev *Device) ProcessRxChain(pkt *VirtIOPacket, chain virtqueue.UsedElement) (int, error) {
-	//read first element to see how many descriptors we need:
-	pkt.Chain = uint16(chain.DescriptorIndex)
-	buf := dev.ReceiveQueue.GetDescriptorItem(pkt.Chain)
-
-	// The specification requires that the first descriptor chain starts
-	// with a virtio-net header. It is not clear, whether it is also
-	// required to be fully contained in the first buffer of that
-	// descriptor chain, but it is reasonable to assume that this is
-	// always the case.
-	// The decode method already does the buffer length check.
-
-	//HACK: we only want the last bit of the header, the NumBuffers field. So, let's grab just that:
-	//numBuffers := binary.BigEndian.Uint16(buf[virtio.NetHdrSize-3:])
-	//even bigger hack: apparently this is hitting some kind of memory access pitfall? Let's only grab the last byte:
-	//numBuffers := buf[virtio.NetHdrSize-2]
-
-	//if err = pkt.header.Decode(buf); err != nil {
-	//	// The device misbehaved. There is no way we can gracefully
-	//	// recover from this, because we don't know how many of the
-	//	// following descriptor chains belong to this packet.
-	//	return 0, fmt.Errorf("decode vnethdr: %w", err)
-	//}
-
-	//we have the header now: what do we need to do?
-	//todo we're ignoring the header lol
-	//if int(numBuffers) != 1 {
-	//	return 0, fmt.Errorf("too smol-brain to handle more than one buffer per Chain item right now: %d chains, %d bufs", 1, int(numBuffers))
-	//}
-
-	if chain.Length > 16000 {
-		//todo!
-		pkt.payload = nil
-		return 1, fmt.Errorf("too big packet length: %d", chain.Length)
-	}
-
-	//shift the buffer out of out:
-	pkt.payload = buf[virtio.NetHdrSize:chain.Length]
-	return 1, nil
-}
-
-type VirtIOPacket struct {
-	payload []byte
-	//header  virtio.NetHdr
-	Chain uint16
-}
-
-func NewVIO() *VirtIOPacket {
-	out := new(VirtIOPacket)
-	out.payload = nil
-	out.Chain = 0
-	return out
-}
-
-func (v *VirtIOPacket) Reset() {
-	v.payload = nil
-	v.Chain = 0
-}
-
-func (v *VirtIOPacket) GetPayload() []byte {
-	return v.payload //todo this could be dev.ReceiveQueue.GetDescriptorItem(idx)
-}
-func (v *VirtIOPacket) SetPayload(x []byte) {
-	v.payload = x
-}

overlay/vhostnet/doc.go

@@ -1,3 +0,0 @@
-// Package vhostnet implements methods to initialize vhost networking devices
-// within the kernel-level virtio implementation and communicate with them.
-package vhostnet

overlay/vhostnet/ioctl.go

@@ -1,31 +0,0 @@
-package vhostnet
-
-import (
-	"fmt"
-	"unsafe"
-
-	"github.com/slackhq/nebula/overlay/vhost"
-)
-
-const (
-	// vhostNetIoctlSetBackend can be used to attach a virtqueue to a RAW socket
-	// or TAP device.
-	//
-	// Request payload: [vhost.QueueFile]
-	// Kernel name: VHOST_NET_SET_BACKEND
-	vhostNetIoctlSetBackend = 0x4008af30
-)
-
-// SetQueueBackend attaches a virtqueue of the vhost networking device
-// described by controlFD to the given backend file descriptor.
-// The backend file descriptor can either be a RAW socket or a TAP device. When
-// it is -1, the queue will be detached.
-func SetQueueBackend(controlFD int, queueIndex uint32, backendFD int) error {
-	if err := vhost.IoctlPtr(controlFD, vhostNetIoctlSetBackend, unsafe.Pointer(&vhost.QueueFile{
-		QueueIndex: queueIndex,
-		FD:         int32(backendFD),
-	})); err != nil {
-		return fmt.Errorf("set queue backend file descriptor: %w", err)
-	}
-	return nil
-}

overlay/vhostnet/options.go

@@ -1,69 +0,0 @@
-package vhostnet
-
-import (
-	"errors"
-
-	"github.com/slackhq/nebula/overlay/virtqueue"
-)
-
-type optionValues struct {
-	queueSize int
-	backendFD int
-}
-
-func (o *optionValues) apply(options []Option) {
-	for _, option := range options {
-		option(o)
-	}
-}
-
-func (o *optionValues) validate() error {
-	if o.queueSize == -1 {
-		return errors.New("queue size is required")
-	}
-	if err := virtqueue.CheckQueueSize(o.queueSize); err != nil {
-		return err
-	}
-	if o.backendFD == -1 {
-		return errors.New("backend file descriptor is required")
-	}
-	return nil
-}
-
-var optionDefaults = optionValues{
-	// Required.
-	queueSize: -1,
-	// Required.
-	backendFD: -1,
-}
-
-// Option can be passed to [NewDevice] to influence device creation.
-type Option func(*optionValues)
-
-// WithQueueSize returns an [Option] that sets the size of the TX and RX queues
-// that are to be created for the device. It specifies the number of
-// entries/buffers each queue can hold. This also affects the memory
-// consumption.
-// This is required and must be an integer from 1 to 32768 that is also a power
-// of 2.
-func WithQueueSize(queueSize int) Option {
-	return func(o *optionValues) { o.queueSize = queueSize }
-}
-
-// WithBackendFD returns an [Option] that sets the file descriptor of the
-// backend that will be used for the queues of the device. The device will write
-// and read packets to/from that backend. The file descriptor can either be of a
-// RAW socket or TUN/TAP device.
-// Either this or [WithBackendDevice] is required.
-func WithBackendFD(backendFD int) Option {
-	return func(o *optionValues) { o.backendFD = backendFD }
-}
-
-//// WithBackendDevice returns an [Option] that sets the given TAP device as the
-//// backend that will be used for the queues of the device. The device will
-//// write and read packets to/from that backend. The TAP device should have been
-//// created with the [tuntap.WithVirtioNetHdr] option enabled.
-//// Either this or [WithBackendFD] is required.
-//func WithBackendDevice(dev *tuntap.Device) Option {
-//	return func(o *optionValues) { o.backendFD = int(dev.File().Fd()) }
-//}

overlay/virtqueue/README.md

@@ -1,23 +0,0 @@
-Significant portions of this code are derived from https://pkg.go.dev/github.com/hetznercloud/virtio-go
-
-MIT License
-
-Copyright (c) 2025 Hetzner Cloud GmbH
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

overlay/virtqueue/available_ring.go

@@ -1,140 +0,0 @@
-package virtqueue
-
-import (
-	"fmt"
-	"unsafe"
-)
-
-// availableRingFlag is a flag that describes an [AvailableRing].
-type availableRingFlag uint16
-
-const (
-	// availableRingFlagNoInterrupt is used by the guest to advise the host to
-	// not interrupt it when consuming a buffer. It's unreliable, so it's simply
-	// an optimization.
-	availableRingFlagNoInterrupt availableRingFlag = 1 << iota
-)
-
-// availableRingSize is the number of bytes needed to store an [AvailableRing]
-// with the given queue size in memory.
-func availableRingSize(queueSize int) int {
-	return 6 + 2*queueSize
-}
-
-// availableRingAlignment is the minimum alignment of an [AvailableRing]
-// in memory, as required by the virtio spec.
-const availableRingAlignment = 2
-
-// AvailableRing is used by the driver to offer descriptor chains to the device.
-// Each ring entry refers to the head of a descriptor chain. It is only written
-// to by the driver and read by the device.
-//
-// Because the size of the ring depends on the queue size, we cannot define a
-// Go struct with a static size that maps to the memory of the ring. Instead,
-// this struct only contains pointers to the corresponding memory areas.
-type AvailableRing struct {
-	initialized bool
-
-	// flags that describe this ring.
-	flags *availableRingFlag
-	// ringIndex indicates where the driver would put the next entry into the
-	// ring (modulo the queue size).
-	ringIndex *uint16
-	// ring references buffers using the index of the head of the descriptor
-	// chain in the [DescriptorTable]. It wraps around at queue size.
-	ring []uint16
-	// usedEvent is not used by this implementation, but we reserve it anyway to
-	// avoid issues in case a device may try to access it, contrary to the
-	// virtio specification.
-	usedEvent *uint16
-}
-
-// newAvailableRing creates an available ring that uses the given underlying
-// memory. The length of the memory slice must match the size needed for the
-// ring (see [availableRingSize]) for the given queue size.
-func newAvailableRing(queueSize int, mem []byte) *AvailableRing {
-	ringSize := availableRingSize(queueSize)
-	if len(mem) != ringSize {
-		panic(fmt.Sprintf("memory size (%v) does not match required size "+
-			"for available ring: %v", len(mem), ringSize))
-	}
-
-	return &AvailableRing{
-		initialized: true,
-		flags:       (*availableRingFlag)(unsafe.Pointer(&mem[0])),
-		ringIndex:   (*uint16)(unsafe.Pointer(&mem[2])),
-		ring:        unsafe.Slice((*uint16)(unsafe.Pointer(&mem[4])), queueSize),
-		usedEvent:   (*uint16)(unsafe.Pointer(&mem[ringSize-2])),
-	}
-}
-
-// Address returns the pointer to the beginning of the ring in memory.
-// Do not modify the memory directly to not interfere with this implementation.
-func (r *AvailableRing) Address() uintptr {
-	if !r.initialized {
-		panic("available ring is not initialized")
-	}
-	return uintptr(unsafe.Pointer(r.flags))
-}
-
-// offer adds the given descriptor chain heads to the available ring and
-// advances the ring index accordingly to make the device process the new
-// descriptor chains.
-func (r *AvailableRing) offerElements(chains []UsedElement) {
-	//always called under lock
-	//r.mu.Lock()
-	//defer r.mu.Unlock()
-
-	// Add descriptor chain heads to the ring.
-	for offset, x := range chains {
-		// The 16-bit ring index may overflow. This is expected and is not an
-		// issue because the size of the ring array (which equals the queue
-		// size) is always a power of 2 and smaller than the highest possible
-		// 16-bit value.
-		insertIndex := int(*r.ringIndex+uint16(offset)) % len(r.ring)
-		r.ring[insertIndex] = x.GetHead()
-	}
-
-	// Increase the ring index by the number of descriptor chains added to the
-	// ring.
-	*r.ringIndex += uint16(len(chains))
-}
-
-func (r *AvailableRing) offer(chains []uint16) {
-	//always called under lock
-	//r.mu.Lock()
-	//defer r.mu.Unlock()
-
-	// Add descriptor chain heads to the ring.
-	for offset, x := range chains {
-		// The 16-bit ring index may overflow. This is expected and is not an
-		// issue because the size of the ring array (which equals the queue
-		// size) is always a power of 2 and smaller than the highest possible
-		// 16-bit value.
-		insertIndex := int(*r.ringIndex+uint16(offset)) % len(r.ring)
-		r.ring[insertIndex] = x
-	}
-
-	// Increase the ring index by the number of descriptor chains added to the
-	// ring.
-	*r.ringIndex += uint16(len(chains))
-}
-
-func (r *AvailableRing) offerSingle(x uint16) {
-	//always called under lock
-	//r.mu.Lock()
-	//defer r.mu.Unlock()
-
-	offset := 0
-	// Add descriptor chain heads to the ring.
-
-	// The 16-bit ring index may overflow. This is expected and is not an
-	// issue because the size of the ring array (which equals the queue
-	// size) is always a power of 2 and smaller than the highest possible
-	// 16-bit value.
-	insertIndex := int(*r.ringIndex+uint16(offset)) % len(r.ring)
-	r.ring[insertIndex] = x
-
-	// Increase the ring index by the number of descriptor chains added to the ring.
-	*r.ringIndex += 1
-}

overlay/virtqueue/available_ring_internal_test.go

@@ -1,71 +0,0 @@
-package virtqueue
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestAvailableRing_MemoryLayout(t *testing.T) {
-	const queueSize = 2
-
-	memory := make([]byte, availableRingSize(queueSize))
-	r := newAvailableRing(queueSize, memory)
-
-	*r.flags = 0x01ff
-	*r.ringIndex = 1
-	r.ring[0] = 0x1234
-	r.ring[1] = 0x5678
-
-	assert.Equal(t, []byte{
-		0xff, 0x01,
-		0x01, 0x00,
-		0x34, 0x12,
-		0x78, 0x56,
-		0x00, 0x00,
-	}, memory)
-}
-
-func TestAvailableRing_Offer(t *testing.T) {
-	const queueSize = 8
-
-	chainHeads := []uint16{42, 33, 69}
-
-	tests := []struct {
-		name              string
-		startRingIndex    uint16
-		expectedRingIndex uint16
-		expectedRing      []uint16
-	}{
-		{
-			name:              "no overflow",
-			startRingIndex:    0,
-			expectedRingIndex: 3,
-			expectedRing:      []uint16{42, 33, 69, 0, 0, 0, 0, 0},
-		},
-		{
-			name:              "ring overflow",
-			startRingIndex:    6,
-			expectedRingIndex: 9,
-			expectedRing:      []uint16{69, 0, 0, 0, 0, 0, 42, 33},
-		},
-		{
-			name:              "index overflow",
-			startRingIndex:    65535,
-			expectedRingIndex: 2,
-			expectedRing:      []uint16{33, 69, 0, 0, 0, 0, 0, 42},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			memory := make([]byte, availableRingSize(queueSize))
-			r := newAvailableRing(queueSize, memory)
-			*r.ringIndex = tt.startRingIndex
-
-			r.offer(chainHeads)
-
-			assert.Equal(t, tt.expectedRingIndex, *r.ringIndex)
-			assert.Equal(t, tt.expectedRing, r.ring)
-		})
-	}
-}

overlay/virtqueue/descriptor.go

@@ -1,43 +0,0 @@
-package virtqueue
-
-// descriptorFlag is a flag that describes a [Descriptor].
-type descriptorFlag uint16
-
-const (
-	// descriptorFlagHasNext marks a descriptor chain as continuing via the next
-	// field.
-	descriptorFlagHasNext descriptorFlag = 1 << iota
-	// descriptorFlagWritable marks a buffer as device write-only (otherwise
-	// device read-only).
-	descriptorFlagWritable
-	// descriptorFlagIndirect means the buffer contains a list of buffer
-	// descriptors to provide an additional layer of indirection.
-	// Only allowed when the [virtio.FeatureIndirectDescriptors] feature was
-	// negotiated.
-	descriptorFlagIndirect
-)
-
-// descriptorSize is the number of bytes needed to store a [Descriptor] in
-// memory.
-const descriptorSize = 16
-
-// Descriptor describes (a part of) a buffer which is either read-only for the
-// device or write-only for the device (depending on [descriptorFlagWritable]).
-// Multiple descriptors can be chained to produce a "descriptor chain" that can
-// contain both device-readable and device-writable buffers. Device-readable
-// descriptors always come first in a chain. A single, large buffer may be
-// split up by chaining multiple similar descriptors that reference different
-// memory pages. This is required, because buffers may exceed a single page size
-// and the memory accessed by the device is expected to be continuous.
-type Descriptor struct {
-	// address is the address to the continuous memory holding the data for this
-	// descriptor.
-	address uintptr
-	// length is the amount of bytes stored at address.
-	length uint32
-	// flags that describe this descriptor.
-	flags descriptorFlag
-	// next contains the index of the next descriptor continuing this descriptor
-	// chain when the [descriptorFlagHasNext] flag is set.
-	next uint16
-}

overlay/virtqueue/descriptor_internal_test.go

@@ -1,12 +0,0 @@
-package virtqueue
-
-import (
-	"testing"
-	"unsafe"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestDescriptor_Size(t *testing.T) {
-	assert.EqualValues(t, descriptorSize, unsafe.Sizeof(Descriptor{}))
-}

overlay/virtqueue/descriptor_table.go

@@ -1,288 +0,0 @@
-package virtqueue
-
-import (
-	"errors"
-	"fmt"
-	"math"
-	"unsafe"
-
-	"golang.org/x/sys/unix"
-)
-
-var (
-	// ErrNotEnoughFreeDescriptors is returned when the free descriptors are
-	// exhausted, meaning that the queue is full.
-	ErrNotEnoughFreeDescriptors = errors.New("not enough free descriptors, queue is full")
-
-	// ErrInvalidDescriptorChain is returned when a descriptor chain is not
-	// valid for a given operation.
-	ErrInvalidDescriptorChain = errors.New("invalid descriptor chain")
-)
-
-// noFreeHead is used to mark when all descriptors are in use and we have no
-// free chain. This value is impossible to occur as an index naturally, because
-// it exceeds the maximum queue size.
-const noFreeHead = uint16(math.MaxUint16)
-
-// descriptorTableSize is the number of bytes needed to store a
-// [DescriptorTable] with the given queue size in memory.
-func descriptorTableSize(queueSize int) int {
-	return descriptorSize * queueSize
-}
-
-// descriptorTableAlignment is the minimum alignment of a [DescriptorTable]
-// in memory, as required by the virtio spec.
-const descriptorTableAlignment = 16
-
-// DescriptorTable is a table that holds [Descriptor]s, addressed via their
-// index in the slice.
-type DescriptorTable struct {
-	descriptors []Descriptor
-
-	// freeHeadIndex is the index of the head of the descriptor chain which
-	// contains all currently unused descriptors. When all descriptors are in
-	// use, this has the special value of noFreeHead.
-	freeHeadIndex uint16
-	// freeNum tracks the number of descriptors which are currently not in use.
-	freeNum uint16
-
-	bufferBase uintptr
-	bufferSize int
-	itemSize   int
-}
-
-// newDescriptorTable creates a descriptor table that uses the given underlying
-// memory. The Length of the memory slice must match the size needed for the
-// descriptor table (see [descriptorTableSize]) for the given queue size.
-//
-// Before this descriptor table can be used, [initialize] must be called.
-func newDescriptorTable(queueSize int, mem []byte, itemSize int) *DescriptorTable {
-	dtSize := descriptorTableSize(queueSize)
-	if len(mem) != dtSize {
-		panic(fmt.Sprintf("memory size (%v) does not match required size "+
-			"for descriptor table: %v", len(mem), dtSize))
-	}
-
-	return &DescriptorTable{
-		descriptors: unsafe.Slice((*Descriptor)(unsafe.Pointer(&mem[0])), queueSize),
-		// We have no free descriptors until they were initialized.
-		freeHeadIndex: noFreeHead,
-		freeNum:       0,
-		itemSize:      itemSize, //todo configurable? needs to be page-aligned
-	}
-}
-
-// Address returns the pointer to the beginning of the descriptor table in
-// memory. Do not modify the memory directly to not interfere with this
-// implementation.
-func (dt *DescriptorTable) Address() uintptr {
-	if dt.descriptors == nil {
-		panic("descriptor table is not initialized")
-	}
-	//should be same as dt.bufferBase
-	return uintptr(unsafe.Pointer(&dt.descriptors[0]))
-}
-
-func (dt *DescriptorTable) Size() uintptr {
-	if dt.descriptors == nil {
-		panic("descriptor table is not initialized")
-	}
-	return uintptr(dt.bufferSize)
-}
-
-// BufferAddresses returns a map of pointer->size for all allocations used by the table
-func (dt *DescriptorTable) BufferAddresses() map[uintptr]int {
-	if dt.descriptors == nil {
-		panic("descriptor table is not initialized")
-	}
-
-	return map[uintptr]int{dt.bufferBase: dt.bufferSize}
-}
-
-// initializeDescriptors allocates buffers with the size of a full memory page
-// for each descriptor in the table. While this may be a bit wasteful, it makes
-// dealing with descriptors way easier. Without this preallocation, we would
-// have to allocate and free memory on demand, increasing complexity.
-//
-// All descriptors will be marked as free and will form a free chain. The
-// addresses of all descriptors will be populated while their length remains
-// zero.
-func (dt *DescriptorTable) initializeDescriptors() error {
-	numDescriptors := len(dt.descriptors)
-
-	// Allocate ONE large region for all buffers
-	totalSize := dt.itemSize * numDescriptors
-	basePtr, err := unix.MmapPtr(-1, 0, nil, uintptr(totalSize),
-		unix.PROT_READ|unix.PROT_WRITE,
-		unix.MAP_PRIVATE|unix.MAP_ANONYMOUS)
-	if err != nil {
-		return fmt.Errorf("allocate buffer memory for descriptors: %w", err)
-	}
-
-	// Store the base for cleanup later
-	dt.bufferBase = uintptr(basePtr)
-	dt.bufferSize = totalSize
-
-	for i := range dt.descriptors {
-		dt.descriptors[i] = Descriptor{
-			address: dt.bufferBase + uintptr(i*dt.itemSize),
-			length:  0,
-			// All descriptors should form a free chain that loops around.
-			flags: descriptorFlagHasNext,
-			next:  uint16((i + 1) % len(dt.descriptors)),
-		}
-	}
-
-	// All descriptors are free to use now.
-	dt.freeHeadIndex = 0
-	dt.freeNum = uint16(len(dt.descriptors))
-
-	return nil
-}
-
-// releaseBuffers releases all allocated buffers for this descriptor table.
-// The implementation will try to release as many buffers as possible and
-// collect potential errors before returning them.
-// The descriptor table should no longer be used after calling this.
-func (dt *DescriptorTable) releaseBuffers() error {
-	for i := range dt.descriptors {
-		descriptor := &dt.descriptors[i]
-		descriptor.address = 0
-	}
-
-	// As a safety measure, make sure no descriptors can be used anymore.
-	dt.freeHeadIndex = noFreeHead
-	dt.freeNum = 0
-
-	if dt.bufferBase != 0 {
-		// The pointer points to memory not managed by Go, so this conversion
-		// is safe. See https://github.com/golang/go/issues/58625
-		dt.bufferBase = 0
-		//goland:noinspection GoVetUnsafePointer
-		err := unix.MunmapPtr(unsafe.Pointer(dt.bufferBase), uintptr(dt.bufferSize))
-		if err != nil {
-			return fmt.Errorf("release buffer memory: %w", err)
-		}
-	}
-
-	return nil
-}
-
-func (dt *DescriptorTable) CreateDescriptorForOutputs() (uint16, uint32, error) {
-	//todo just fill the damn table
-	// Do we still have enough free descriptors?
-
-	if 1 > dt.freeNum {
-		return 0, 0, ErrNotEnoughFreeDescriptors
-	}
-
-	// Above validation ensured that there is at least one free descriptor, so
-	// the free descriptor chain head should be valid.
-	if dt.freeHeadIndex == noFreeHead {
-		panic("free descriptor chain head is unset but there should be free descriptors")
-	}
-
-	// To avoid having to iterate over the whole table to find the descriptor
-	// pointing to the head just to replace the free head, we instead always
-	// create descriptor chains from the descriptors coming after the head.
-	// This way we only have to touch the head as a last resort, when all other
-	// descriptors are already used.
-	head := dt.descriptors[dt.freeHeadIndex].next
-	desc := &dt.descriptors[head]
-	next := desc.next
-
-	checkUnusedDescriptorLength(head, desc)
-
-	// Give the device the maximum available number of bytes to write into.
-	desc.length = uint32(dt.itemSize)
-	desc.flags = 0 // descriptorFlagWritable
-	desc.next = 0  // Not necessary to clear this, it's just for looks.
-
-	dt.freeNum -= 1
-
-	if dt.freeNum == 0 {
-		// The last descriptor in the chain should be the free chain head
-		// itself.
-		if next != dt.freeHeadIndex {
-			panic("descriptor chain takes up all free descriptors but does not end with the free chain head")
-		}
-
-		// When this new chain takes up all remaining descriptors, we no longer
-		// have a free chain.
-		dt.freeHeadIndex = noFreeHead
-	} else {
-		// We took some descriptors out of the free chain, so make sure to close
-		// the circle again.
-		dt.descriptors[dt.freeHeadIndex].next = next
-	}
-
-	return head, desc.length, nil
-}
-
-func (dt *DescriptorTable) createDescriptorForInputs() (uint16, error) {
-	// Do we still have enough free descriptors?
-	if 1 > dt.freeNum {
-		return 0, ErrNotEnoughFreeDescriptors
-	}
-
-	// Above validation ensured that there is at least one free descriptor, so
-	// the free descriptor chain head should be valid.
-	if dt.freeHeadIndex == noFreeHead {
-		panic("free descriptor chain head is unset but there should be free descriptors")
-	}
-
-	// To avoid having to iterate over the whole table to find the descriptor
-	// pointing to the head just to replace the free head, we instead always
-	// create descriptor chains from the descriptors coming after the head.
-	// This way we only have to touch the head as a last resort, when all other
-	// descriptors are already used.
-	head := dt.descriptors[dt.freeHeadIndex].next
-	desc := &dt.descriptors[head]
-	next := desc.next
-
-	checkUnusedDescriptorLength(head, desc)
-
-	// Give the device the maximum available number of bytes to write into.
-	desc.length = uint32(dt.itemSize)
-	desc.flags = descriptorFlagWritable
-	desc.next = 0 // Not necessary to clear this, it's just for looks.
-
-	dt.freeNum -= 1
-
-	if dt.freeNum == 0 {
-		// The last descriptor in the chain should be the free chain head
-		// itself.
-		if next != dt.freeHeadIndex {
-			panic("descriptor chain takes up all free descriptors but does not end with the free chain head")
-		}
-
-		// When this new chain takes up all remaining descriptors, we no longer
-		// have a free chain.
-		dt.freeHeadIndex = noFreeHead
-	} else {
-		// We took some descriptors out of the free chain, so make sure to close
-		// the circle again.
-		dt.descriptors[dt.freeHeadIndex].next = next
-	}
-
-	return head, nil
-}
-
-func (dt *DescriptorTable) getDescriptorItem(head uint16) []byte {
-	desc := &dt.descriptors[head] //todo this is a pretty nasty hack with no checks
-
-	// The descriptor address points to memory not managed by Go, so this
-	// conversion is safe. See https://github.com/golang/go/issues/58625
-	//goland:noinspection GoVetUnsafePointer
-	return unsafe.Slice((*byte)(unsafe.Pointer(desc.address)), desc.length)
-}
-
-// checkUnusedDescriptorLength asserts that the length of an unused descriptor
-// is zero, as it should be.
-// This is not a requirement by the virtio spec but rather a thing we do to
-// notice when our algorithm goes sideways.
-func checkUnusedDescriptorLength(index uint16, desc *Descriptor) {
-	if desc.length != 0 {
-		panic(fmt.Sprintf("descriptor %d should be unused but has a non-zero length", index))
-	}
-}

overlay/virtqueue/doc.go

@@ -1,7 +0,0 @@
-// Package virtqueue implements the driver-side for a virtio queue as described
-// in the specification:
-// https://docs.oasis-open.org/virtio/virtio/v1.2/csd01/virtio-v1.2-csd01.html#x1-270006
-// This package does not make assumptions about the device that consumes the
-// queue. It rather just allocates the queue structures in memory and provides
-// methods to interact with it.
-package virtqueue

overlay/virtqueue/eventfd_test.go

@@ -1,45 +0,0 @@
-package virtqueue
-
-import (
-	"testing"
-	"time"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"gvisor.dev/gvisor/pkg/eventfd"
-)
-
-// Tests how an eventfd and a waiting goroutine can be gracefully closed.
-// Extends the eventfd test suite:
-// https://github.com/google/gvisor/blob/0799336d64be65eb97d330606c30162dc3440cab/pkg/eventfd/eventfd_test.go
-func TestEventFD_CancelWait(t *testing.T) {
-	efd, err := eventfd.Create()
-	require.NoError(t, err)
-	t.Cleanup(func() {
-		assert.NoError(t, efd.Close())
-	})
-
-	var stop bool
-
-	done := make(chan struct{})
-	go func() {
-		for !stop {
-			_ = efd.Wait()
-		}
-		close(done)
-	}()
-	select {
-	case <-done:
-		t.Fatalf("goroutine ended early")
-	case <-time.After(500 * time.Millisecond):
-	}
-
-	stop = true
-	assert.NoError(t, efd.Notify())
-	select {
-	case <-done:
-		break
-	case <-time.After(5 * time.Second):
-		t.Error("goroutine did not end")
-	}
-}

overlay/virtqueue/size.go

@@ -1,33 +0,0 @@
-package virtqueue
-
-import (
-	"errors"
-	"fmt"
-)
-
-// ErrQueueSizeInvalid is returned when a queue size is invalid.
-var ErrQueueSizeInvalid = errors.New("queue size is invalid")
-
-// CheckQueueSize checks if the given value would be a valid size for a
-// virtqueue and returns an [ErrQueueSizeInvalid], if not.
-func CheckQueueSize(queueSize int) error {
-	if queueSize <= 0 {
-		return fmt.Errorf("%w: %d is too small", ErrQueueSizeInvalid, queueSize)
-	}
-
-	// The queue size must always be a power of 2.
-	// This ensures that ring indexes wrap correctly when the 16-bit integers
-	// overflow.
-	if queueSize&(queueSize-1) != 0 {
-		return fmt.Errorf("%w: %d is not a power of 2", ErrQueueSizeInvalid, queueSize)
-	}
-
-	// The largest power of 2 that fits into a 16-bit integer is 32768.
-	// 2 * 32768 would be 65536 which no longer fits.
-	if queueSize > 32768 {
-		return fmt.Errorf("%w: %d is larger than the maximum possible queue size 32768",
-			ErrQueueSizeInvalid, queueSize)
-	}
-
-	return nil
-}

overlay/virtqueue/size_test.go

@@ -1,59 +0,0 @@
-package virtqueue
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-)
-
-func TestCheckQueueSize(t *testing.T) {
-	tests := []struct {
-		name        string
-		queueSize   int
-		containsErr string
-	}{
-		{
-			name:        "negative",
-			queueSize:   -1,
-			containsErr: "too small",
-		},
-		{
-			name:        "zero",
-			queueSize:   0,
-			containsErr: "too small",
-		},
-		{
-			name:        "not a power of 2",
-			queueSize:   24,
-			containsErr: "not a power of 2",
-		},
-		{
-			name:        "too large",
-			queueSize:   65536,
-			containsErr: "larger than the maximum",
-		},
-		{
-			name:      "valid 1",
-			queueSize: 1,
-		},
-		{
-			name:      "valid 256",
-			queueSize: 256,
-		},
-
-		{
-			name:      "valid 32768",
-			queueSize: 32768,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			err := CheckQueueSize(tt.queueSize)
-			if tt.containsErr != "" {
-				assert.ErrorContains(t, err, tt.containsErr)
-			} else {
-				assert.NoError(t, err)
-			}
-		})
-	}
-}