add locking for stop crash

.github/workflows/gofmt.yml

@@ -14,7 +14,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-go@v6
       with:

.github/workflows/release.yml

@@ -10,7 +10,7 @@ jobs:
     name: Build Linux/BSD All
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
 
       - uses: actions/setup-go@v6
         with:
@@ -24,7 +24,7 @@ jobs:
           mv build/*.tar.gz release
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v4
         with:
           name: linux-latest
           path: release
@@ -33,7 +33,7 @@ jobs:
     name: Build Windows
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
 
       - uses: actions/setup-go@v6
         with:
@@ -55,7 +55,7 @@ jobs:
           mv dist\windows\wintun build\dist\windows\
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v4
         with:
           name: windows-latest
           path: build
@@ -66,7 +66,7 @@ jobs:
       HAS_SIGNING_CREDS: ${{ secrets.AC_USERNAME != '' }}
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
 
       - uses: actions/setup-go@v6
         with:
@@ -75,7 +75,7 @@ jobs:
 
       - name: Import certificates
         if: env.HAS_SIGNING_CREDS == 'true'
-        uses: Apple-Actions/import-codesign-certs@v6
+        uses: Apple-Actions/import-codesign-certs@v5
         with:
           p12-file-base64: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_P12_BASE64 }}
           p12-password: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_PASSWORD }}
@@ -104,7 +104,7 @@ jobs:
           fi
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v4
         with:
           name: darwin-latest
           path: ./release/*
@@ -124,11 +124,11 @@ jobs:
       # be overwritten
       - name: Checkout code
         if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
 
       - name: Download artifacts
         if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v4
         with:
           name: linux-latest
           path: artifacts
@@ -160,10 +160,10 @@ jobs:
     needs: [build-linux, build-darwin, build-windows]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
 
       - name: Download artifacts
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v4
         with:
           path: artifacts
 
@@ -209,11 +209,10 @@ jobs:
         id: create_release
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GITHUB_REF_NAME: ${{ github.ref_name }}
         run: |
           cd artifacts
           gh release create \
             --verify-tag \
-            --title "Release ${GITHUB_REF_NAME}" \
-            "${GITHUB_REF_NAME}" \
+            --title "Release ${{ github.ref_name }}" \
+            "${{ github.ref_name }}" \
             SHASUM256.txt *-latest/*.zip *-latest/*.tar.gz

.github/workflows/smoke-extra.yml

@@ -18,11 +18,9 @@ jobs:
     if: github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'smoke-test-extra')
     name: Run extra smoke tests
     runs-on: ubuntu-latest
-    env:
-      VAGRANT_DEFAULT_PROVIDER: libvirt
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-go@v6
       with:
@@ -32,13 +30,8 @@ jobs:
     - name: add hashicorp source
       run: wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg && echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
 
-    - name: install vagrant and libvirt
-      run: |
-        sudo apt-get update && sudo apt-get install -y vagrant libvirt-daemon-system libvirt-dev
-        sudo chmod 666 /dev/kvm
-        sudo usermod -aG libvirt $(whoami)
-        sudo chmod 666 /var/run/libvirt/libvirt-sock
-        vagrant plugin install vagrant-libvirt
+    - name: install vagrant
+      run: sudo apt-get update && sudo apt-get install -y vagrant virtualbox
 
     - name: freebsd-amd64
       run: make smoke-vagrant/freebsd-amd64
@@ -49,19 +42,10 @@ jobs:
     - name: netbsd-amd64
       run: make smoke-vagrant/netbsd-amd64
 
+    - name: linux-386
+      run: make smoke-vagrant/linux-386
+
     - name: linux-amd64-ipv6disable
       run: make smoke-vagrant/linux-amd64-ipv6disable
 
-    # linux-386 runs last because it requires disabling KVM to use VirtualBox,
-    # which prevents libvirt (used by the other tests) from working after this point.
-    - name: install virtualbox for i386 test
-      run: |
-        sudo apt-get install -y virtualbox
-        sudo rmmod kvm_amd kvm_intel kvm 2>/dev/null || true
-
-    - name: linux-386
-      env:
-        VAGRANT_DEFAULT_PROVIDER: virtualbox
-      run: make smoke-vagrant/linux-386
-
     timeout-minutes: 30

.github/workflows/smoke.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-go@v6
       with:

.github/workflows/smoke/build-relay.sh

@@ -16,10 +16,8 @@ relay:
   am_relay: true
 EOF
 
-    # TEST-NET-3 placeholder IPs; smoke-relay.sh seds them to real container IPs.
-    # Mapping: .2 lighthouse1, .3 host2, .4 host3, .5 host4.
-    export LIGHTHOUSES="192.168.100.1 203.0.113.2:4242"
-    export REMOTE_ALLOW_LIST='{"203.0.113.4/32": false, "203.0.113.5/32": false}'
+    export LIGHTHOUSES="192.168.100.1 172.17.0.2:4242"
+    export REMOTE_ALLOW_LIST='{"172.17.0.4/32": false, "172.17.0.5/32": false}'
 
     HOST="host2" ../genconfig.sh >host2.yml <<EOF
 relay:
@@ -27,7 +25,7 @@ relay:
     - 192.168.100.1
 EOF
 
-    export REMOTE_ALLOW_LIST='{"203.0.113.3/32": false}'
+    export REMOTE_ALLOW_LIST='{"172.17.0.3/32": false}'
 
     HOST="host3" ../genconfig.sh >host3.yml
 

.github/workflows/smoke/build.sh

@@ -5,15 +5,9 @@ set -e -x
 rm -rf ./build
 mkdir ./build
 
-# Smoke containers run on a dedicated docker network whose subnet is allocated
-# at smoke time, not known at build time. Configs are written with TEST-NET-3
-# placeholder IPs (RFC 5737) and smoke.sh / smoke-vagrant.sh / smoke-relay.sh
-# sed the real container IPs in before starting nebula.
-#
-# Placeholder mapping (last octet == fixed container slot):
-#   203.0.113.2 -> lighthouse1, 203.0.113.3 -> host2,
-#   203.0.113.4 -> host3,       203.0.113.5 -> host4.
-LIGHTHOUSE_IP="203.0.113.2"
+# TODO: Assumes your docker bridge network is a /24, and the first container that launches will be .1
+# - We could make this better by launching the lighthouse first and then fetching what IP it is.
+NET="$(docker network inspect bridge -f '{{ range .IPAM.Config }}{{ .Subnet }}{{ end }}' | cut -d. -f1-3)"
 
 (
     cd build
@@ -31,16 +25,16 @@ LIGHTHOUSE_IP="203.0.113.2"
         ../genconfig.sh >lighthouse1.yml
 
     HOST="host2" \
-        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
+        LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
         ../genconfig.sh >host2.yml
 
     HOST="host3" \
-        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
+        LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
         INBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
         ../genconfig.sh >host3.yml
 
     HOST="host4" \
-        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
+        LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
         OUTBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
         ../genconfig.sh >host4.yml
 

.github/workflows/smoke/smoke-relay.sh

@@ -6,8 +6,6 @@ set -o pipefail
 
 mkdir -p logs
 
-NETWORK="nebula-smoke-relay"
-
 cleanup() {
     echo
     echo " *** cleanup"
@@ -18,53 +16,22 @@ cleanup() {
     then
         docker kill lighthouse1 host2 host3 host4
     fi
-    docker network rm "$NETWORK" >/dev/null 2>&1
 }
 
 trap cleanup EXIT
 
-# Create a dedicated smoke network with an explicit subnet (required for --ip
-# below). Probe a short list of candidates so a locally-used range doesn't
-# fail the whole test — we only need one to be free.
-docker network rm "$NETWORK" >/dev/null 2>&1 || true
-for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
-    if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
-        break
-    fi
-done
-if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
-    echo "failed to create $NETWORK: every candidate subnet is in use" >&2
-    exit 1
-fi
-
-# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
-# .3 host2, .4 host3, .5 host4 — matches the placeholders in build-relay.sh.
-SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
-PREFIX="${SUBNET%/*}"
-PREFIX="${PREFIX%.*}"
-LIGHTHOUSE_IP="$PREFIX.2"
-HOST2_IP="$PREFIX.3"
-HOST3_IP="$PREFIX.4"
-HOST4_IP="$PREFIX.5"
-
-# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
-for f in build/host2.yml build/host3.yml build/host4.yml; do
-    sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
-    mv "$f.tmp" "$f"
-done
-
 docker run --name lighthouse1 --rm nebula:smoke-relay -config lighthouse1.yml -test
-docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" nebula:smoke-relay -config host2.yml -test
-docker run --name host3 --rm -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" nebula:smoke-relay -config host3.yml -test
-docker run --name host4 --rm -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" nebula:smoke-relay -config host4.yml -test
+docker run --name host2 --rm nebula:smoke-relay -config host2.yml -test
+docker run --name host3 --rm nebula:smoke-relay -config host3.yml -test
+docker run --name host4 --rm nebula:smoke-relay -config host4.yml -test
 
-docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
+docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
 sleep 1
-docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
+docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
 sleep 1
-docker run --name host3 --network "$NETWORK" --ip "$HOST3_IP" -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
+docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
 sleep 1
-docker run --name host4 --network "$NETWORK" --ip "$HOST4_IP" -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
+docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
 sleep 1
 
 set +x
@@ -109,13 +76,7 @@ docker exec host4 sh -c 'kill 1'
 docker exec host3 sh -c 'kill 1'
 docker exec host2 sh -c 'kill 1'
 docker exec lighthouse1 sh -c 'kill 1'
-
-# Wait up to 30s for all backgrounded jobs to exit rather than relying on a
-# fixed sleep.
-for _ in $(seq 1 30); do
-    [ -z "$(jobs -r)" ] && break
-    sleep 1
-done
+sleep 5
 
 if [ "$(jobs -r)" ]
 then

.github/workflows/smoke/smoke-vagrant.sh

@@ -8,8 +8,6 @@ export VAGRANT_CWD="$PWD/vagrant-$1"
 
 mkdir -p logs
 
-NETWORK="nebula-smoke"
-
 cleanup() {
     echo
     echo " *** cleanup"
@@ -21,51 +19,21 @@ cleanup() {
         docker kill lighthouse1 host2
     fi
     vagrant destroy -f
-    docker network rm "$NETWORK" >/dev/null 2>&1
 }
 
 trap cleanup EXIT
 
-# Create a dedicated smoke network with an explicit subnet (required for --ip
-# below). Probe a short list of candidates so a locally-used range doesn't
-# fail the whole test — we only need one to be free.
-docker network rm "$NETWORK" >/dev/null 2>&1 || true
-for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
-    if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
-        break
-    fi
-done
-if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
-    echo "failed to create $NETWORK: every candidate subnet is in use" >&2
-    exit 1
-fi
-
-# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
-# .3 host2 — matches the placeholders in build.sh.
-SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
-PREFIX="${SUBNET%/*}"
-PREFIX="${PREFIX%.*}"
-LIGHTHOUSE_IP="$PREFIX.2"
-HOST2_IP="$PREFIX.3"
-
-# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
-# This must happen before `vagrant up` rsyncs build/ into the VM for host3.
-for f in build/host2.yml build/host3.yml; do
-    sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
-    mv "$f.tmp" "$f"
-done
-
 CONTAINER="nebula:${NAME:-smoke}"
 
 docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test
-docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" "$CONTAINER" -config host2.yml -test
+docker run --name host2 --rm "$CONTAINER" -config host2.yml -test
 
 vagrant up
 vagrant ssh -c "cd /nebula && /nebula/$1-nebula -config host3.yml -test" -- -T
 
-docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
+docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
 sleep 1
-docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
+docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
 sleep 1
 vagrant ssh -c "cd /nebula && sudo sh -c 'echo \$\$ >/nebula/pid && exec /nebula/$1-nebula -config host3.yml'" 2>&1 -- -T | tee logs/host3 | sed -u 's/^/  [host3]  /' &
 sleep 15
@@ -128,14 +96,7 @@ vagrant ssh -c "ping -c1 192.168.100.2" -- -T
 vagrant ssh -c "sudo xargs kill </nebula/pid" -- -T
 docker exec host2 sh -c 'kill 1'
 docker exec lighthouse1 sh -c 'kill 1'
-
-# Wait up to 30s for all backgrounded jobs to exit. vagrant ssh in particular
-# takes a beat to tear down after nebula exits on the VM, so a fixed sleep is
-# racy.
-for _ in $(seq 1 30); do
-    [ -z "$(jobs -r)" ] && break
-    sleep 1
-done
+sleep 1
 
 if [ "$(jobs -r)" ]
 then

.github/workflows/smoke/smoke.sh

@@ -6,8 +6,6 @@ set -o pipefail
 
 mkdir -p logs
 
-NETWORK="nebula-smoke"
-
 cleanup() {
     echo
     echo " *** cleanup"
@@ -18,71 +16,38 @@ cleanup() {
     then
         docker kill lighthouse1 host2 host3 host4
     fi
-    docker network rm "$NETWORK" >/dev/null 2>&1
 }
 
 trap cleanup EXIT
 
-# Create a dedicated smoke network with an explicit subnet (required for --ip
-# below). Probe a short list of candidates so a locally-used range doesn't
-# fail the whole test — we only need one to be free.
-docker network rm "$NETWORK" >/dev/null 2>&1 || true
-for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
-    if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
-        break
-    fi
-done
-if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
-    echo "failed to create $NETWORK: every candidate subnet is in use" >&2
-    exit 1
-fi
-
-# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
-# .3 host2, .4 host3, .5 host4 — matches the placeholders in build.sh.
-SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
-PREFIX="${SUBNET%/*}"
-PREFIX="${PREFIX%.*}"
-LIGHTHOUSE_IP="$PREFIX.2"
-HOST2_IP="$PREFIX.3"
-HOST3_IP="$PREFIX.4"
-HOST4_IP="$PREFIX.5"
-
-# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
-# build/lighthouse1.yml has no IPs to rewrite so it's skipped.
-for f in build/host2.yml build/host3.yml build/host4.yml; do
-    sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
-    mv "$f.tmp" "$f"
-done
-
 CONTAINER="nebula:${NAME:-smoke}"
 
 docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test
-docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" "$CONTAINER" -config host2.yml -test
-docker run --name host3 --rm -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" "$CONTAINER" -config host3.yml -test
-docker run --name host4 --rm -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" "$CONTAINER" -config host4.yml -test
+docker run --name host2 --rm "$CONTAINER" -config host2.yml -test
+docker run --name host3 --rm "$CONTAINER" -config host3.yml -test
+docker run --name host4 --rm "$CONTAINER" -config host4.yml -test
 
-docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
+docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
 sleep 1
-docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
+docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
 sleep 1
-docker run --name host3 --network "$NETWORK" --ip "$HOST3_IP" -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
+docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
 sleep 1
-docker run --name host4 --network "$NETWORK" --ip "$HOST4_IP" -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
+docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
 sleep 1
 
 # grab tcpdump pcaps for debugging
-docker exec lighthouse1 tcpdump -i tun0 -q -w - -U 2>logs/lighthouse1.inside.log >logs/lighthouse1.inside.pcap &
+docker exec lighthouse1 tcpdump -i nebula1 -q -w - -U 2>logs/lighthouse1.inside.log >logs/lighthouse1.inside.pcap &
 docker exec lighthouse1 tcpdump -i eth0 -q -w - -U 2>logs/lighthouse1.outside.log >logs/lighthouse1.outside.pcap &
-docker exec host2 tcpdump -i tun0 -q -w - -U 2>logs/host2.inside.log >logs/host2.inside.pcap &
+docker exec host2 tcpdump -i nebula1 -q -w - -U 2>logs/host2.inside.log >logs/host2.inside.pcap &
 docker exec host2 tcpdump -i eth0 -q -w - -U 2>logs/host2.outside.log >logs/host2.outside.pcap &
-docker exec host3 tcpdump -i tun0 -q -w - -U 2>logs/host3.inside.log >logs/host3.inside.pcap &
+docker exec host3 tcpdump -i nebula1 -q -w - -U 2>logs/host3.inside.log >logs/host3.inside.pcap &
 docker exec host3 tcpdump -i eth0 -q -w - -U 2>logs/host3.outside.log >logs/host3.outside.pcap &
-docker exec host4 tcpdump -i tun0 -q -w - -U 2>logs/host4.inside.log >logs/host4.inside.pcap &
+docker exec host4 tcpdump -i nebula1 -q -w - -U 2>logs/host4.inside.log >logs/host4.inside.pcap &
 docker exec host4 tcpdump -i eth0 -q -w - -U 2>logs/host4.outside.log >logs/host4.outside.pcap &
 
 docker exec host2 ncat -nklv 0.0.0.0 2000 &
 docker exec host3 ncat -nklv 0.0.0.0 2000 &
-docker exec host4 ncat -nkluv 0.0.0.0 4000 &
 docker exec host2 ncat -e '/usr/bin/echo host2' -nkluv 0.0.0.0 3000 &
 docker exec host3 ncat -e '/usr/bin/echo host3' -nkluv 0.0.0.0 3000 &
 
@@ -154,24 +119,17 @@ echo
 echo " *** Testing conntrack"
 echo
 set -x
-
-# host2 speaking to host4 on UDP 4000 should allow it to reply, when firewall rules would normally not permit this
-docker exec host2 sh -c "/usr/bin/echo host2 | ncat -nuv 192.168.100.4 4000"
-docker exec host2 ncat -e '/usr/bin/echo helloagainfromhost2' -nkluv 0.0.0.0 4000 &
-sleep 1
-docker exec host4 sh -c "/usr/bin/echo host4 | ncat -nuv 192.168.100.2 4000"
+# host2 can ping host3 now that host3 pinged it first
+docker exec host2 ping -c1 192.168.100.3
+# host4 can ping host2 once conntrack established
+docker exec host2 ping -c1 192.168.100.4
+docker exec host4 ping -c1 192.168.100.2
 
 docker exec host4 sh -c 'kill 1'
 docker exec host3 sh -c 'kill 1'
 docker exec host2 sh -c 'kill 1'
 docker exec lighthouse1 sh -c 'kill 1'
-
-# Wait up to 30s for all backgrounded jobs to exit rather than relying on a
-# fixed sleep.
-for _ in $(seq 1 30); do
-    [ -z "$(jobs -r)" ] && break
-    sleep 1
-done
+sleep 5
 
 if [ "$(jobs -r)" ]
 then

.github/workflows/smoke/vagrant-linux-amd64-ipv6disable/Vagrantfile

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 Vagrant.configure("2") do |config|
-  config.vm.box = "bento/ubuntu-24.04"
+  config.vm.box = "ubuntu/jammy64"
 
   config.vm.synced_folder "../build", "/nebula"
 

.github/workflows/smoke/vagrant-openbsd-amd64/Vagrantfile

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 Vagrant.configure("2") do |config|
-  config.vm.box = "DefinedNet/openbsd78"
+  config.vm.box = "generic/openbsd7"
 
   config.vm.synced_folder "../build", "/nebula", type: "rsync"
 end

.github/workflows/test.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-go@v6
       with:
@@ -32,7 +32,7 @@ jobs:
       run: make vet
 
     - name: golangci-lint
-      uses: golangci/golangci-lint-action@v9
+      uses: golangci/golangci-lint-action@v8
       with:
         version: v2.5
 
@@ -45,7 +45,7 @@ jobs:
     - name: Build test mobile
       run: make build-test-mobile
 
-    - uses: actions/upload-artifact@v6
+    - uses: actions/upload-artifact@v4
       with:
         name: e2e packet flow linux-latest
         path: e2e/mermaid/linux-latest
@@ -56,7 +56,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-go@v6
       with:
@@ -77,7 +77,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-go@v6
       with:
@@ -98,7 +98,7 @@ jobs:
         os: [windows-latest, macos-latest]
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
     - uses: actions/setup-go@v6
       with:
@@ -115,7 +115,7 @@ jobs:
       run: make vet
 
     - name: golangci-lint
-      uses: golangci/golangci-lint-action@v9
+      uses: golangci/golangci-lint-action@v8
       with:
         version: v2.5
 
@@ -125,7 +125,7 @@ jobs:
     - name: End 2 end
       run: make e2evv
 
-    - uses: actions/upload-artifact@v6
+    - uses: actions/upload-artifact@v4
       with:
         name: e2e packet flow ${{ matrix.os }}
         path: e2e/mermaid/${{ matrix.os }}

CHANGELOG.md

@@ -7,129 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-## [1.10.3] - 2026-02-06
-
-### Security
-
-- Fix an issue where blocklist bypass is possible when using curve P256 since the signature can have 2 valid representations.
-  Both fingerprint representations will be tested against the blocklist.
-  Any newly issued P256 based certificates will have their signature clamped to the low-s form.
-  Nebula will assert the low-s signature form when validating certificates in a future version. [GHSA-69x3-g4r3-p962](https://github.com/slackhq/nebula/security/advisories/GHSA-69x3-g4r3-p962)
-
 ### Changed
 
-- Improve error reporting if nebula fails to start due to a tun device naming issue. (#1588)
-
-## [1.10.2] - 2026-01-21
-
-### Fixed
-
-- Fix panic when using `use_system_route_table` that was introduced in v1.10.1. (#1580)
-
-### Changed
-
-- Fix some typos in comments. (#1582)
-- Dependency updates. (#1581)
-
-## [1.10.1] - 2026-01-16
-
-See the [v1.10.1](https://github.com/slackhq/nebula/milestone/26?closed=1) milestone for a complete list of changes.
-
-### Fixed
-
-- Fix a bug where an unsafe route derived from the system route table could be lost on a config reload. (#1573)
-- Fix the PEM banner for ECDSA P256 public keys. (#1552)
-- Fix a regression on Windows from 1.9.x where nebula could fall back to a less performant UDP listener if 
-  non-critical ioctls failed. (#1568)
-- Fix a bug in handshake processing when a peer sends an unexpected public key. (#1566)
-
-### Added
-
-- Add a config option to control accepting `recv_error` packets which defaults to `always`. (#1569)
-
-### Changed
-
-- Various dependency updates. (#1541, #1549, #1550, #1557, #1558, #1560, #1561, #1570, #1571)
-
-## [1.10.0] - 2025-12-04
-
-See the [v1.10.0](https://github.com/slackhq/nebula/milestone/16?closed=1) milestone for a complete list of changes.
-
-### Added
-
-- Support for ipv6 and multiple ipv4/6 addresses in the overlay.
-  A new v2 ASN.1 based certificate format.
-  Certificates now have a unified interface for external implementations.
-  (#1212, #1216, #1345, #1359, #1381, #1419, #1464, #1466, #1451, #1476, #1467, #1481, #1399, #1488, #1492, #1495, #1468, #1521, #1535, #1538)
-- Add the ability to mark packets on linux to better target nebula packets in iptables/nftables. (#1331)
-- Add ECMP support for `unsafe_routes`. (#1332)
-- PKCS11 support for P256 keys when built with `pkcs11` tag (#1153, #1482)
-
-### Changed
-
-- **NOTE**: `default_local_cidr_any` now defaults to false, meaning that any firewall rule
+- `default_local_cidr_any` now defaults to false, meaning that any firewall rule
   intended to target an `unsafe_routes` entry must explicitly declare it via the
   `local_cidr` field. This is almost always the intended behavior. This flag is
-  deprecated and will be removed in a future release. (#1373)
-- Improve logging when a relay is in use on an inbound packet. (#1533)
-- Avoid fatal errors if `rountines` is > 1 on systems that don't support more than 1 routine. (#1531)
-- Log a warning if a firewall rule contains an `any` that negates a more restrictive filter. (#1513)
-- Accept encrypted CA passphrase from an environment variable. (#1421)
-- Allow handshaking with any trusted remote. (#1509)
-- Log only the count of blocklisted certificate fingerprints instead of the entire list. (#1525)
-- Don't fatal when the ssh server is unable to be configured successfully. (#1520)
-- Update to build against go v1.25. (#1483)
-- Allow projects using `nebula` as a library with userspace networking to configure the `logger` and build version. (#1239) 
-- Upgrade to `yaml.v3`. (#1148, #1371, #1438, #1478)
-
-### Fixed
-
-- Fix a potential bug with udp ipv4 only on darwin. (#1532)
-- Improve lost packet statistics. (#1441, #1537)
-- Honor `remote_allow_list` in hole punch response. (#1186)
-- Fix a panic when `tun.use_system_route_table` is `true` and a route lacks a destination. (#1437) 
-- Fix an issue when `tun.use_system_route_table: true` could result in heavy CPU utilization when many thousands of routes
-  are present. (#1326) 
-- Fix tests for 32 bit machines. (#1394)
-- Fix a possible 32bit integer underflow in config handling. (#1353)
-- Fix moving a udp address from one vpn address to another in the `static_host_map`
-  which could cause rapid re-handshaking with an incorrect remote. (#1259)
-- Improve smoke tests in environments where the docker network is not the default. (#1347)
-
-## [1.9.7] - 2025-10-10
-
-### Security
-
-- Fix an issue where Nebula could incorrectly accept and process a packet from an erroneous source IP when the sender's
-  certificate is configured with unsafe_routes (cert v1/v2) or multiple IPs (cert v2). (#1494)
-
-### Changed
-
-- Disable sending `recv_error` messages when a packet is received outside the allowable counter window. (#1459)
-- Improve error messages and remove some unnecessary fatal conditions in the Windows and generic udp listener. (#1453)
-
-## [1.9.6] - 2025-7-15
-
-### Added
-
-- Support dropping inactive tunnels. This is disabled by default in this release but can be enabled with `tunnels.drop_inactive`. See example config for more details. (#1413)
-
-### Fixed
-
-- Fix Darwin freeze due to presence of some Network Extensions (#1426)
-- Ensure the same relay tunnel is always used when multiple relay tunnels are present (#1422)
-- Fix Windows freeze due to ICMP error handling (#1412)
-- Fix relay migration panic (#1403)
-
-## [1.9.5] - 2024-12-05
-
-### Added
-
-- Gracefully ignore v2 certificates. (#1282)
-
-### Fixed
-
-- Fix relays that refuse to re-establish after one of the remote tunnel pairs breaks. (#1277)
+  deprecated and will be removed in a future release.
 
 ## [1.9.4] - 2024-09-09
 
@@ -788,14 +671,7 @@ created.)
 
 - Initial public release.
 
-[Unreleased]: https://github.com/slackhq/nebula/compare/v1.10.3...HEAD
-[1.10.3]: https://github.com/slackhq/nebula/releases/tag/v1.10.3
-[1.10.2]: https://github.com/slackhq/nebula/releases/tag/v1.10.2
-[1.10.1]: https://github.com/slackhq/nebula/releases/tag/v1.10.1
-[1.10.0]: https://github.com/slackhq/nebula/releases/tag/v1.10.0
-[1.9.7]: https://github.com/slackhq/nebula/releases/tag/v1.9.7
-[1.9.6]: https://github.com/slackhq/nebula/releases/tag/v1.9.6
-[1.9.5]: https://github.com/slackhq/nebula/releases/tag/v1.9.5
+[Unreleased]: https://github.com/slackhq/nebula/compare/v1.9.4...HEAD
 [1.9.4]: https://github.com/slackhq/nebula/releases/tag/v1.9.4
 [1.9.3]: https://github.com/slackhq/nebula/releases/tag/v1.9.3
 [1.9.2]: https://github.com/slackhq/nebula/releases/tag/v1.9.2

CODEOWNERS

@@ -1 +0,0 @@
-#ECCN:Open Source

README.md

@@ -57,7 +57,7 @@ Check the [releases](https://github.com/slackhq/nebula/releases/latest) page for
     docker pull nebulaoss/nebula
     ```
 
-#### Mobile ([source code](https://github.com/DefinedNet/mobile_nebula))
+#### Mobile
 
 - [iOS](https://apps.apple.com/us/app/mobile-nebula/id1509587936?itsct=apps_box&itscg=30200)
 - [Android](https://play.google.com/store/apps/details?id=net.defined.mobile_nebula&pcampaignid=pcampaignidMKT-Other-global-all-co-prtnr-py-PartBadge-Mar2515-1)
@@ -76,8 +76,6 @@ Nebula was created to provide a mechanism for groups of hosts to communicate sec
 
 ## Getting started (quickly)
 
-**Don't want to manage your own PKI and lighthouses?** [Managed Nebula](https://www.defined.net/) from Defined Networking handles all of this for you.
-
 To set up a Nebula network, you'll need:
 
 #### 1. The [Nebula binaries](https://github.com/slackhq/nebula/releases) or [Distribution Packages](https://github.com/slackhq/nebula#distribution-packages) for your specific platform. Specifically you'll need `nebula-cert` and the specific nebula binary for each platform you use.

batch_pipeline.go

@@ -0,0 +1,164 @@
+package nebula
+
+import (
+	"net/netip"
+
+	"github.com/slackhq/nebula/overlay"
+	"github.com/slackhq/nebula/udp"
+)
+
+// batchPipelines tracks whether the inside device can operate on packet batches
+// and, if so, holds the shared packet pool sized for the virtio headroom and
+// payload limits advertised by the device. It also owns the fan-in/fan-out
+// queues between the TUN readers, encrypt/decrypt workers, and the UDP writers.
+type batchPipelines struct {
+	enabled    bool
+	inside     overlay.BatchCapableDevice
+	headroom   int
+	payloadCap int
+	pool       *overlay.PacketPool
+	batchSize  int
+	routines   int
+	rxQueues   []chan *overlay.Packet
+	txQueues   []chan queuedDatagram
+	tunQueues  []chan *overlay.Packet
+}
+
+type queuedDatagram struct {
+	packet *overlay.Packet
+	addr   netip.AddrPort
+}
+
+func (bp *batchPipelines) init(device overlay.Device, routines int, queueDepth int, maxSegments int) {
+	if device == nil || routines <= 0 {
+		return
+	}
+	bcap, ok := device.(overlay.BatchCapableDevice)
+	if !ok {
+		return
+	}
+	headroom := bcap.BatchHeadroom()
+	payload := bcap.BatchPayloadCap()
+	if maxSegments < 1 {
+		maxSegments = 1
+	}
+	requiredPayload := udp.MTU * maxSegments
+	if payload < requiredPayload {
+		payload = requiredPayload
+	}
+	batchSize := bcap.BatchSize()
+	if headroom <= 0 || payload <= 0 || batchSize <= 0 {
+		return
+	}
+	bp.enabled = true
+	bp.inside = bcap
+	bp.headroom = headroom
+	bp.payloadCap = payload
+	bp.batchSize = batchSize
+	bp.routines = routines
+	bp.pool = overlay.NewPacketPool(headroom, payload)
+	queueCap := batchSize * defaultBatchQueueDepthFactor
+	if queueDepth > 0 {
+		queueCap = queueDepth
+	}
+	if queueCap < batchSize {
+		queueCap = batchSize
+	}
+	bp.rxQueues = make([]chan *overlay.Packet, routines)
+	bp.txQueues = make([]chan queuedDatagram, routines)
+	bp.tunQueues = make([]chan *overlay.Packet, routines)
+	for i := 0; i < routines; i++ {
+		bp.rxQueues[i] = make(chan *overlay.Packet, queueCap)
+		bp.txQueues[i] = make(chan queuedDatagram, queueCap)
+		bp.tunQueues[i] = make(chan *overlay.Packet, queueCap)
+	}
+}
+
+func (bp *batchPipelines) Pool() *overlay.PacketPool {
+	if bp == nil || !bp.enabled {
+		return nil
+	}
+	return bp.pool
+}
+
+func (bp *batchPipelines) Enabled() bool {
+	return bp != nil && bp.enabled
+}
+
+func (bp *batchPipelines) batchSizeHint() int {
+	if bp == nil || bp.batchSize <= 0 {
+		return 1
+	}
+	return bp.batchSize
+}
+
+func (bp *batchPipelines) rxQueue(i int) chan *overlay.Packet {
+	if bp == nil || !bp.enabled || i < 0 || i >= len(bp.rxQueues) {
+		return nil
+	}
+	return bp.rxQueues[i]
+}
+
+func (bp *batchPipelines) txQueue(i int) chan queuedDatagram {
+	if bp == nil || !bp.enabled || i < 0 || i >= len(bp.txQueues) {
+		return nil
+	}
+	return bp.txQueues[i]
+}
+
+func (bp *batchPipelines) tunQueue(i int) chan *overlay.Packet {
+	if bp == nil || !bp.enabled || i < 0 || i >= len(bp.tunQueues) {
+		return nil
+	}
+	return bp.tunQueues[i]
+}
+
+func (bp *batchPipelines) txQueueLen(i int) int {
+	q := bp.txQueue(i)
+	if q == nil {
+		return 0
+	}
+	return len(q)
+}
+
+func (bp *batchPipelines) tunQueueLen(i int) int {
+	q := bp.tunQueue(i)
+	if q == nil {
+		return 0
+	}
+	return len(q)
+}
+
+func (bp *batchPipelines) enqueueRx(i int, pkt *overlay.Packet) bool {
+	q := bp.rxQueue(i)
+	if q == nil {
+		return false
+	}
+	q <- pkt
+	return true
+}
+
+func (bp *batchPipelines) enqueueTx(i int, pkt *overlay.Packet, addr netip.AddrPort) bool {
+	q := bp.txQueue(i)
+	if q == nil {
+		return false
+	}
+	q <- queuedDatagram{packet: pkt, addr: addr}
+	return true
+}
+
+func (bp *batchPipelines) enqueueTun(i int, pkt *overlay.Packet) bool {
+	q := bp.tunQueue(i)
+	if q == nil {
+		return false
+	}
+	q <- pkt
+	return true
+}
+
+func (bp *batchPipelines) newPacket() *overlay.Packet {
+	if bp == nil || !bp.enabled || bp.pool == nil {
+		return nil
+	}
+	return bp.pool.Get()
+}

bits.go

@@ -9,13 +9,14 @@ type Bits struct {
 	length             uint64
 	current            uint64
 	bits               []bool
+	firstSeen          bool
 	lostCounter        metrics.Counter
 	dupeCounter        metrics.Counter
 	outOfWindowCounter metrics.Counter
 }
 
 func NewBits(bits uint64) *Bits {
-	b := &Bits{
+	return &Bits{
 		length:             bits,
 		bits:               make([]bool, bits, bits),
 		current:            0,
@@ -23,37 +24,34 @@ func NewBits(bits uint64) *Bits {
 		dupeCounter:        metrics.GetOrRegisterCounter("network.packets.duplicate", nil),
 		outOfWindowCounter: metrics.GetOrRegisterCounter("network.packets.out_of_window", nil),
 	}
-
-	// There is no counter value 0, mark it to avoid counting a lost packet later.
-	b.bits[0] = true
-	b.current = 0
-	return b
 }
 
-func (b *Bits) Check(l *logrus.Logger, i uint64) bool {
+func (b *Bits) Check(l logrus.FieldLogger, i uint64) bool {
 	// If i is the next number, return true.
-	if i > b.current {
+	if i > b.current || (i == 0 && b.firstSeen == false && b.current < b.length) {
 		return true
 	}
 
-	// If i is within the window, check if it's been set already.
-	if i > b.current-b.length || i < b.length && b.current < b.length {
+	// If i is within the window, check if it's been set already. The first window will fail this check
+	if i > b.current-b.length {
+		return !b.bits[i%b.length]
+	}
+
+	// If i is within the first window
+	if i < b.length {
 		return !b.bits[i%b.length]
 	}
 
 	// Not within the window
-	if l.Level >= logrus.DebugLevel {
-		l.Debugf("rejected a packet (top) %d %d\n", b.current, i)
-	}
+	l.Debugf("rejected a packet (top) %d %d\n", b.current, i)
 	return false
 }
 
 func (b *Bits) Update(l *logrus.Logger, i uint64) bool {
 	// If i is the next number, return true and update current.
 	if i == b.current+1 {
-		// Check if the oldest bit was lost since we are shifting the window by 1 and occupying it with this counter
-		// The very first window can only be tracked as lost once we are on the 2nd window or greater
-		if b.bits[i%b.length] == false && i > b.length {
+		// Report missed packets, we can only understand what was missed after the first window has been gone through
+		if i > b.length && b.bits[i%b.length] == false {
 			b.lostCounter.Inc(1)
 		}
 		b.bits[i%b.length] = true
@@ -61,32 +59,61 @@ func (b *Bits) Update(l *logrus.Logger, i uint64) bool {
 		return true
 	}
 
-	// If i is a jump, adjust the window, record lost, update current, and return true
-	if i > b.current {
-		lost := int64(0)
-		// Zero out the bits between the current and the new counter value, limited by the window size,
-		// since the window is shifting
-		for n := b.current + 1; n <= min(i, b.current+b.length); n++ {
-			if b.bits[n%b.length] == false && n > b.length {
-				lost++
-			}
+	// If i packet is greater than current but less than the maximum length of our bitmap,
+	// flip everything in between to false and move ahead.
+	if i > b.current && i < b.current+b.length {
+		// In between current and i need to be zero'd to allow those packets to come in later
+		for n := b.current + 1; n < i; n++ {
 			b.bits[n%b.length] = false
 		}
 
-		// Only record any skipped packets as a result of the window moving further than the window length
-		// Any loss within the new window will be accounted for in future calls
-		lost += max(0, int64(i-b.current-b.length))
+		b.bits[i%b.length] = true
+		b.current = i
+		//l.Debugf("missed %d packets between %d and %d\n", i-b.current, i, b.current)
+		return true
+	}
+
+	// If i is greater than the delta between current and the total length of our bitmap,
+	// just flip everything in the map and move ahead.
+	if i >= b.current+b.length {
+		// The current window loss will be accounted for later, only record the jump as loss up until then
+		lost := maxInt64(0, int64(i-b.current-b.length))
+		//TODO: explain this
+		if b.current == 0 {
+			lost++
+		}
+
+		for n := range b.bits {
+			// Don't want to count the first window as a loss
+			//TODO: this is likely wrong, we are wanting to track only the bit slots that we aren't going to track anymore and this is marking everything as missed
+			//if b.bits[n] == false {
+			//	lost++
+			//}
+			b.bits[n] = false
+		}
+
 		b.lostCounter.Inc(lost)
 
+		if l.Level >= logrus.DebugLevel {
+			l.WithField("receiveWindow", m{"accepted": true, "currentCounter": b.current, "incomingCounter": i, "reason": "window shifting"}).
+				Debug("Receive window")
+		}
 		b.bits[i%b.length] = true
 		b.current = i
 		return true
 	}
 
-	// If i is within the current window but below the current counter,
-	// Check to see if it's a duplicate
-	if i > b.current-b.length || i < b.length && b.current < b.length {
-		if b.current == i || b.bits[i%b.length] == true {
+	// Allow for the 0 packet to come in within the first window
+	if i == 0 && b.firstSeen == false && b.current < b.length {
+		b.firstSeen = true
+		b.bits[i%b.length] = true
+		return true
+	}
+
+	// If i is within the window of current minus length (the total pat window size),
+	// allow it and flip to true but to NOT change current. We also have to account for the first window
+	if ((b.current >= b.length && i > b.current-b.length) || (b.current < b.length && i < b.length)) && i <= b.current {
+		if b.current == i {
 			if l.Level >= logrus.DebugLevel {
 				l.WithField("receiveWindow", m{"accepted": false, "currentCounter": b.current, "incomingCounter": i, "reason": "duplicate"}).
 					Debug("Receive window")
@@ -95,8 +122,18 @@ func (b *Bits) Update(l *logrus.Logger, i uint64) bool {
 			return false
 		}
 
+		if b.bits[i%b.length] == true {
+			if l.Level >= logrus.DebugLevel {
+				l.WithField("receiveWindow", m{"accepted": false, "currentCounter": b.current, "incomingCounter": i, "reason": "old duplicate"}).
+					Debug("Receive window")
+			}
+			b.dupeCounter.Inc(1)
+			return false
+		}
+
 		b.bits[i%b.length] = true
 		return true
+
 	}
 
 	// In all other cases, fail and don't change current.
@@ -110,3 +147,11 @@ func (b *Bits) Update(l *logrus.Logger, i uint64) bool {
 	}
 	return false
 }
+
+func maxInt64(a, b int64) int64 {
+	if a > b {
+		return a
+	}
+
+	return b
+}

bits_test.go

@@ -15,41 +15,48 @@ func TestBits(t *testing.T) {
 	assert.Len(t, b.bits, 10)
 
 	// This is initialized to zero - receive one. This should work.
+
 	assert.True(t, b.Check(l, 1))
-	assert.True(t, b.Update(l, 1))
+	u := b.Update(l, 1)
+	assert.True(t, u)
 	assert.EqualValues(t, 1, b.current)
-	g := []bool{true, true, false, false, false, false, false, false, false, false}
+	g := []bool{false, true, false, false, false, false, false, false, false, false}
 	assert.Equal(t, g, b.bits)
 
 	// Receive two
 	assert.True(t, b.Check(l, 2))
-	assert.True(t, b.Update(l, 2))
+	u = b.Update(l, 2)
+	assert.True(t, u)
 	assert.EqualValues(t, 2, b.current)
-	g = []bool{true, true, true, false, false, false, false, false, false, false}
+	g = []bool{false, true, true, false, false, false, false, false, false, false}
 	assert.Equal(t, g, b.bits)
 
 	// Receive two again - it will fail
 	assert.False(t, b.Check(l, 2))
-	assert.False(t, b.Update(l, 2))
+	u = b.Update(l, 2)
+	assert.False(t, u)
 	assert.EqualValues(t, 2, b.current)
 
 	// Jump ahead to 15, which should clear everything and set the 6th element
 	assert.True(t, b.Check(l, 15))
-	assert.True(t, b.Update(l, 15))
+	u = b.Update(l, 15)
+	assert.True(t, u)
 	assert.EqualValues(t, 15, b.current)
 	g = []bool{false, false, false, false, false, true, false, false, false, false}
 	assert.Equal(t, g, b.bits)
 
 	// Mark 14, which is allowed because it is in the window
 	assert.True(t, b.Check(l, 14))
-	assert.True(t, b.Update(l, 14))
+	u = b.Update(l, 14)
+	assert.True(t, u)
 	assert.EqualValues(t, 15, b.current)
 	g = []bool{false, false, false, false, true, true, false, false, false, false}
 	assert.Equal(t, g, b.bits)
 
 	// Mark 5, which is not allowed because it is not in the window
 	assert.False(t, b.Check(l, 5))
-	assert.False(t, b.Update(l, 5))
+	u = b.Update(l, 5)
+	assert.False(t, u)
 	assert.EqualValues(t, 15, b.current)
 	g = []bool{false, false, false, false, true, true, false, false, false, false}
 	assert.Equal(t, g, b.bits)
@@ -62,29 +69,10 @@ func TestBits(t *testing.T) {
 
 	// Walk through a few windows in order
 	b = NewBits(10)
-	for i := uint64(1); i <= 100; i++ {
+	for i := uint64(0); i <= 100; i++ {
 		assert.True(t, b.Check(l, i), "Error while checking %v", i)
 		assert.True(t, b.Update(l, i), "Error while updating %v", i)
 	}
-
-	assert.False(t, b.Check(l, 1), "Out of window check")
-}
-
-func TestBitsLargeJumps(t *testing.T) {
-	l := test.NewLogger()
-	b := NewBits(10)
-	b.lostCounter.Clear()
-
-	b = NewBits(10)
-	b.lostCounter.Clear()
-	assert.True(t, b.Update(l, 55)) // We saw packet 55 and can still track 45,46,47,48,49,50,51,52,53,54
-	assert.Equal(t, int64(45), b.lostCounter.Count())
-
-	assert.True(t, b.Update(l, 100)) // We saw packet 55 and 100 and can still track 90,91,92,93,94,95,96,97,98,99
-	assert.Equal(t, int64(89), b.lostCounter.Count())
-
-	assert.True(t, b.Update(l, 200)) // We saw packet 55, 100, and 200 and can still track 190,191,192,193,194,195,196,197,198,199
-	assert.Equal(t, int64(188), b.lostCounter.Count())
 }
 
 func TestBitsDupeCounter(t *testing.T) {
@@ -136,7 +124,8 @@ func TestBitsOutOfWindowCounter(t *testing.T) {
 	assert.False(t, b.Update(l, 0))
 	assert.Equal(t, int64(1), b.outOfWindowCounter.Count())
 
-	assert.Equal(t, int64(19), b.lostCounter.Count()) // packet 0 wasn't lost
+	//tODO: make sure lostcounter doesn't increase in orderly increment
+	assert.Equal(t, int64(20), b.lostCounter.Count())
 	assert.Equal(t, int64(0), b.dupeCounter.Count())
 	assert.Equal(t, int64(1), b.outOfWindowCounter.Count())
 }
@@ -148,6 +137,8 @@ func TestBitsLostCounter(t *testing.T) {
 	b.dupeCounter.Clear()
 	b.outOfWindowCounter.Clear()
 
+	//assert.True(t, b.Update(0))
+	assert.True(t, b.Update(l, 0))
 	assert.True(t, b.Update(l, 20))
 	assert.True(t, b.Update(l, 21))
 	assert.True(t, b.Update(l, 22))
@@ -158,7 +149,7 @@ func TestBitsLostCounter(t *testing.T) {
 	assert.True(t, b.Update(l, 27))
 	assert.True(t, b.Update(l, 28))
 	assert.True(t, b.Update(l, 29))
-	assert.Equal(t, int64(19), b.lostCounter.Count()) // packet 0 wasn't lost
+	assert.Equal(t, int64(20), b.lostCounter.Count())
 	assert.Equal(t, int64(0), b.dupeCounter.Count())
 	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())
 
@@ -167,6 +158,8 @@ func TestBitsLostCounter(t *testing.T) {
 	b.dupeCounter.Clear()
 	b.outOfWindowCounter.Clear()
 
+	assert.True(t, b.Update(l, 0))
+	assert.Equal(t, int64(0), b.lostCounter.Count())
 	assert.True(t, b.Update(l, 9))
 	assert.Equal(t, int64(0), b.lostCounter.Count())
 	// 10 will set 0 index, 0 was already set, no lost packets
@@ -221,62 +214,6 @@ func TestBitsLostCounter(t *testing.T) {
 	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())
 }
 
-func TestBitsLostCounterIssue1(t *testing.T) {
-	l := test.NewLogger()
-	b := NewBits(10)
-	b.lostCounter.Clear()
-	b.dupeCounter.Clear()
-	b.outOfWindowCounter.Clear()
-
-	assert.True(t, b.Update(l, 4))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 1))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 9))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 2))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 3))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 5))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 6))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 7))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	// assert.True(t, b.Update(l, 8))
-	assert.True(t, b.Update(l, 10))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 11))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-
-	assert.True(t, b.Update(l, 14))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	// Issue seems to be here, we reset missing packet 8 to false here and don't increment the lost counter
-	assert.True(t, b.Update(l, 19))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 12))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 13))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 15))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 16))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 17))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 18))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 20))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 21))
-
-	// We missed packet 8 above
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.Equal(t, int64(0), b.dupeCounter.Count())
-	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())
-}
-
 func BenchmarkBits(b *testing.B) {
 	z := NewBits(10)
 	for n := 0; n < b.N; n++ {

boring.go

@@ -1,4 +1,5 @@
 //go:build boringcrypto
+// +build boringcrypto
 
 package nebula
 

cert/ca_pool.go

@@ -1,14 +1,11 @@
 package cert
 
 import (
-	"bufio"
-	"bytes"
-	"encoding/pem"
 	"errors"
 	"fmt"
-	"io"
 	"net/netip"
 	"slices"
+	"strings"
 	"time"
 )
 
@@ -32,46 +29,22 @@ func NewCAPool() *CAPool {
 // If the pool contains any expired certificates, an ErrExpired will be
 // returned along with the pool. The caller must handle any such errors.
 func NewCAPoolFromPEM(caPEMs []byte) (*CAPool, error) {
-	return NewCAPoolFromPEMReader(bytes.NewReader(caPEMs))
-}
-
-// NewCAPoolFromPEMReader will create a new CA pool from the provided reader.
-// The reader must contain a PEM-encoded set of nebula certificates.
-func NewCAPoolFromPEMReader(r io.Reader) (*CAPool, error) {
 	pool := NewCAPool()
-
+	var err error
 	var expired bool
-
-	scanner := bufio.NewScanner(r)
-	scanner.Split(SplitPEM)
-
-	for scanner.Scan() {
-		pemBytes := scanner.Bytes()
-
-		block, rest := pem.Decode(pemBytes)
-		if len(bytes.TrimSpace(rest)) > 0 {
-			return nil, ErrInvalidPEMBlock
+	for {
+		caPEMs, err = pool.AddCAFromPEM(caPEMs)
+		if errors.Is(err, ErrExpired) {
+			expired = true
+			err = nil
 		}
-		if block == nil {
-			return nil, ErrInvalidPEMBlock
-		}
-
-		c, err := unmarshalCertificateBlock(block)
 		if err != nil {
 			return nil, err
 		}
-
-		err = pool.AddCA(c)
-		if errors.Is(err, ErrExpired) {
-			expired = true
-			continue
-		} else if err != nil {
-			return nil, err
+		if len(caPEMs) == 0 || strings.TrimSpace(string(caPEMs)) == "" {
+			break
 		}
 	}
-	if err := scanner.Err(); err != nil {
-		return nil, ErrInvalidPEMBlock
-	}
 
 	if expired {
 		return pool, ErrExpired
@@ -168,23 +141,10 @@ func (ncp *CAPool) VerifyCertificate(now time.Time, c Certificate) (*CachedCerti
 		return nil, err
 	}
 
-	// Pre nebula v1.10.3 could generate signatures in either high or low s form and validation
-	// of signatures allowed for either. Nebula v1.10.3 and beyond clamps signature generation to low-s form
-	// but validation still allows for either. Since a change in the signature bytes affects the fingerprint, we
-	// need to test both forms until such a time comes that we enforce low-s form on signature validation.
-	fp2, err := CalculateAlternateFingerprint(c)
-	if err != nil {
-		return nil, fmt.Errorf("could not calculate alternate fingerprint to verify: %w", err)
-	}
-	if fp2 != "" && ncp.IsBlocklisted(fp2) {
-		return nil, ErrBlockListed
-	}
-
 	cc := CachedCertificate{
 		Certificate:       c,
 		InvertedGroups:    make(map[string]struct{}),
 		Fingerprint:       fp,
-		fingerprint2:      fp2,
 		signerFingerprint: signer.Fingerprint,
 	}
 
@@ -198,11 +158,6 @@ func (ncp *CAPool) VerifyCertificate(now time.Time, c Certificate) (*CachedCerti
 // VerifyCachedCertificate is the same as VerifyCertificate other than it operates on a pre-verified structure and
 // is a cheaper operation to perform as a result.
 func (ncp *CAPool) VerifyCachedCertificate(now time.Time, c *CachedCertificate) error {
-	// Check any available alternate fingerprint forms for this certificate, re P256 high-s/low-s
-	if c.fingerprint2 != "" && ncp.IsBlocklisted(c.fingerprint2) {
-		return ErrBlockListed
-	}
-
 	_, err := ncp.verify(c.Certificate, now, c.Fingerprint, c.signerFingerprint)
 	return err
 }

cert/ca_pool_test.go

@@ -1,14 +1,10 @@
 package cert
 
 import (
-	"bytes"
-	"io"
 	"net/netip"
-	"strings"
 	"testing"
 	"time"
 
-	"github.com/slackhq/nebula/cert/p256"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -115,60 +111,6 @@ k+coOv04r+zh33ISyhbsafnYduN17p2eD7CmHvHuerguXD9f32gcxo/KsFCKEjMe
 	assert.Len(t, ppppp.CAs, 1)
 }
 
-// oneByteReader wraps a reader to return at most 1 byte per Read call,
-// exercising the streaming accumulation logic in NewCAPoolFromPEMReader.
-type oneByteReader struct {
-	r io.Reader
-}
-
-func (o *oneByteReader) Read(p []byte) (int, error) {
-	if len(p) == 0 {
-		return 0, nil
-	}
-	return o.r.Read(p[:1])
-}
-
-func TestNewCAPoolFromPEMReader_EmptyReader(t *testing.T) {
-	pool, err := NewCAPoolFromPEMReader(bytes.NewReader(nil))
-	require.NoError(t, err)
-	assert.Empty(t, pool.CAs)
-
-	pool, err = NewCAPoolFromPEMReader(strings.NewReader("   \n\t\n  "))
-	require.NoError(t, err)
-	assert.Empty(t, pool.CAs)
-}
-
-func TestNewCAPoolFromPEMReader_OneByteReads(t *testing.T) {
-	ca1, _, _, pem1 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil)
-	ca2, _, _, pem2 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil)
-
-	bundle := append(pem1, pem2...)
-	pool, err := NewCAPoolFromPEMReader(&oneByteReader{r: bytes.NewReader(bundle)})
-	require.NoError(t, err)
-	assert.Len(t, pool.CAs, 2)
-
-	fp1, err := ca1.Fingerprint()
-	require.NoError(t, err)
-	fp2, err := ca2.Fingerprint()
-	require.NoError(t, err)
-
-	assert.Contains(t, pool.CAs, fp1)
-	assert.Contains(t, pool.CAs, fp2)
-}
-
-func TestNewCAPoolFromPEMReader_TruncatedPEM(t *testing.T) {
-	_, err := NewCAPoolFromPEMReader(strings.NewReader("-----BEGIN NEBULA CERTIFICATE-----\npartialdata"))
-	assert.ErrorIs(t, err, ErrInvalidPEMBlock)
-}
-
-func TestNewCAPoolFromPEMReader_TrailingGarbage(t *testing.T) {
-	_, _, _, pem1 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil)
-
-	bundle := append(pem1, []byte("some trailing garbage")...)
-	_, err := NewCAPoolFromPEMReader(bytes.NewReader(bundle))
-	assert.ErrorIs(t, err, ErrInvalidPEMBlock)
-}
-
 func TestCertificateV1_Verify(t *testing.T) {
 	ca, _, caKey, _ := NewTestCaCert(Version1, Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, nil)
 	c, _, _, _ := NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test cert", time.Now(), time.Now().Add(5*time.Minute), nil, nil, nil)
@@ -228,15 +170,6 @@ func TestCertificateV1_VerifyP256(t *testing.T) {
 	_, err = caPool.VerifyCertificate(time.Now(), c)
 	require.EqualError(t, err, "certificate is in the block list")
 
-	// Create a copy of the cert and swap to the alternate form for the signature
-	nc := c.Copy()
-	b, err := p256.Swap(c.Signature())
-	require.NoError(t, err)
-	require.NoError(t, nc.(*certificateV1).setSignature(b))
-
-	_, err = caPool.VerifyCertificate(time.Now(), nc)
-	require.EqualError(t, err, "certificate is in the block list")
-
 	caPool.ResetCertBlocklist()
 	_, err = caPool.VerifyCertificate(time.Now(), c)
 	require.NoError(t, err)
@@ -254,7 +187,7 @@ func TestCertificateV1_VerifyP256(t *testing.T) {
 	require.NoError(t, err)
 
 	caPool = NewCAPool()
-	b, err = caPool.AddCAFromPEM(caPem)
+	b, err := caPool.AddCAFromPEM(caPem)
 	require.NoError(t, err)
 	assert.Empty(t, b)
 
@@ -263,17 +196,7 @@ func TestCertificateV1_VerifyP256(t *testing.T) {
 	})
 
 	c, _, _, _ = NewTestCert(Version1, Curve_P256, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, nil, []string{"test1"})
-	cc, err := caPool.VerifyCertificate(time.Now(), c)
-	require.NoError(t, err)
-
-	// Reset the blocklist and block the alternate form fingerprint
-	caPool.ResetCertBlocklist()
-	caPool.BlocklistFingerprint(cc.fingerprint2)
-	err = caPool.VerifyCachedCertificate(time.Now(), cc)
-	require.EqualError(t, err, "certificate is in the block list")
-
-	caPool.ResetCertBlocklist()
-	err = caPool.VerifyCachedCertificate(time.Now(), cc)
+	_, err = caPool.VerifyCertificate(time.Now(), c)
 	require.NoError(t, err)
 }
 
@@ -471,15 +394,6 @@ func TestCertificateV2_VerifyP256(t *testing.T) {
 	_, err = caPool.VerifyCertificate(time.Now(), c)
 	require.EqualError(t, err, "certificate is in the block list")
 
-	// Create a copy of the cert and swap to the alternate form for the signature
-	nc := c.Copy()
-	b, err := p256.Swap(c.Signature())
-	require.NoError(t, err)
-	require.NoError(t, nc.(*certificateV2).setSignature(b))
-
-	_, err = caPool.VerifyCertificate(time.Now(), nc)
-	require.EqualError(t, err, "certificate is in the block list")
-
 	caPool.ResetCertBlocklist()
 	_, err = caPool.VerifyCertificate(time.Now(), c)
 	require.NoError(t, err)
@@ -497,7 +411,7 @@ func TestCertificateV2_VerifyP256(t *testing.T) {
 	require.NoError(t, err)
 
 	caPool = NewCAPool()
-	b, err = caPool.AddCAFromPEM(caPem)
+	b, err := caPool.AddCAFromPEM(caPem)
 	require.NoError(t, err)
 	assert.Empty(t, b)
 
@@ -506,17 +420,7 @@ func TestCertificateV2_VerifyP256(t *testing.T) {
 	})
 
 	c, _, _, _ = NewTestCert(Version2, Curve_P256, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, nil, []string{"test1"})
-	cc, err := caPool.VerifyCertificate(time.Now(), c)
-	require.NoError(t, err)
-
-	// Reset the blocklist and block the alternate form fingerprint
-	caPool.ResetCertBlocklist()
-	caPool.BlocklistFingerprint(cc.fingerprint2)
-	err = caPool.VerifyCachedCertificate(time.Now(), cc)
-	require.EqualError(t, err, "certificate is in the block list")
-
-	caPool.ResetCertBlocklist()
-	err = caPool.VerifyCachedCertificate(time.Now(), cc)
+	_, err = caPool.VerifyCertificate(time.Now(), c)
 	require.NoError(t, err)
 }
 

cert/cert.go

@@ -4,8 +4,6 @@ import (
 	"fmt"
 	"net/netip"
 	"time"
-
-	"github.com/slackhq/nebula/cert/p256"
 )
 
 type Version uint8
@@ -112,9 +110,6 @@ type CachedCertificate struct {
 	InvertedGroups    map[string]struct{}
 	Fingerprint       string
 	signerFingerprint string
-
-	// A place to store a 2nd fingerprint if the certificate could have one, such as with P256
-	fingerprint2 string
 }
 
 func (cc *CachedCertificate) String() string {
@@ -124,7 +119,6 @@ func (cc *CachedCertificate) String() string {
 // Recombine will attempt to unmarshal a certificate received in a handshake.
 // Handshakes save space by placing the peers public key in a different part of the packet, we have to
 // reassemble the actual certificate structure with that in mind.
-// Implementations MUST assert the public key is not in the raw certificate bytes if the passed in public key is not empty.
 func Recombine(v Version, rawCertBytes, publicKey []byte, curve Curve) (Certificate, error) {
 	if publicKey == nil {
 		return nil, ErrNoPeerStaticKey
@@ -157,31 +151,3 @@ func Recombine(v Version, rawCertBytes, publicKey []byte, curve Curve) (Certific
 
 	return c, nil
 }
-
-// CalculateAlternateFingerprint calculates a 2nd fingerprint representation for P256 certificates
-// CAPool blocklist testing through `VerifyCertificate` and `VerifyCachedCertificate` automatically performs this step.
-func CalculateAlternateFingerprint(c Certificate) (string, error) {
-	if c.Curve() != Curve_P256 {
-		return "", nil
-	}
-
-	nc := c.Copy()
-	b, err := p256.Swap(nc.Signature())
-	if err != nil {
-		return "", err
-	}
-
-	switch v := nc.(type) {
-	case *certificateV1:
-		err = v.setSignature(b)
-	case *certificateV2:
-		err = v.setSignature(b)
-	default:
-		return "", ErrUnknownVersion
-	}
-
-	if err != nil {
-		return "", err
-	}
-	return nc.Fingerprint()
-}

cert/cert_v1.go

@@ -426,7 +426,7 @@ func unmarshalCertificateV1(b []byte, publicKey []byte) (*certificateV1, error)
 			unsafeNetworks: make([]netip.Prefix, len(rc.Details.Subnets)/2),
 			notBefore:      time.Unix(rc.Details.NotBefore, 0),
 			notAfter:       time.Unix(rc.Details.NotAfter, 0),
-			publicKey:      nil,
+			publicKey:      make([]byte, len(rc.Details.PublicKey)),
 			isCA:           rc.Details.IsCA,
 			curve:          rc.Details.Curve,
 		},
@@ -437,19 +437,12 @@ func unmarshalCertificateV1(b []byte, publicKey []byte) (*certificateV1, error)
 	copy(nc.details.groups, rc.Details.Groups)
 	nc.details.issuer = hex.EncodeToString(rc.Details.Issuer)
 
-	// If a public key is passed in as an argument, the certificate pubkey must be empty
-	// and the passed-in pubkey copied into the cert.
 	if len(publicKey) > 0 {
-		if len(rc.Details.PublicKey) != 0 {
-			return nil, ErrCertPubkeyPresent
-		}
-		nc.details.publicKey = make([]byte, len(publicKey))
-		copy(nc.details.publicKey, publicKey)
-	} else {
-		nc.details.publicKey = make([]byte, len(rc.Details.PublicKey))
-		copy(nc.details.publicKey, rc.Details.PublicKey)
+		nc.details.publicKey = publicKey
 	}
 
+	copy(nc.details.publicKey, rc.Details.PublicKey)
+
 	var ip netip.Addr
 	for i, rawIp := range rc.Details.Ips {
 		if i%2 == 0 {

cert/cert_v1_test.go

@@ -62,62 +62,6 @@ func TestCertificateV1_Marshal(t *testing.T) {
 	assert.Equal(t, nc.Groups(), nc2.Groups())
 }
 
-func TestCertificateV1_Unmarshal(t *testing.T) {
-	t.Parallel()
-	before := time.Now().Add(time.Second * -60).Round(time.Second)
-	after := time.Now().Add(time.Second * 60).Round(time.Second)
-	pubKey := []byte("1234567890abcedfghij1234567890ab")
-	invalidPubkey := []byte("00000000000000000000000000000000")
-
-	nc := certificateV1{
-		details: detailsV1{
-			name: "testing",
-			networks: []netip.Prefix{
-				mustParsePrefixUnmapped("10.1.1.1/24"),
-				mustParsePrefixUnmapped("10.1.1.2/16"),
-			},
-			unsafeNetworks: []netip.Prefix{
-				mustParsePrefixUnmapped("9.1.1.2/24"),
-				mustParsePrefixUnmapped("9.1.1.3/16"),
-			},
-			groups:    []string{"test-group1", "test-group2", "test-group3"},
-			notBefore: before,
-			notAfter:  after,
-			publicKey: pubKey,
-			isCA:      false,
-			issuer:    "1234567890abcedfghij1234567890ab",
-		},
-		signature: []byte("1234567890abcedfghij1234567890ab"),
-	}
-
-	// This certificate has a pubkey included
-	certWithPubkey, err := nc.Marshal()
-	require.NoError(t, err)
-
-	// This certificate is missing the pubkey section
-	certWithoutPubkey, err := nc.MarshalForHandshakes()
-	require.NoError(t, err)
-
-	// Cert has no pubkey and no pubkey passed in must fail to validate
-	isNil, err := unmarshalCertificateV1(certWithoutPubkey, nil)
-	require.Error(t, err)
-
-	// Cert has different pubkey than one passed in must fail
-	isNil, err = unmarshalCertificateV1(certWithPubkey, invalidPubkey)
-	require.Nil(t, isNil)
-	require.Error(t, err)
-
-	// Cert has pubkey and no pubkey argument works ok
-	_, err = unmarshalCertificateV1(certWithPubkey, nil)
-	require.NoError(t, err)
-
-	// Cert has no pubkey and valid, correctly signed pubkey passed in
-	nc2, err := unmarshalCertificateV1(certWithoutPubkey, pubKey)
-	require.NoError(t, err)
-
-	assert.Equal(t, pubKey, nc2.PublicKey())
-}
-
 func TestCertificateV1_PublicKeyPem(t *testing.T) {
 	t.Parallel()
 	before := time.Now().Add(time.Second * -60).Round(time.Second)
@@ -155,19 +99,13 @@ func TestCertificateV1_PublicKeyPem(t *testing.T) {
 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
 AAAAAAAAAAAAAAAAAAAAAAA=
 -----END NEBULA P256 PUBLIC KEY-----
-`)
-
-	pubP256KeyPemCA := []byte(`-----BEGIN NEBULA ECDSA P256 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA ECDSA P256 PUBLIC KEY-----
 `)
 	pubP256Key, _, _, err := UnmarshalPublicKeyFromPEM(pubP256KeyPem)
 	require.NoError(t, err)
 	nc.details.curve = Curve_P256
 	nc.details.publicKey = pubP256Key
 	assert.Equal(t, Curve_P256, nc.Curve())
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPemCA))
+	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPem))
 	assert.True(t, nc.IsCA())
 
 	nc.details.isCA = false

cert/cert_v2.go

@@ -592,13 +592,7 @@ func unmarshalCertificateV2(b []byte, publicKey []byte, curve Curve) (*certifica
 	// Maybe grab the public key
 	var rawPublicKey cryptobyte.String
 	if len(publicKey) > 0 {
-		// If a public key is passed in, then the handshake certificate must
-		// not have a public key present
-		if input.PeekASN1Tag(TagCertPublicKey) {
-			return nil, ErrCertPubkeyPresent
-		}
-		rawPublicKey = make(cryptobyte.String, len(publicKey))
-		copy(rawPublicKey, publicKey)
+		rawPublicKey = publicKey
 	} else if !input.ReadOptionalASN1(&rawPublicKey, nil, TagCertPublicKey) {
 		return nil, ErrBadFormat
 	}

cert/cert_v2_test.go

@@ -76,58 +76,6 @@ func TestCertificateV2_Marshal(t *testing.T) {
 	assert.Equal(t, nc.Groups(), nc2.Groups())
 }
 
-func TestCertificateV2_Unmarshal(t *testing.T) {
-	t.Parallel()
-	before := time.Now().Add(time.Second * -60).Round(time.Second)
-	after := time.Now().Add(time.Second * 60).Round(time.Second)
-	pubKey := []byte("1234567890abcedfghij1234567890ab")
-
-	nc := certificateV2{
-		details: detailsV2{
-			name: "testing",
-			networks: []netip.Prefix{
-				mustParsePrefixUnmapped("10.1.1.2/16"),
-				mustParsePrefixUnmapped("10.1.1.1/24"),
-			},
-			unsafeNetworks: []netip.Prefix{
-				mustParsePrefixUnmapped("9.1.1.3/16"),
-				mustParsePrefixUnmapped("9.1.1.2/24"),
-			},
-			groups:    []string{"test-group1", "test-group2", "test-group3"},
-			notBefore: before,
-			notAfter:  after,
-			isCA:      false,
-			issuer:    "1234567890abcdef1234567890abcdef",
-		},
-		signature: []byte("1234567890abcdef1234567890abcdef"),
-		publicKey: pubKey,
-	}
-
-	db, err := nc.details.Marshal()
-	require.NoError(t, err)
-	nc.rawDetails = db
-
-	certWithPubkey, err := nc.Marshal()
-	require.NoError(t, err)
-	//t.Log("Cert size:", len(b))
-	certWithoutPubkey, err := nc.MarshalForHandshakes()
-	require.NoError(t, err)
-
-	// Cert must not have a pubkey if one is passed in as an argument
-	_, err = unmarshalCertificateV2(certWithPubkey, pubKey, Curve_CURVE25519)
-	require.ErrorIs(t, err, ErrCertPubkeyPresent)
-
-	// Certs must have pubkeys
-	_, err = unmarshalCertificateV2(certWithoutPubkey, nil, Curve_CURVE25519)
-	require.ErrorIs(t, err, ErrBadFormat)
-
-	// Ensure proper unmarshal if a pubkey is passed in
-	nc2, err := unmarshalCertificateV2(certWithoutPubkey, pubKey, Curve_CURVE25519)
-	require.NoError(t, err)
-
-	assert.Equal(t, nc.PublicKey(), nc2.PublicKey())
-}
-
 func TestCertificateV2_PublicKeyPem(t *testing.T) {
 	t.Parallel()
 	before := time.Now().Add(time.Second * -60).Round(time.Second)
@@ -166,19 +114,12 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
 AAAAAAAAAAAAAAAAAAAAAAA=
 -----END NEBULA P256 PUBLIC KEY-----
 `)
-
-	pubP256KeyPemCA := []byte(`-----BEGIN NEBULA ECDSA P256 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA ECDSA P256 PUBLIC KEY-----
-`)
-
 	pubP256Key, _, _, err := UnmarshalPublicKeyFromPEM(pubP256KeyPem)
 	require.NoError(t, err)
 	nc.curve = Curve_P256
 	nc.publicKey = pubP256Key
 	assert.Equal(t, Curve_P256, nc.Curve())
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPemCA))
+	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPem))
 	assert.True(t, nc.IsCA())
 
 	nc.details.isCA = false

cert/crypto_test.go

@@ -79,7 +79,7 @@ qrlJ69wer3ZUHFXA
 	assert.Nil(t, k)
 	assert.Equal(t, rest, invalidPem)
 
-	// Fail due to invalid PEM format, because
+	// Fail due to ivalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	curve, k, rest, err = DecryptAndUnmarshalSigningPrivateKey(passphrase, rest)
 	require.EqualError(t, err, "input did not contain a valid PEM encoded block")

cert/errors.go

@@ -21,7 +21,6 @@ var (
 	ErrPrivateKeyEncrypted        = errors.New("private key must be decrypted")
 	ErrCaNotFound                 = errors.New("could not find ca for the certificate")
 	ErrUnknownVersion             = errors.New("certificate version unrecognized")
-	ErrCertPubkeyPresent          = errors.New("certificate has unexpected pubkey present")
 
 	ErrInvalidPEMBlock                   = errors.New("input did not contain a valid PEM encoded block")
 	ErrInvalidPEMCertificateBanner       = errors.New("bytes did not contain a proper certificate banner")

cert/p256/p256.go

@@ -1,127 +0,0 @@
-package p256
-
-import (
-	"crypto/elliptic"
-	"errors"
-	"math/big"
-
-	"filippo.io/bigmod"
-
-	"golang.org/x/crypto/cryptobyte"
-	"golang.org/x/crypto/cryptobyte/asn1"
-)
-
-var halfN = new(big.Int).Rsh(elliptic.P256().Params().N, 1)
-var nMod *bigmod.Modulus
-
-func init() {
-	n, err := bigmod.NewModulus(elliptic.P256().Params().N.Bytes())
-	if err != nil {
-		panic(err)
-	}
-	nMod = n
-}
-
-func IsNormalized(sig []byte) (bool, error) {
-	r, s, err := parseSignature(sig)
-	if err != nil {
-		return false, err
-	}
-	return checkLowS(r, s), nil
-}
-
-func checkLowS(_, s []byte) bool {
-	bigS := new(big.Int).SetBytes(s)
-	// Check if S <= (N/2), because we want to include the midpoint in the set of low-s
-	return bigS.Cmp(halfN) <= 0
-}
-
-func swap(r, s []byte) ([]byte, []byte, error) {
-	var err error
-	bigS, err := bigmod.NewNat().SetBytes(s, nMod)
-	if err != nil {
-		return nil, nil, err
-	}
-	sNormalized := nMod.Nat().Sub(bigS, nMod)
-
-	result := sNormalized.Bytes(nMod)
-	for len(result) > 1 && result[0] == 0 {
-		result = result[1:]
-	}
-
-	return r, result, nil
-}
-
-func Normalize(sig []byte) ([]byte, error) {
-	r, s, err := parseSignature(sig)
-	if err != nil {
-		return nil, err
-	}
-
-	if checkLowS(r, s) {
-		return sig, nil
-	}
-
-	newR, newS, err := swap(r, s)
-	if err != nil {
-		return nil, err
-	}
-
-	return encodeSignature(newR, newS)
-}
-
-// Swap will change sig between its current form to the opposite high or low form.
-func Swap(sig []byte) ([]byte, error) {
-	r, s, err := parseSignature(sig)
-	if err != nil {
-		return nil, err
-	}
-
-	newR, newS, err := swap(r, s)
-	if err != nil {
-		return nil, err
-	}
-
-	return encodeSignature(newR, newS)
-}
-
-// parseSignature taken exactly from crypto/ecdsa/ecdsa.go
-func parseSignature(sig []byte) (r, s []byte, err error) {
-	var inner cryptobyte.String
-	input := cryptobyte.String(sig)
-	if !input.ReadASN1(&inner, asn1.SEQUENCE) ||
-		!input.Empty() ||
-		!inner.ReadASN1Integer(&r) ||
-		!inner.ReadASN1Integer(&s) ||
-		!inner.Empty() {
-		return nil, nil, errors.New("invalid ASN.1")
-	}
-	return r, s, nil
-}
-
-func encodeSignature(r, s []byte) ([]byte, error) {
-	var b cryptobyte.Builder
-	b.AddASN1(asn1.SEQUENCE, func(b *cryptobyte.Builder) {
-		addASN1IntBytes(b, r)
-		addASN1IntBytes(b, s)
-	})
-	return b.Bytes()
-}
-
-// addASN1IntBytes encodes in ASN.1 a positive integer represented as
-// a big-endian byte slice with zero or more leading zeroes.
-func addASN1IntBytes(b *cryptobyte.Builder, bytes []byte) {
-	for len(bytes) > 0 && bytes[0] == 0 {
-		bytes = bytes[1:]
-	}
-	if len(bytes) == 0 {
-		b.SetError(errors.New("invalid integer"))
-		return
-	}
-	b.AddASN1(asn1.INTEGER, func(c *cryptobyte.Builder) {
-		if bytes[0]&0x80 != 0 {
-			c.AddUint8(0)
-		}
-		c.AddBytes(bytes)
-	})
-}

cert/p256/p256_test.go

@@ -1,28 +0,0 @@
-package p256
-
-import (
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestFlipping(t *testing.T) {
-	priv, err1 := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	require.NoError(t, err1)
-
-	out, err := ecdsa.SignASN1(rand.Reader, priv, []byte("big chungus"))
-	require.NoError(t, err)
-
-	r, s, err := parseSignature(out)
-	require.NoError(t, err)
-
-	r, s1, err := swap(r, s)
-	require.NoError(t, err)
-	r, s2, err := swap(r, s1)
-	require.NoError(t, err)
-	require.Equal(t, s, s2)
-	require.NotEqual(t, s, s1)
-}

cert/pem.go

@@ -1,66 +1,14 @@
 package cert
 
 import (
-	"bytes"
+	"encoding/hex"
 	"encoding/pem"
-	"errors"
 	"fmt"
+	"time"
 
 	"golang.org/x/crypto/ed25519"
 )
 
-var ErrTruncatedPEMBlock = errors.New("truncated PEM block")
-
-// SplitPEM is a split function for bufio.Scanner that returns each PEM block.
-func SplitPEM(data []byte, atEOF bool) (advance int, token []byte, err error) {
-	// Look for the start of a PEM block
-	start := bytes.Index(data, []byte("-----BEGIN "))
-	if start == -1 {
-		if atEOF && len(bytes.TrimSpace(data)) > 0 {
-			// Non-whitespace content with no PEM block
-			return 0, nil, ErrTruncatedPEMBlock
-		}
-		if atEOF {
-			return len(data), nil, nil
-		}
-		// Request more data
-		return 0, nil, nil
-	}
-
-	// Look for the end marker
-	endMarkerStart := bytes.Index(data[start:], []byte("-----END "))
-	if endMarkerStart == -1 {
-		if atEOF {
-			// Incomplete PEM block at EOF
-			return 0, nil, ErrTruncatedPEMBlock
-		}
-		// Need more data to find the end
-		return 0, nil, nil
-	}
-
-	// Find the actual end of the END line (after the newline)
-	endMarkerStart += start
-	endLineEnd := bytes.IndexByte(data[endMarkerStart:], '\n')
-	var end int
-	if endLineEnd == -1 {
-		if atEOF {
-			// END marker without newline at EOF - take it anyway
-			end = len(data)
-		} else {
-			// Need more data
-			return 0, nil, nil
-		}
-	} else {
-		end = endMarkerStart + endLineEnd + 1
-	}
-
-	// Extract the PEM block
-	pemBlock := data[start:end]
-
-	// Return the valid PEM block
-	return end, pemBlock, nil
-}
-
 const ( //cert banners
 	CertificateBanner   = "NEBULA CERTIFICATE"
 	CertificateV2Banner = "NEBULA CERTIFICATE V2"
@@ -91,7 +39,19 @@ func UnmarshalCertificateFromPEM(b []byte) (Certificate, []byte, error) {
 		return nil, r, ErrInvalidPEMBlock
 	}
 
-	c, err := unmarshalCertificateBlock(p)
+	var c Certificate
+	var err error
+
+	switch p.Type {
+	// Implementations must validate the resulting certificate contains valid information
+	case CertificateBanner:
+		c, err = unmarshalCertificateV1(p.Bytes, nil)
+	case CertificateV2Banner:
+		c, err = unmarshalCertificateV2(p.Bytes, nil, Curve_CURVE25519)
+	default:
+		return nil, r, ErrInvalidPEMCertificateBanner
+	}
+
 	if err != nil {
 		return nil, r, err
 	}
@@ -100,20 +60,6 @@ func UnmarshalCertificateFromPEM(b []byte) (Certificate, []byte, error) {
 
 }
 
-// unmarshalCertificateBlock decodes a single PEM block into a certificate.
-// It expects a Nebula certificate banner and returns ErrInvalidPEMCertificateBanner otherwise.
-func unmarshalCertificateBlock(block *pem.Block) (Certificate, error) {
-	switch block.Type {
-	// Implementations must validate the resulting certificate contains valid information
-	case CertificateBanner:
-		return unmarshalCertificateV1(block.Bytes, nil)
-	case CertificateV2Banner:
-		return unmarshalCertificateV2(block.Bytes, nil, Curve_CURVE25519)
-	default:
-		return nil, ErrInvalidPEMCertificateBanner
-	}
-}
-
 func marshalCertPublicKeyToPEM(c Certificate) []byte {
 	if c.IsCA() {
 		return MarshalSigningPublicKeyToPEM(c.Curve(), c.PublicKey())
@@ -142,7 +88,7 @@ func MarshalSigningPublicKeyToPEM(curve Curve, b []byte) []byte {
 	case Curve_CURVE25519:
 		return pem.EncodeToMemory(&pem.Block{Type: Ed25519PublicKeyBanner, Bytes: b})
 	case Curve_P256:
-		return pem.EncodeToMemory(&pem.Block{Type: ECDSAP256PublicKeyBanner, Bytes: b})
+		return pem.EncodeToMemory(&pem.Block{Type: P256PublicKeyBanner, Bytes: b})
 	default:
 		return nil
 	}
@@ -194,6 +140,101 @@ func MarshalSigningPrivateKeyToPEM(curve Curve, b []byte) []byte {
 	}
 }
 
+// Backward compatibility functions for older API
+func MarshalX25519PublicKey(b []byte) []byte {
+	return MarshalPublicKeyToPEM(Curve_CURVE25519, b)
+}
+
+func MarshalX25519PrivateKey(b []byte) []byte {
+	return MarshalPrivateKeyToPEM(Curve_CURVE25519, b)
+}
+
+func MarshalPublicKey(curve Curve, b []byte) []byte {
+	return MarshalPublicKeyToPEM(curve, b)
+}
+
+func MarshalPrivateKey(curve Curve, b []byte) []byte {
+	return MarshalPrivateKeyToPEM(curve, b)
+}
+
+// NebulaCertificate is a compatibility wrapper for the old API
+type NebulaCertificate struct {
+	Details   NebulaCertificateDetails
+	Signature []byte
+	cert      Certificate
+}
+
+// NebulaCertificateDetails is a compatibility wrapper for certificate details
+type NebulaCertificateDetails struct {
+	Name      string
+	NotBefore time.Time
+	NotAfter  time.Time
+	PublicKey []byte
+	IsCA      bool
+	Issuer    []byte
+	Curve     Curve
+}
+
+// UnmarshalNebulaCertificateFromPEM provides backward compatibility with the old API
+func UnmarshalNebulaCertificateFromPEM(b []byte) (*NebulaCertificate, []byte, error) {
+	c, rest, err := UnmarshalCertificateFromPEM(b)
+	if err != nil {
+		return nil, rest, err
+	}
+
+	issuerBytes, err := func() ([]byte, error) {
+		issuer := c.Issuer()
+		if issuer == "" {
+			return nil, nil
+		}
+		decoded, err := hex.DecodeString(issuer)
+		if err != nil {
+			return nil, fmt.Errorf("failed to decode issuer fingerprint: %w", err)
+		}
+		return decoded, nil
+	}()
+	if err != nil {
+		return nil, rest, err
+	}
+
+	pubKey := c.PublicKey()
+	if pubKey != nil {
+		pubKey = append([]byte(nil), pubKey...)
+	}
+
+	sig := c.Signature()
+	if sig != nil {
+		sig = append([]byte(nil), sig...)
+	}
+
+	return &NebulaCertificate{
+		Details: NebulaCertificateDetails{
+			Name:      c.Name(),
+			NotBefore: c.NotBefore(),
+			NotAfter:  c.NotAfter(),
+			PublicKey: pubKey,
+			IsCA:      c.IsCA(),
+			Issuer:    issuerBytes,
+			Curve:     c.Curve(),
+		},
+		Signature: sig,
+		cert:      c,
+	}, rest, nil
+}
+
+// IssuerString returns the issuer in hex format for compatibility
+func (n *NebulaCertificate) IssuerString() string {
+	if n.Details.Issuer == nil {
+		return ""
+	}
+	return hex.EncodeToString(n.Details.Issuer)
+}
+
+// Certificate returns the underlying certificate (read-only)
+func (n *NebulaCertificate) Certificate() Certificate {
+	return n.cert
+}
+
 // UnmarshalPrivateKeyFromPEM will try to unmarshal the first pem block in a byte array, returning any non
 // consumed data or an error on failure
 func UnmarshalPrivateKeyFromPEM(b []byte) ([]byte, []byte, Curve, error) {

cert/pem_test.go

@@ -1,88 +1,12 @@
 package cert
 
 import (
-	"bufio"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func scanAll(t *testing.T, input string) ([]string, error) {
-	t.Helper()
-	scanner := bufio.NewScanner(strings.NewReader(input))
-	scanner.Split(SplitPEM)
-	var blocks []string
-	for scanner.Scan() {
-		blocks = append(blocks, scanner.Text())
-	}
-	return blocks, scanner.Err()
-}
-
-func TestSplitPEM_Single(t *testing.T) {
-	input := "-----BEGIN TEST-----\ndata\n-----END TEST-----\n"
-	blocks, err := scanAll(t, input)
-	require.NoError(t, err)
-	require.Len(t, blocks, 1)
-	require.Equal(t, input, blocks[0])
-}
-
-func TestSplitPEM_Multiple(t *testing.T) {
-	block1 := "-----BEGIN TEST-----\naaa\n-----END TEST-----\n"
-	block2 := "-----BEGIN TEST-----\nbbb\n-----END TEST-----\n"
-	blocks, err := scanAll(t, block1+block2)
-	require.NoError(t, err)
-	require.Len(t, blocks, 2)
-	require.Equal(t, block1, blocks[0])
-	require.Equal(t, block2, blocks[1])
-}
-
-func TestSplitPEM_CommentsAndWhitespaceBetweenBlocks(t *testing.T) {
-	input := "# comment\n\n-----BEGIN TEST-----\naaa\n-----END TEST-----\n\n# another comment\n\n-----BEGIN TEST-----\nbbb\n-----END TEST-----\n"
-	blocks, err := scanAll(t, input)
-	require.NoError(t, err)
-	require.Len(t, blocks, 2)
-}
-
-func TestSplitPEM_Empty(t *testing.T) {
-	blocks, err := scanAll(t, "")
-	require.NoError(t, err)
-	require.Empty(t, blocks)
-}
-
-func TestSplitPEM_WhitespaceOnly(t *testing.T) {
-	blocks, err := scanAll(t, "  \n\t\n  ")
-	require.NoError(t, err)
-	require.Empty(t, blocks)
-}
-
-func TestSplitPEM_TrailingGarbage(t *testing.T) {
-	input := "-----BEGIN TEST-----\ndata\n-----END TEST-----\ngarbage"
-	blocks, err := scanAll(t, input)
-	require.ErrorIs(t, err, ErrTruncatedPEMBlock)
-	require.Len(t, blocks, 1)
-}
-
-func TestSplitPEM_TruncatedBlock(t *testing.T) {
-	input := "-----BEGIN TEST-----\npartial data with no end"
-	_, err := scanAll(t, input)
-	require.ErrorIs(t, err, ErrTruncatedPEMBlock)
-}
-
-func TestSplitPEM_NoEndNewline(t *testing.T) {
-	input := "-----BEGIN TEST-----\ndata\n-----END TEST-----"
-	blocks, err := scanAll(t, input)
-	require.NoError(t, err)
-	require.Len(t, blocks, 1)
-	require.Equal(t, input, blocks[0])
-}
-
-func TestSplitPEM_GarbageOnly(t *testing.T) {
-	_, err := scanAll(t, "this is not PEM data")
-	require.ErrorIs(t, err, ErrTruncatedPEMBlock)
-}
-
 func TestUnmarshalCertificateFromPEM(t *testing.T) {
 	goodCert := []byte(`
 # A good cert
@@ -120,7 +44,7 @@ bzBEr00kERQxxTzTsH8cpYEgRoipvmExvg8WP8NdAJEYJosB
 	assert.Equal(t, rest, invalidPem)
 	require.EqualError(t, err, "bytes did not contain a proper certificate banner")
 
-	// Fail due to invalid PEM format, because
+	// Fail due to ivalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	cert, rest, err = UnmarshalCertificateFromPEM(rest)
 	assert.Nil(t, cert)
@@ -182,7 +106,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
 	assert.Equal(t, rest, invalidPem)
 	require.EqualError(t, err, "bytes did not contain a proper Ed25519/ECDSA private key banner")
 
-	// Fail due to invalid PEM format, because
+	// Fail due to ivalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	k, rest, curve, err = UnmarshalSigningPrivateKeyFromPEM(rest)
 	assert.Nil(t, k)
@@ -244,7 +168,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 	assert.Equal(t, rest, invalidPem)
 	require.EqualError(t, err, "bytes did not contain a proper private key banner")
 
-	// Fail due to invalid PEM format, because
+	// Fail due to ivalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	k, rest, curve, err = UnmarshalPrivateKeyFromPEM(rest)
 	assert.Nil(t, k)
@@ -297,7 +221,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 	require.EqualError(t, err, "bytes did not contain a proper public key banner")
 	assert.Equal(t, rest, invalidPem)
 
-	// Fail due to invalid PEM format, because
+	// Fail due to ivalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
 	assert.Nil(t, k)
@@ -375,7 +299,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 	require.EqualError(t, err, "bytes did not contain a proper public key banner")
 	assert.Equal(t, rest, invalidPem)
 
-	// Fail due to invalid PEM format, because
+	// Fail due to ivalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
 	assert.Nil(t, k)

cert/sign.go

@@ -9,8 +9,6 @@ import (
 	"fmt"
 	"net/netip"
 	"time"
-
-	"github.com/slackhq/nebula/cert/p256"
 )
 
 // TBSCertificate represents a certificate intended to be signed.
@@ -128,13 +126,6 @@ func (t *TBSCertificate) SignWith(signer Certificate, curve Curve, sp SignerLamb
 		return nil, err
 	}
 
-	if curve == Curve_P256 {
-		sig, err = p256.Normalize(sig)
-		if err != nil {
-			return nil, err
-		}
-	}
-
 	err = c.setSignature(sig)
 	if err != nil {
 		return nil, err

cert/sign_test.go

@@ -9,7 +9,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/slackhq/nebula/cert/p256"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -90,48 +89,3 @@ func TestCertificateV1_SignP256(t *testing.T) {
 	require.NoError(t, err)
 	assert.NotNil(t, uc)
 }
-
-func TestCertificate_SignP256_AlwaysNormalized(t *testing.T) {
-	before := time.Now().Add(time.Second * -60).Round(time.Second)
-	after := time.Now().Add(time.Second * 60).Round(time.Second)
-	pubKey := []byte("01234567890abcedfghij1234567890ab1234567890abcedfghij1234567890ab")
-
-	tbs := TBSCertificate{
-		Version: Version1,
-		Name:    "testing",
-		Networks: []netip.Prefix{
-			mustParsePrefixUnmapped("10.1.1.1/24"),
-			mustParsePrefixUnmapped("10.1.1.2/16"),
-		},
-		UnsafeNetworks: []netip.Prefix{
-			mustParsePrefixUnmapped("9.1.1.2/24"),
-			mustParsePrefixUnmapped("9.1.1.3/16"),
-		},
-		Groups:    []string{"test-group1", "test-group2", "test-group3"},
-		NotBefore: before,
-		NotAfter:  after,
-		PublicKey: pubKey,
-		IsCA:      true,
-		Curve:     Curve_P256,
-	}
-
-	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	require.NoError(t, err)
-	pub := elliptic.Marshal(elliptic.P256(), priv.PublicKey.X, priv.PublicKey.Y)
-	rawPriv := priv.D.FillBytes(make([]byte, 32))
-
-	for i := 0; i < 1000; i++ {
-		if i&1 == 1 {
-			tbs.Version = Version1
-		} else {
-			tbs.Version = Version2
-		}
-		c, err := tbs.Sign(nil, Curve_P256, rawPriv)
-		require.NoError(t, err)
-		assert.NotNil(t, c)
-		assert.True(t, c.CheckSignature(pub))
-		normie, err := p256.IsNormalized(c.Signature())
-		require.NoError(t, err)
-		assert.True(t, normie)
-	}
-}

cmd/nebula-cert/ca.go

@@ -173,26 +173,23 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error
 
 	var passphrase []byte
 	if !isP11 && *cf.encryption {
-		passphrase = []byte(os.Getenv("NEBULA_CA_PASSPHRASE"))
+		for i := 0; i < 5; i++ {
+			out.Write([]byte("Enter passphrase: "))
+			passphrase, err = pr.ReadPassword()
+
+			if err == ErrNoTerminal {
+				return fmt.Errorf("out-key must be encrypted interactively")
+			} else if err != nil {
+				return fmt.Errorf("error reading passphrase: %s", err)
+			}
+
+			if len(passphrase) > 0 {
+				break
+			}
+		}
+
 		if len(passphrase) == 0 {
-			for i := 0; i < 5; i++ {
-				out.Write([]byte("Enter passphrase: "))
-				passphrase, err = pr.ReadPassword()
-
-				if err == ErrNoTerminal {
-					return fmt.Errorf("out-key must be encrypted interactively")
-				} else if err != nil {
-					return fmt.Errorf("error reading passphrase: %s", err)
-				}
-
-				if len(passphrase) > 0 {
-					break
-				}
-			}
-
-			if len(passphrase) == 0 {
-				return fmt.Errorf("no passphrase specified, remove -encrypt flag to write out-key in plaintext")
-			}
+			return fmt.Errorf("no passphrase specified, remove -encrypt flag to write out-key in plaintext")
 		}
 	}
 

cmd/nebula-cert/ca_test.go

@@ -171,17 +171,6 @@ func Test_ca(t *testing.T) {
 	assert.Equal(t, pwPromptOb, ob.String())
 	assert.Empty(t, eb.String())
 
-	// test encrypted key with passphrase environment variable
-	os.Remove(keyF.Name())
-	os.Remove(crtF.Name())
-	ob.Reset()
-	eb.Reset()
-	args = []string{"-version", "1", "-encrypt", "-name", "test", "-duration", "100m", "-groups", "1,2,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()}
-	os.Setenv("NEBULA_CA_PASSPHRASE", string(passphrase))
-	require.NoError(t, ca(args, ob, eb, testpw))
-	assert.Empty(t, eb.String())
-	os.Setenv("NEBULA_CA_PASSPHRASE", "")
-
 	// read encrypted key file and verify default params
 	rb, _ = os.ReadFile(keyF.Name())
 	k, _ := pem.Decode(rb)
@@ -200,7 +189,7 @@ func Test_ca(t *testing.T) {
 	assert.Empty(t, b)
 	assert.Len(t, lKey, 64)
 
-	// test when reading password results in an error
+	// test when reading passsword results in an error
 	os.Remove(keyF.Name())
 	os.Remove(crtF.Name())
 	ob.Reset()

cmd/nebula-cert/main.go

@@ -5,28 +5,10 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"runtime/debug"
-	"strings"
 )
 
-// A version string that can be set with
-//
-//	-ldflags "-X main.Build=SOMEVERSION"
-//
-// at compile-time.
 var Build string
 
-func init() {
-	if Build == "" {
-		info, ok := debug.ReadBuildInfo()
-		if !ok {
-			return
-		}
-
-		Build = strings.TrimPrefix(info.Main.Version, "v")
-	}
-}
-
 type helpError struct {
 	s string
 }

cmd/nebula-cert/sign.go

@@ -43,7 +43,7 @@ type signFlags struct {
 func newSignFlags() *signFlags {
 	sf := signFlags{set: flag.NewFlagSet("sign", flag.ContinueOnError)}
 	sf.set.Usage = func() {}
-	sf.version = sf.set.Uint("version", 0, "Optional: version of the certificate format to use. The default is to match the version of the signing CA")
+	sf.version = sf.set.Uint("version", 0, "Optional: version of the certificate format to use, the default is to create both v1 and v2 certificates.")
 	sf.caKeyPath = sf.set.String("ca-key", "ca.key", "Optional: path to the signing CA key")
 	sf.caCertPath = sf.set.String("ca-crt", "ca.crt", "Optional: path to the signing CA cert")
 	sf.name = sf.set.String("name", "", "Required: name of the cert, usually a hostname")
@@ -116,28 +116,26 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 		// naively attempt to decode the private key as though it is not encrypted
 		caKey, _, curve, err = cert.UnmarshalSigningPrivateKeyFromPEM(rawCAKey)
 		if errors.Is(err, cert.ErrPrivateKeyEncrypted) {
+			// ask for a passphrase until we get one
 			var passphrase []byte
-			passphrase = []byte(os.Getenv("NEBULA_CA_PASSPHRASE"))
-			if len(passphrase) == 0 {
-				// ask for a passphrase until we get one
-				for i := 0; i < 5; i++ {
-					out.Write([]byte("Enter passphrase: "))
-					passphrase, err = pr.ReadPassword()
+			for i := 0; i < 5; i++ {
+				out.Write([]byte("Enter passphrase: "))
+				passphrase, err = pr.ReadPassword()
 
-					if errors.Is(err, ErrNoTerminal) {
-						return fmt.Errorf("ca-key is encrypted and must be decrypted interactively")
-					} else if err != nil {
-						return fmt.Errorf("error reading password: %s", err)
-					}
-
-					if len(passphrase) > 0 {
-						break
-					}
+				if errors.Is(err, ErrNoTerminal) {
+					return fmt.Errorf("ca-key is encrypted and must be decrypted interactively")
+				} else if err != nil {
+					return fmt.Errorf("error reading password: %s", err)
 				}
-				if len(passphrase) == 0 {
-					return fmt.Errorf("cannot open encrypted ca-key without passphrase")
+
+				if len(passphrase) > 0 {
+					break
 				}
 			}
+			if len(passphrase) == 0 {
+				return fmt.Errorf("cannot open encrypted ca-key without passphrase")
+			}
+
 			curve, caKey, _, err = cert.DecryptAndUnmarshalSigningPrivateKey(passphrase, rawCAKey)
 			if err != nil {
 				return fmt.Errorf("error while parsing encrypted ca-key: %s", err)
@@ -167,10 +165,6 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 		return fmt.Errorf("ca certificate is expired")
 	}
 
-	if version == 0 {
-		version = caCert.Version()
-	}
-
 	// if no duration is given, expire one second before the root expires
 	if *sf.duration <= 0 {
 		*sf.duration = time.Until(caCert.NotAfter()) - time.Second*1
@@ -283,19 +277,21 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 	notBefore := time.Now()
 	notAfter := notBefore.Add(*sf.duration)
 
-	switch version {
-	case cert.Version1:
-		// Make sure we have only one ipv4 address
+	if version == 0 || version == cert.Version1 {
+		// Make sure we at least have an ip
 		if len(v4Networks) != 1 {
 			return newHelpErrorf("invalid -networks definition: v1 certificates can only have a single ipv4 address")
 		}
 
-		if len(v6Networks) > 0 {
-			return newHelpErrorf("invalid -networks definition: v1 certificates can only contain ipv4 addresses")
-		}
+		if version == cert.Version1 {
+			// If we are asked to mint a v1 certificate only then we cant just ignore any v6 addresses
+			if len(v6Networks) > 0 {
+				return newHelpErrorf("invalid -networks definition: v1 certificates can only be ipv4")
+			}
 
-		if len(v6UnsafeNetworks) > 0 {
-			return newHelpErrorf("invalid -unsafe-networks definition: v1 certificates can only contain ipv4 addresses")
+			if len(v6UnsafeNetworks) > 0 {
+				return newHelpErrorf("invalid -unsafe-networks definition: v1 certificates can only be ipv4")
+			}
 		}
 
 		t := &cert.TBSCertificate{
@@ -325,8 +321,9 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 		}
 
 		crts = append(crts, nc)
+	}
 
-	case cert.Version2:
+	if version == 0 || version == cert.Version2 {
 		t := &cert.TBSCertificate{
 			Version:        cert.Version2,
 			Name:           *sf.name,
@@ -354,9 +351,6 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 		}
 
 		crts = append(crts, nc)
-	default:
-		// this should be unreachable
-		return fmt.Errorf("invalid version: %d", version)
 	}
 
 	if !isP11 && *sf.inPubPath == "" {

cmd/nebula-cert/sign_test.go

@@ -55,7 +55,7 @@ func Test_signHelp(t *testing.T) {
 			"  -unsafe-networks string\n"+
 			"    \tOptional: comma separated list of ip address and network in CIDR notation. Unsafe networks this cert can route for\n"+
 			"  -version uint\n"+
-			"    \tOptional: version of the certificate format to use. The default is to match the version of the signing CA\n",
+			"    \tOptional: version of the certificate format to use, the default is to create both v1 and v2 certificates.\n",
 		ob.String(),
 	)
 }
@@ -204,7 +204,7 @@ func Test_signCert(t *testing.T) {
 	ob.Reset()
 	eb.Reset()
 	args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m", "-subnets", "100::100/100"}
-	assertHelpError(t, signCert(args, ob, eb, nopw), "invalid -unsafe-networks definition: v1 certificates can only contain ipv4 addresses")
+	assertHelpError(t, signCert(args, ob, eb, nopw), "invalid -unsafe-networks definition: v1 certificates can only be ipv4")
 	assert.Empty(t, ob.String())
 	assert.Empty(t, eb.String())
 
@@ -379,15 +379,6 @@ func Test_signCert(t *testing.T) {
 	assert.Equal(t, "Enter passphrase: ", ob.String())
 	assert.Empty(t, eb.String())
 
-	// test with the proper password in the environment
-	os.Remove(crtF.Name())
-	os.Remove(keyF.Name())
-	args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, ,   10.2.2.2/32   ,   ,  ,, 10.5.5.5/32", "-groups", "1,,   2    ,        ,,,3,4,5"}
-	os.Setenv("NEBULA_CA_PASSPHRASE", string(passphrase))
-	require.NoError(t, signCert(args, ob, eb, testpw))
-	assert.Empty(t, eb.String())
-	os.Setenv("NEBULA_CA_PASSPHRASE", "")
-
 	// test with the wrong password
 	ob.Reset()
 	eb.Reset()
@@ -398,17 +389,6 @@ func Test_signCert(t *testing.T) {
 	assert.Equal(t, "Enter passphrase: ", ob.String())
 	assert.Empty(t, eb.String())
 
-	// test with the wrong password in environment
-	ob.Reset()
-	eb.Reset()
-
-	os.Setenv("NEBULA_CA_PASSPHRASE", "invalid password")
-	args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, ,   10.2.2.2/32   ,   ,  ,, 10.5.5.5/32", "-groups", "1,,   2    ,        ,,,3,4,5"}
-	require.EqualError(t, signCert(args, ob, eb, nopw), "error while parsing encrypted ca-key: invalid passphrase or corrupt private key")
-	assert.Empty(t, ob.String())
-	assert.Empty(t, eb.String())
-	os.Setenv("NEBULA_CA_PASSPHRASE", "")
-
 	// test with the user not entering a password
 	ob.Reset()
 	eb.Reset()

cmd/nebula-cert/verify.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/slackhq/nebula/cert"
@@ -39,15 +40,21 @@ func verify(args []string, out io.Writer, errOut io.Writer) error {
 		return err
 	}
 
-	caFile, err := os.Open(*vf.caPath)
+	rawCACert, err := os.ReadFile(*vf.caPath)
 	if err != nil {
 		return fmt.Errorf("error while reading ca: %w", err)
 	}
-	defer caFile.Close()
 
-	caPool, err := cert.NewCAPoolFromPEMReader(caFile)
-	if err != nil && !errors.Is(err, cert.ErrExpired) {
-		return fmt.Errorf("error while adding ca cert to pool: %w", err)
+	caPool := cert.NewCAPool()
+	for {
+		rawCACert, err = caPool.AddCAFromPEM(rawCACert)
+		if err != nil {
+			return fmt.Errorf("error while adding ca cert to pool: %w", err)
+		}
+
+		if rawCACert == nil || len(rawCACert) == 0 || strings.TrimSpace(string(rawCACert)) == "" {
+			break
+		}
 	}
 
 	rawCert, err := os.ReadFile(*vf.certPath)

cmd/nebula-cert/verify_test.go

@@ -64,7 +64,7 @@ func Test_verify(t *testing.T) {
 	err = verify([]string{"-ca", caFile.Name(), "-crt", "does_not_exist"}, ob, eb)
 	assert.Empty(t, ob.String())
 	assert.Empty(t, eb.String())
-	require.ErrorIs(t, err, cert.ErrInvalidPEMBlock)
+	require.EqualError(t, err, "error while adding ca cert to pool: input did not contain a valid PEM encoded block")
 
 	// make a ca for later
 	caPub, caPriv, _ := ed25519.GenerateKey(rand.Reader)

cmd/nebula-service/main.go

@@ -4,8 +4,6 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"runtime/debug"
-	"strings"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
@@ -20,17 +18,6 @@ import (
 // at compile-time.
 var Build string
 
-func init() {
-	if Build == "" {
-		info, ok := debug.ReadBuildInfo()
-		if !ok {
-			return
-		}
-
-		Build = strings.TrimPrefix(info.Main.Version, "v")
-	}
-}
-
 func main() {
 	serviceFlag := flag.String("service", "", "Control the system service.")
 	configPath := flag.String("config", "", "Path to either a file or directory to load configuration from")
@@ -50,15 +37,9 @@ func main() {
 		os.Exit(0)
 	}
 
-	l := logrus.New()
-	l.Out = os.Stdout
-
 	if *serviceFlag != "" {
-		if err := doService(configPath, configTest, Build, serviceFlag); err != nil {
-			l.WithError(err).Error("Service command failed")
-			os.Exit(1)
-		}
-		return
+		doService(configPath, configTest, Build, serviceFlag)
+		os.Exit(1)
 	}
 
 	if *configPath == "" {
@@ -67,6 +48,9 @@ func main() {
 		os.Exit(1)
 	}
 
+	l := logrus.New()
+	l.Out = os.Stdout
+
 	c := config.NewC(l)
 	err := c.Load(*configPath)
 	if err != nil {
@@ -81,20 +65,8 @@ func main() {
 	}
 
 	if !*configTest {
-		wait, err := ctrl.Start()
-		if err != nil {
-			util.LogWithContextIfNeeded("Error while running", err, l)
-			os.Exit(1)
-		}
-
-		go ctrl.ShutdownBlock()
-
-		if err := wait(); err != nil {
-			l.WithError(err).Error("Nebula stopped due to fatal error")
-			os.Exit(2)
-		}
-
-		l.Info("Goodbye")
+		ctrl.Start()
+		ctrl.ShutdownBlock()
 	}
 
 	os.Exit(0)

cmd/nebula-service/service.go

@@ -57,11 +57,11 @@ func fileExists(filename string) bool {
 	return true
 }
 
-func doService(configPath *string, configTest *bool, build string, serviceFlag *string) error {
+func doService(configPath *string, configTest *bool, build string, serviceFlag *string) {
 	if *configPath == "" {
 		ex, err := os.Executable()
 		if err != nil {
-			return err
+			panic(err)
 		}
 		*configPath = filepath.Dir(ex) + "/config.yaml"
 		if !fileExists(*configPath) {
@@ -88,13 +88,13 @@ func doService(configPath *string, configTest *bool, build string, serviceFlag *
 	// - above, in `Run` we create a `logrus.Logger` which is what nebula expects to use
 	s, err := service.New(prg, svcConfig)
 	if err != nil {
-		return err
+		log.Fatal(err)
 	}
 
 	errs := make(chan error, 5)
 	logger, err = s.Logger(errs)
 	if err != nil {
-		return err
+		log.Fatal(err)
 	}
 
 	go func() {
@@ -109,16 +109,18 @@ func doService(configPath *string, configTest *bool, build string, serviceFlag *
 
 	switch *serviceFlag {
 	case "run":
-		if err := s.Run(); err != nil {
+		err = s.Run()
+		if err != nil {
 			// Route any errors to the system logger
 			logger.Error(err)
 		}
 	default:
-		if err := service.Control(s, *serviceFlag); err != nil {
+		err := service.Control(s, *serviceFlag)
+		if err != nil {
 			log.Printf("Valid actions: %q\n", service.ControlAction)
-			return err
+			log.Fatal(err)
 		}
+		return
 	}
 
-	return nil
 }

cmd/nebula/main.go

@@ -4,8 +4,6 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"runtime/debug"
-	"strings"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
@@ -20,17 +18,6 @@ import (
 // at compile-time.
 var Build string
 
-func init() {
-	if Build == "" {
-		info, ok := debug.ReadBuildInfo()
-		if !ok {
-			return
-		}
-
-		Build = strings.TrimPrefix(info.Main.Version, "v")
-	}
-}
-
 func main() {
 	configPath := flag.String("config", "", "Path to either a file or directory to load configuration from")
 	configTest := flag.Bool("test", false, "Test the config and print the end result. Non zero exit indicates a faulty config")
@@ -72,21 +59,9 @@ func main() {
 	}
 
 	if !*configTest {
-		wait, err := ctrl.Start()
-		if err != nil {
-			util.LogWithContextIfNeeded("Error while running", err, l)
-			os.Exit(1)
-		}
-
-		go ctrl.ShutdownBlock()
+		ctrl.Start()
 		notifyReady(l)
-
-		if err := wait(); err != nil {
-			l.WithError(err).Error("Nebula stopped due to fatal error")
-			os.Exit(2)
-		}
-
-		l.Info("Goodbye")
+		ctrl.ShutdownBlock()
 	}
 
 	os.Exit(0)

config/config.go

@@ -17,7 +17,7 @@ import (
 
 	"dario.cat/mergo"
 	"github.com/sirupsen/logrus"
-	"go.yaml.in/yaml/v3"
+	"gopkg.in/yaml.v3"
 )
 
 type C struct {

config/config_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.yaml.in/yaml/v3"
+	"gopkg.in/yaml.v3"
 )
 
 func TestConfig_Load(t *testing.T) {

connection_state.go

@@ -50,6 +50,11 @@ func NewConnectionState(l *logrus.Logger, cs *CertState, crt cert.Certificate, i
 	}
 
 	static := noise.DHKey{Private: cs.privateKey, Public: crt.PublicKey()}
+
+	b := NewBits(ReplayWindow)
+	// Clear out bit 0, we never transmit it, and we don't want it showing as packet loss
+	b.Update(l, 0)
+
 	hs, err := noise.NewHandshakeState(noise.Config{
 		CipherSuite:   ncs,
 		Random:        rand.Reader,
@@ -69,7 +74,7 @@ func NewConnectionState(l *logrus.Logger, cs *CertState, crt cert.Certificate, i
 	ci := &ConnectionState{
 		H:         hs,
 		initiator: initiator,
-		window:    NewBits(ReplayWindow),
+		window:    b,
 		myCert:    crt,
 	}
 	// always start the counter from 2, as packet 1 and packet 2 are handshake packets.

control.go

@@ -2,34 +2,17 @@ package nebula
 
 import (
 	"context"
-	"errors"
 	"net/netip"
 	"os"
 	"os/signal"
-	"sync"
 	"syscall"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/firewall/events"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/overlay"
 )
 
-type RunState int
-
-const (
-	StateUnknown RunState = iota
-	StateReady
-	StateStarted
-	StateStopping
-	StateStopped
-)
-
-var ErrAlreadyStarted = errors.New("nebula is already started")
-var ErrAlreadyStopped = errors.New("nebula cannot be restarted")
-var ErrUnknownState = errors.New("nebula state is invalid")
-
 // Every interaction here needs to take extra care to copy memory and not return or use arguments "as is" when touching
 // core. This means copying IP objects, slices, de-referencing pointers and taking the actual value, etc
 
@@ -43,9 +26,6 @@ type controlHostLister interface {
 }
 
 type Control struct {
-	stateLock sync.Mutex
-	state     RunState
-
 	f                      *Interface
 	l                      *logrus.Logger
 	ctx                    context.Context
@@ -69,31 +49,10 @@ type ControlHostInfo struct {
 	CurrentRelaysThroughMe []netip.Addr     `json:"currentRelaysThroughMe"`
 }
 
-// Start actually runs nebula, this is a nonblocking call.
-// The returned function blocks until nebula has fully stopped and returns the
-// first fatal reader error (if any). A nil error means nebula shut down
-// gracefully; a non-nil error means a reader hit an unexpected failure that
-// triggered the shutdown.
-func (c *Control) Start() (func() error, error) {
-	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
-	switch c.state {
-	case StateReady:
-		//yay!
-	case StateStopped, StateStopping:
-		return nil, ErrAlreadyStopped
-	case StateStarted:
-		return nil, ErrAlreadyStarted
-	default:
-		return nil, ErrUnknownState
-	}
-
+// Start actually runs nebula, this is a nonblocking call. To block use Control.ShutdownBlock()
+func (c *Control) Start() {
 	// Activate the interface
-	err := c.f.activate()
-	if err != nil {
-		c.state = StateStopped
-		return nil, err
-	}
+	c.f.activate()
 
 	// Call all the delayed funcs that waited patiently for the interface to be created.
 	if c.sshStart != nil {
@@ -112,40 +71,16 @@ func (c *Control) Start() (func() error, error) {
 		c.lighthouseStart()
 	}
 
-	c.f.triggerShutdown = c.Stop
-
 	// Start reading packets.
-	out, err := c.f.run()
-	if err != nil {
-		c.state = StateStopped
-		return nil, err
-	}
-	c.state = StateStarted
-	return out, nil
-}
-
-func (c *Control) State() RunState {
-	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
-	return c.state
+	c.f.run()
 }
 
 func (c *Control) Context() context.Context {
 	return c.ctx
 }
 
-// Stop is a non-blocking call that signals nebula to close all tunnels and shut down
+// Stop signals nebula to shutdown and close all tunnels, returns after the shutdown is complete
 func (c *Control) Stop() {
-	c.stateLock.Lock()
-	if c.state != StateStarted {
-		c.stateLock.Unlock()
-		// We are stopping or stopped already
-		return
-	}
-
-	c.state = StateStopping
-	c.stateLock.Unlock()
-
 	// Stop the handshakeManager (and other services), to prevent new tunnels from
 	// being created while we're shutting them all down.
 	c.cancel()
@@ -154,9 +89,7 @@ func (c *Control) Stop() {
 	if err := c.f.Close(); err != nil {
 		c.l.WithError(err).Error("Close interface failed")
 	}
-	c.stateLock.Lock()
-	c.state = StateStopped
-	c.stateLock.Unlock()
+	c.l.Info("Goodbye")
 }
 
 // ShutdownBlock will listen for and block on term and interrupt signals, calling Control.Stop() once signalled
@@ -341,36 +274,6 @@ func (c *Control) Device() overlay.Device {
 	return c.f.inside
 }
 
-// SetFirewallEventReporter installs an event reporter on the current firewall.
-// Passing nil clears any installed reporter. The reporter is carried across
-// firewall rule reloads. Report* methods are invoked while nebula holds
-// internal locks and must be non-blocking; in particular they must not call
-// back into *Control methods that touch the firewall, or deadlock will
-// result.
-//
-// Installation is performed by shallow-copying the current *Firewall,
-// setting the reporter field on the copy, and swapping the pointer under
-// the conntrack lock. Every Firewall the data path sees therefore has an
-// immutable reporter slot, and emit sites can read it without any
-// synchronization of their own.
-func (c *Control) SetFirewallEventReporter(r events.Reporter) {
-	old := c.f.firewall
-	if old == nil {
-		return
-	}
-	old.Conntrack.Lock()
-	defer old.Conntrack.Unlock()
-
-	// Re-read under the lock in case a concurrent reload swapped in a new
-	// Firewall between the unlocked load above and here. Both Firewalls share
-	// the same Conntrack pointer in the normal (non-overflow) reload path,
-	// so the lock we hold is the right one for whichever we see now.
-	current := c.f.firewall
-	fw := *current
-	fw.reporter = r
-	c.f.firewall = &fw
-}
-
 func copyHostInfo(h *HostInfo, preferredRanges []netip.Prefix) ControlHostInfo {
 	chi := ControlHostInfo{
 		VpnAddrs:               make([]netip.Addr, len(h.vpnAddrs)),

control_test.go

@@ -79,7 +79,6 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) {
 	}, &Interface{})
 
 	c := Control{
-		state: StateReady,
 		f: &Interface{
 			hostMap: hm,
 		},

control_tester.go

@@ -1,4 +1,5 @@
 //go:build e2e_testing
+// +build e2e_testing
 
 package nebula
 
@@ -173,10 +174,6 @@ func (c *Control) GetHostmap() *HostMap {
 	return c.f.hostMap
 }
 
-func (c *Control) GetF() *Interface {
-	return c.f
-}
-
 func (c *Control) GetCertState() *CertState {
 	return c.f.pki.getCertState()
 }

dist/wireshark/nebula.lua

@@ -84,24 +84,30 @@ end
 
 function nebula.prefs_changed()
     if default_settings.all_ports == nebula.prefs.all_ports and default_settings.port == nebula.prefs.port then
+        -- Nothing changed, bail
         return
     end
 
-    -- Remove all existing registrations
+    -- Remove our old dissector
     DissectorTable.get("udp.port"):remove_all(nebula)
 
-    if nebula.prefs.all_ports then
-        -- Register on every port for hole punch capture
+    if nebula.prefs.all_ports and default_settings.all_ports ~= nebula.prefs.all_ports then
+        default_settings.all_port = nebula.prefs.all_ports
+
         for i=0, 65535 do
             DissectorTable.get("udp.port"):add(i, nebula)
         end
-    else
-        -- Register on the configured port only
-        DissectorTable.get("udp.port"):add(nebula.prefs.port, nebula)
+
+        -- no need to establish again on specific ports
+        return
     end
 
-    default_settings.all_ports = nebula.prefs.all_ports
-    default_settings.port = nebula.prefs.port
+
+    if default_settings.all_ports ~= nebula.prefs.all_ports then
+        -- Add our new port dissector
+        default_settings.port = nebula.prefs.port
+        DissectorTable.get("udp.port"):add(default_settings.port, nebula)
+    end
 end
 
 DissectorTable.get("udp.port"):add(default_settings.port, nebula)

dns_server.go

@@ -1,14 +1,12 @@
 package nebula
 
 import (
-	"context"
 	"fmt"
 	"net"
 	"net/netip"
 	"strconv"
 	"strings"
 	"sync"
-	"sync/atomic"
 
 	"github.com/gaissmai/bart"
 	"github.com/miekg/dns"
@@ -16,207 +14,32 @@ import (
 	"github.com/slackhq/nebula/config"
 )
 
-type dnsServer struct {
+// This whole thing should be rewritten to use context
+
+var dnsR *dnsRecords
+var dnsServer *dns.Server
+var dnsAddr string
+
+type dnsRecords struct {
 	sync.RWMutex
 	l               *logrus.Logger
-	ctx             context.Context
 	dnsMap4         map[string]netip.Addr
 	dnsMap6         map[string]netip.Addr
 	hostMap         *HostMap
 	myVpnAddrsTable *bart.Lite
-
-	mux *dns.ServeMux
-
-	// enabled mirrors `lighthouse.serve_dns && lighthouse.am_lighthouse`.
-	// Start, Add, and reload consult it so callers don't need to know the
-	// gating rules. When it toggles off via reload, accumulated records are
-	// cleared so a later re-enable starts with a fresh map populated from
-	// new handshakes.
-	enabled atomic.Bool
-
-	serverMu sync.Mutex
-	server   *dns.Server
-	// started is closed once `server` has finished binding (or after
-	// ListenAndServe returns on a bind failure). Stop waits on it before
-	// calling Shutdown to avoid the miekg/dns "server not started" race
-	// where a Shutdown that arrives before bind completes is silently
-	// ignored, leaving the listener running forever.
-	started chan struct{}
-	addr    string
 }
 
-// newDnsServerFromConfig builds a dnsServer, applies the initial config, and
-// registers a reload callback. The reload callback is registered before the
-// initial config is applied, so a SIGHUP can later enable, fix, or disable
-// DNS even if the initial application failed.
-//
-// The dnsServer internally gates on `lighthouse.serve_dns &&
-// lighthouse.am_lighthouse`. Start and Add are safe to call unconditionally,
-// they no-op when DNS isn't enabled. Each Start invocation owns a ctx-cancel
-// watcher that tears the listener down on nebula shutdown. The returned
-// pointer is always non-nil, even on error.
-func newDnsServerFromConfig(ctx context.Context, l *logrus.Logger, cs *CertState, hostMap *HostMap, c *config.C) (*dnsServer, error) {
-	ds := &dnsServer{
+func newDnsRecords(l *logrus.Logger, cs *CertState, hostMap *HostMap) *dnsRecords {
+	return &dnsRecords{
 		l:               l,
-		ctx:             ctx,
 		dnsMap4:         make(map[string]netip.Addr),
 		dnsMap6:         make(map[string]netip.Addr),
 		hostMap:         hostMap,
 		myVpnAddrsTable: cs.myVpnAddrsTable,
 	}
-	ds.mux = dns.NewServeMux()
-	ds.mux.HandleFunc(".", ds.handleDnsRequest)
-
-	c.RegisterReloadCallback(func(c *config.C) {
-		if err := ds.reload(c, false); err != nil {
-			l.WithError(err).Error("Failed to reload DNS responder from config")
-		}
-	})
-
-	if err := ds.reload(c, true); err != nil {
-		return ds, err
-	}
-	return ds, nil
 }
 
-// reload applies the latest config and reconciles the running state with it:
-//   - enabled toggled on  -> spawn a runner
-//   - enabled toggled off -> stop the runner
-//   - listen address changed (while running) -> restart on the new address
-//   - everything else     -> no-op
-//
-// On the initial call it only records configuration; Control.Start is what
-// launches the first runner via dnsStart.
-func (d *dnsServer) reload(c *config.C, initial bool) error {
-	wantsDns := c.GetBool("lighthouse.serve_dns", false)
-	amLighthouse := c.GetBool("lighthouse.am_lighthouse", false)
-	enabled := wantsDns && amLighthouse
-	newAddr := getDnsServerAddr(c)
-
-	d.serverMu.Lock()
-	running := d.server
-	runningStarted := d.started
-	sameAddr := d.addr == newAddr
-	d.addr = newAddr
-	d.enabled.Store(enabled)
-	d.serverMu.Unlock()
-
-	if initial {
-		if wantsDns && !amLighthouse {
-			d.l.Warn("DNS server refusing to run because this host is not a lighthouse.")
-		}
-		return nil
-	}
-
-	if !enabled {
-		if running != nil {
-			d.Stop()
-		}
-		// Drop any records that accumulated while enabled; a later re-enable
-		// will repopulate from fresh handshakes.
-		d.clearRecords()
-		return nil
-	}
-
-	if running == nil {
-		// Was disabled (or never started); bring it up now.
-		go d.Start()
-		return nil
-	}
-
-	if sameAddr {
-		return nil
-	}
-
-	d.shutdownServer(running, runningStarted, "reload")
-	// Old Start goroutine has now exited; bring up a fresh listener on the
-	// new address.
-	go d.Start()
-	return nil
-}
-
-// shutdownServer waits for the server to finish binding (so Shutdown actually
-// stops it rather than no-oping) and then shuts it down.
-func (d *dnsServer) shutdownServer(srv *dns.Server, started chan struct{}, reason string) {
-	if srv == nil {
-		return
-	}
-	if started != nil {
-		<-started
-	}
-	if err := srv.Shutdown(); err != nil {
-		d.l.WithError(err).WithField("reason", reason).Warn("Failed to shut down the DNS responder")
-	}
-}
-
-// Start binds and serves the DNS responder. Blocks until Stop is called or
-// the listener errors. Safe to call when DNS is disabled (returns
-// immediately). This is what Control.dnsStart points at.
-//
-// Must be invoked after the tun device is active so that lighthouse.dns.host
-// may bind to a nebula IP.
-func (d *dnsServer) Start() {
-	if !d.enabled.Load() {
-		return
-	}
-
-	started := make(chan struct{})
-	d.serverMu.Lock()
-	if d.ctx.Err() != nil {
-		d.serverMu.Unlock()
-		return
-	}
-	addr := d.addr
-	server := &dns.Server{
-		Addr:              addr,
-		Net:               "udp",
-		Handler:           d.mux,
-		NotifyStartedFunc: func() { close(started) },
-	}
-	d.server = server
-	d.started = started
-	d.serverMu.Unlock()
-
-	// Per-invocation ctx watcher. Exits when Start does, so we don't leak a
-	// watcher per reload-driven restart.
-	done := make(chan struct{})
-	go func() {
-		select {
-		case <-d.ctx.Done():
-			d.shutdownServer(server, started, "shutdown")
-		case <-done:
-		}
-	}()
-
-	d.l.WithField("dnsListener", addr).Info("Starting DNS responder")
-	err := server.ListenAndServe()
-	close(done)
-
-	// If the listener never bound (bind error) NotifyStartedFunc never fires,
-	// so close started here to release any Stop caller waiting on it.
-	select {
-	case <-started:
-	default:
-		close(started)
-	}
-
-	if err != nil {
-		d.l.WithError(err).Warn("Failed to run the DNS responder")
-	}
-}
-
-// Stop shuts down the active server, if any. Idempotent.
-func (d *dnsServer) Stop() {
-	d.serverMu.Lock()
-	srv := d.server
-	started := d.started
-	d.server = nil
-	d.started = nil
-	d.serverMu.Unlock()
-	d.shutdownServer(srv, started, "stop")
-}
-
-func (d *dnsServer) Query(q uint16, data string) netip.Addr {
+func (d *dnsRecords) Query(q uint16, data string) netip.Addr {
 	data = strings.ToLower(data)
 	d.RLock()
 	defer d.RUnlock()
@@ -234,7 +57,7 @@ func (d *dnsServer) Query(q uint16, data string) netip.Addr {
 	return netip.Addr{}
 }
 
-func (d *dnsServer) QueryCert(data string) string {
+func (d *dnsRecords) QueryCert(data string) string {
 	ip, err := netip.ParseAddr(data[:len(data)-1])
 	if err != nil {
 		return ""
@@ -257,19 +80,8 @@ func (d *dnsServer) QueryCert(data string) string {
 	return string(b)
 }
 
-// clearRecords drops all DNS records.
-func (d *dnsServer) clearRecords() {
-	d.Lock()
-	defer d.Unlock()
-	clear(d.dnsMap4)
-	clear(d.dnsMap6)
-}
-
 // Add adds the first IPv4 and IPv6 address that appears in `addresses` as the record for `host`
-func (d *dnsServer) Add(host string, addresses []netip.Addr) {
-	if !d.enabled.Load() {
-		return
-	}
+func (d *dnsRecords) Add(host string, addresses []netip.Addr) {
 	host = strings.ToLower(host)
 	d.Lock()
 	defer d.Unlock()
@@ -289,7 +101,7 @@ func (d *dnsServer) Add(host string, addresses []netip.Addr) {
 	}
 }
 
-func (d *dnsServer) isSelfNebulaOrLocalhost(addr string) bool {
+func (d *dnsRecords) isSelfNebulaOrLocalhost(addr string) bool {
 	a, _, _ := net.SplitHostPort(addr)
 	b, err := netip.ParseAddr(a)
 	if err != nil {
@@ -304,7 +116,7 @@ func (d *dnsServer) isSelfNebulaOrLocalhost(addr string) bool {
 	return d.myVpnAddrsTable.Contains(b)
 }
 
-func (d *dnsServer) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
+func (d *dnsRecords) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
 	for _, q := range m.Question {
 		switch q.Qtype {
 		case dns.TypeA, dns.TypeAAAA:
@@ -338,7 +150,7 @@ func (d *dnsServer) parseQuery(m *dns.Msg, w dns.ResponseWriter) {
 	}
 }
 
-func (d *dnsServer) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
+func (d *dnsRecords) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
 	m := new(dns.Msg)
 	m.SetReply(r)
 	m.Compress = false
@@ -351,6 +163,21 @@ func (d *dnsServer) handleDnsRequest(w dns.ResponseWriter, r *dns.Msg) {
 	w.WriteMsg(m)
 }
 
+func dnsMain(l *logrus.Logger, cs *CertState, hostMap *HostMap, c *config.C) func() {
+	dnsR = newDnsRecords(l, cs, hostMap)
+
+	// attach request handler func
+	dns.HandleFunc(".", dnsR.handleDnsRequest)
+
+	c.RegisterReloadCallback(func(c *config.C) {
+		reloadDns(l, c)
+	})
+
+	return func() {
+		startDns(l, c)
+	}
+}
+
 func getDnsServerAddr(c *config.C) string {
 	dnsHost := strings.TrimSpace(c.GetString("lighthouse.dns.host", ""))
 	// Old guidance was to provide the literal `[::]` in `lighthouse.dns.host` but that won't resolve.
@@ -359,3 +186,25 @@ func getDnsServerAddr(c *config.C) string {
 	}
 	return net.JoinHostPort(dnsHost, strconv.Itoa(c.GetInt("lighthouse.dns.port", 53)))
 }
+
+func startDns(l *logrus.Logger, c *config.C) {
+	dnsAddr = getDnsServerAddr(c)
+	dnsServer = &dns.Server{Addr: dnsAddr, Net: "udp"}
+	l.WithField("dnsListener", dnsAddr).Info("Starting DNS responder")
+	err := dnsServer.ListenAndServe()
+	defer dnsServer.Shutdown()
+	if err != nil {
+		l.Errorf("Failed to start server: %s\n ", err.Error())
+	}
+}
+
+func reloadDns(l *logrus.Logger, c *config.C) {
+	if dnsAddr == getDnsServerAddr(c) {
+		l.Debug("No DNS server config change detected")
+		return
+	}
+
+	l.Debug("Restarting DNS server")
+	dnsServer.Shutdown()
+	go startDns(l, c)
+}

dns_server_test.go

@@ -1,31 +1,19 @@
 package nebula
 
 import (
-	"context"
-	"io"
-	"net"
 	"net/netip"
-	"strconv"
 	"testing"
-	"time"
 
 	"github.com/miekg/dns"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestParsequery(t *testing.T) {
 	l := logrus.New()
 	hostMap := &HostMap{}
-	ds := &dnsServer{
-		l:       l,
-		dnsMap4: make(map[string]netip.Addr),
-		dnsMap6: make(map[string]netip.Addr),
-		hostMap: hostMap,
-	}
-	ds.enabled.Store(true)
+	ds := newDnsRecords(l, &CertState{}, hostMap)
 	addrs := []netip.Addr{
 		netip.MustParseAddr("1.2.3.4"),
 		netip.MustParseAddr("1.2.3.5"),
@@ -83,209 +71,3 @@ func Test_getDnsServerAddr(t *testing.T) {
 	}
 	assert.Equal(t, "[::]:1", getDnsServerAddr(c))
 }
-
-func newTestDnsServer(t *testing.T) (*dnsServer, *config.C) {
-	t.Helper()
-	l := logrus.New()
-	l.Out = io.Discard
-	ds := &dnsServer{
-		l:       l,
-		ctx:     context.Background(),
-		dnsMap4: make(map[string]netip.Addr),
-		dnsMap6: make(map[string]netip.Addr),
-		hostMap: &HostMap{},
-	}
-	ds.mux = dns.NewServeMux()
-	ds.mux.HandleFunc(".", ds.handleDnsRequest)
-	return ds, config.NewC(l)
-}
-
-func setDnsConfig(c *config.C, host string, port string, amLighthouse, serveDns bool) {
-	c.Settings["lighthouse"] = map[string]any{
-		"am_lighthouse": amLighthouse,
-		"serve_dns":     serveDns,
-		"dns": map[string]any{
-			"host": host,
-			"port": port,
-		},
-	}
-}
-
-func TestDnsServer_reload_initial_disabled(t *testing.T) {
-	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", "0", true, false)
-
-	require.NoError(t, ds.reload(c, true))
-	assert.False(t, ds.enabled.Load())
-	assert.Equal(t, "127.0.0.1:0", ds.addr)
-	assert.Nil(t, ds.server)
-}
-
-func TestDnsServer_reload_initial_enabled(t *testing.T) {
-	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", "0", true, true)
-
-	require.NoError(t, ds.reload(c, true))
-	assert.True(t, ds.enabled.Load())
-	assert.Equal(t, "127.0.0.1:0", ds.addr)
-	// initial never starts a runner; that's Control.Start's job
-	assert.Nil(t, ds.server)
-}
-
-func TestDnsServer_reload_initial_serveDnsWithoutLighthouse(t *testing.T) {
-	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", "0", false, true)
-
-	require.NoError(t, ds.reload(c, true))
-	// Wants DNS but isn't a lighthouse: gated off, no runner.
-	assert.False(t, ds.enabled.Load())
-}
-
-func TestDnsServer_reload_sameAddr_noOp(t *testing.T) {
-	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", "0", true, true)
-
-	require.NoError(t, ds.reload(c, true))
-	// No server running yet, no addr change. Reload should not spawn anything.
-	require.NoError(t, ds.reload(c, false))
-	assert.True(t, ds.enabled.Load())
-	assert.Nil(t, ds.server)
-}
-
-func TestDnsServer_StartStop_lifecycle(t *testing.T) {
-	// Bind to a real (random) UDP port so we exercise the actual
-	// ListenAndServe + Shutdown plumbing including the started-chan race fix.
-	port := freeUDPPort(t)
-
-	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", port, true, true)
-	require.NoError(t, ds.reload(c, true))
-
-	done := make(chan struct{})
-	go func() {
-		ds.Start()
-		close(done)
-	}()
-
-	waitFor(t, func() bool {
-		ds.serverMu.Lock()
-		started := ds.started
-		ds.serverMu.Unlock()
-		if started == nil {
-			return false
-		}
-		select {
-		case <-started:
-			return true
-		default:
-			return false
-		}
-	})
-
-	ds.Stop()
-	select {
-	case <-done:
-	case <-time.After(5 * time.Second):
-		t.Fatal("Start did not return after Stop")
-	}
-}
-
-func TestDnsServer_Stop_beforeBind_doesNotHang(t *testing.T) {
-	// Stop called immediately after Start should not deadlock even if bind
-	// hasn't completed yet. This exercises the started-chan close-on-bind-fail
-	// path: by binding to an obviously bad port (privileged) we get a fast
-	// bind error before NotifyStartedFunc fires.
-	ds, c := newTestDnsServer(t)
-	// Use a port that should fail to bind (negative would be invalid, use a
-	// host that won't resolve to ensure listenUDP fails quickly).
-	setDnsConfig(c, "256.256.256.256", "53", true, true)
-	require.NoError(t, ds.reload(c, true))
-
-	done := make(chan struct{})
-	go func() {
-		ds.Start()
-		close(done)
-	}()
-
-	// Give Start a moment to attempt the bind and fail.
-	select {
-	case <-done:
-		// Bind failed and Start returned; Stop should be a no-op.
-	case <-time.After(time.Second):
-		t.Fatal("Start did not return after a bad bind")
-	}
-
-	stopped := make(chan struct{})
-	go func() {
-		ds.Stop()
-		close(stopped)
-	}()
-	select {
-	case <-stopped:
-	case <-time.After(time.Second):
-		t.Fatal("Stop hung after a failed bind")
-	}
-}
-
-func TestDnsServer_reload_disable_stopsRunningServer(t *testing.T) {
-	port := freeUDPPort(t)
-	ds, c := newTestDnsServer(t)
-	setDnsConfig(c, "127.0.0.1", port, true, true)
-	require.NoError(t, ds.reload(c, true))
-
-	startReturned := make(chan struct{})
-	go func() {
-		ds.Start()
-		close(startReturned)
-	}()
-	waitForBind(t, ds)
-
-	// Toggle serve_dns off; reload should shut the running server down.
-	setDnsConfig(c, "127.0.0.1", port, true, false)
-	require.NoError(t, ds.reload(c, false))
-	select {
-	case <-startReturned:
-	case <-time.After(5 * time.Second):
-		t.Fatal("Start did not return after reload disabled DNS")
-	}
-	assert.False(t, ds.enabled.Load())
-}
-
-func freeUDPPort(t *testing.T) string {
-	t.Helper()
-	conn, err := net.ListenPacket("udp", "127.0.0.1:0")
-	require.NoError(t, err)
-	port := conn.LocalAddr().(*net.UDPAddr).Port
-	require.NoError(t, conn.Close())
-	return strconv.Itoa(port)
-}
-
-func waitForBind(t *testing.T, ds *dnsServer) {
-	t.Helper()
-	waitFor(t, func() bool {
-		ds.serverMu.Lock()
-		started := ds.started
-		ds.serverMu.Unlock()
-		if started == nil {
-			return false
-		}
-		select {
-		case <-started:
-			return true
-		default:
-			return false
-		}
-	})
-}
-
-func waitFor(t *testing.T, cond func() bool) {
-	t.Helper()
-	deadline := time.Now().Add(5 * time.Second)
-	for time.Now().Before(deadline) {
-		if cond() {
-			return
-		}
-		time.Sleep(5 * time.Millisecond)
-	}
-	t.Fatal("timed out waiting for condition")
-}

e2e/handshake_manager_test.go

@@ -1,565 +0,0 @@
-//go:build e2e_testing
-// +build e2e_testing
-
-package e2e
-
-import (
-	"net/netip"
-	"testing"
-	"time"
-
-	"github.com/slackhq/nebula"
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/e2e/router"
-	"github.com/slackhq/nebula/header"
-	"github.com/slackhq/nebula/udp"
-	"github.com/stretchr/testify/assert"
-)
-
-// makeHandshakePacket creates a handshake packet with the given parameters.
-func makeHandshakePacket(from, to netip.AddrPort, subtype header.MessageSubType, remoteIndex uint32, counter uint64) *udp.Packet {
-	data := make([]byte, 200)
-	header.Encode(data, header.Version, header.Handshake, subtype, remoteIndex, counter)
-	for i := header.Len; i < len(data); i++ {
-		data[i] = byte(i)
-	}
-	return &udp.Packet{To: to, From: from, Data: data}
-}
-
-func TestHandshakeRetransmitDuplicate(t *testing.T) {
-	// Verify the responder correctly handles receiving the same msg1 multiple times
-	// (retransmission). The duplicate goes through CheckAndComplete -> ErrAlreadySeen
-	// and the cached response is resent.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	t.Log("Trigger handshake from me to them")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
-
-	t.Log("Grab my msg1")
-	msg1 := myControl.GetFromUDP(true)
-
-	t.Log("Inject msg1 into them, first time")
-	theirControl.InjectUDPPacket(msg1)
-	_ = theirControl.GetFromUDP(true)
-
-	t.Log("Inject the SAME msg1 again, tests ErrAlreadySeen path")
-	theirControl.InjectUDPPacket(msg1)
-	resp2 := theirControl.GetFromUDP(true)
-	assert.NotNil(t, resp2, "should get cached response on duplicate msg1")
-
-	t.Log("Complete handshake with cached response")
-	myControl.InjectUDPPacket(resp2)
-	myControl.WaitForType(1, 0, theirControl)
-
-	t.Log("Drain cached packet and verify tunnel works")
-	cachedPacket := theirControl.GetFromTun(true)
-	assertUdpPacket(t, []byte("Hi"), cachedPacket, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	t.Log("Verify only one tunnel exists on each side")
-	assert.Len(t, myControl.ListHostmapHosts(false), 1)
-	assert.Len(t, theirControl.ListHostmapHosts(false), 1)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestHandshakeTruncatedPacketRecovery(t *testing.T) {
-	// Verify that a truncated handshake packet is ignored and the real
-	// packet can still complete the handshake.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	t.Log("Trigger handshake")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
-
-	t.Log("Get msg1 and deliver to responder")
-	msg1 := myControl.GetFromUDP(true)
-	theirControl.InjectUDPPacket(msg1)
-
-	t.Log("Get the real response")
-	realResp := theirControl.GetFromUDP(true)
-
-	t.Log("Truncate the response and inject, should be ignored")
-	truncResp := realResp.Copy()
-	truncResp.Data = truncResp.Data[:header.Len]
-	myControl.InjectUDPPacket(truncResp)
-
-	t.Log("Verify pending handshake survived the truncated packet")
-	assert.NotEmpty(t, myControl.ListHostmapHosts(true), "pending handshake should still exist")
-
-	t.Log("Inject real response, should complete handshake")
-	myControl.InjectUDPPacket(realResp)
-	myControl.WaitForType(1, 0, theirControl)
-
-	t.Log("Drain and verify tunnel")
-	cachedPacket := theirControl.GetFromTun(true)
-	assertUdpPacket(t, []byte("Hi"), cachedPacket, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestHandshakeOrphanedMsg2Dropped(t *testing.T) {
-	// A msg2 arriving with no matching pending index should be silently dropped
-	// with no response sent and no state changes.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	t.Log("Complete a normal handshake")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
-	r.RouteForAllUntilTxTun(theirControl)
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	t.Log("Record hostmap state")
-	myIndexes := len(myControl.ListHostmapIndexes(false))
-
-	t.Log("Inject a fake msg2 with unknown RemoteIndex")
-	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0xDEADBEEF, 2))
-
-	t.Log("Verify no new indexes created")
-	assert.Equal(t, myIndexes, len(myControl.ListHostmapIndexes(false)))
-
-	t.Log("Verify no UDP response was sent")
-	time.Sleep(100 * time.Millisecond)
-	assert.Nil(t, myControl.GetFromUDP(false), "should not send a response to orphaned msg2")
-
-	t.Log("Verify existing tunnel still works")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestHandshakeUnknownMessageCounter(t *testing.T) {
-	// A handshake packet with an unexpected message counter should be silently
-	// dropped with no side effects and no UDP response.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	myControl.Start()
-	theirControl.Start()
-
-	t.Log("Inject handshake with MessageCounter=3")
-	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 3))
-
-	t.Log("Inject handshake with MessageCounter=99")
-	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 99))
-
-	t.Log("Verify no tunnels or pending handshakes")
-	assert.Empty(t, myControl.ListHostmapHosts(false))
-	assert.Empty(t, myControl.ListHostmapHosts(true))
-
-	t.Log("Verify no UDP response was sent")
-	time.Sleep(100 * time.Millisecond)
-	assert.Nil(t, myControl.GetFromUDP(false))
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestHandshakeUnknownSubtype(t *testing.T) {
-	// A handshake packet with an unknown subtype should be silently dropped.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
-	theirControl, _, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.Start()
-	theirControl.Start()
-
-	t.Log("Inject handshake with unknown subtype 99")
-	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.MessageSubType(99), 0, 1))
-
-	t.Log("Verify no tunnels or pending handshakes")
-	assert.Empty(t, myControl.ListHostmapHosts(false))
-	assert.Empty(t, myControl.ListHostmapHosts(true))
-
-	t.Log("Verify no UDP response was sent")
-	time.Sleep(100 * time.Millisecond)
-	assert.Nil(t, myControl.GetFromUDP(false))
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestHandshakeLateResponse(t *testing.T) {
-	// After a handshake times out, a late response should be silently ignored
-	// with no new tunnels created.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{
-		"handshakes": m{
-			"try_interval": "200ms",
-			"retries":      2,
-		},
-	})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	myControl.Start()
-	theirControl.Start()
-
-	t.Log("Trigger handshake from me")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
-
-	t.Log("Grab msg1 but don't deliver")
-	msg1 := myControl.GetFromUDP(true)
-
-	t.Log("Wait for handshake to time out")
-	for i := 0; i < 5; i++ {
-		time.Sleep(300 * time.Millisecond)
-		myControl.GetFromUDP(false)
-	}
-
-	t.Log("Confirm no pending handshakes remain")
-	assert.Empty(t, myControl.ListHostmapHosts(true))
-
-	t.Log("Deliver old msg1 to them, they create a tunnel")
-	theirControl.InjectUDPPacket(msg1)
-	resp := theirControl.GetFromUDP(true)
-	assert.NotNil(t, resp)
-
-	t.Log("Inject late response into me, should be ignored")
-	myControl.InjectUDPPacket(resp)
-
-	t.Log("No tunnel should exist on my side")
-	assert.Empty(t, myControl.ListHostmapHosts(false))
-	assert.Empty(t, myControl.ListHostmapHosts(true))
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestHandshakeSelfConnectionRejected(t *testing.T) {
-	// Verify that a node rejects a handshake containing its own VPN IP in the
-	// peer cert. We do this by sending the initiator's own msg1 back to itself.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
-
-	// Need a lighthouse entry to trigger a handshake
-	myControl.InjectLightHouseAddr(netip.MustParseAddr("10.128.0.2"), netip.MustParseAddrPort("10.0.0.2:4242"))
-
-	myControl.Start()
-
-	t.Log("Trigger handshake from me")
-	myControl.InjectTunUDPPacket(netip.MustParseAddr("10.128.0.2"), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
-	msg1 := myControl.GetFromUDP(true)
-
-	t.Log("Drain any handshake retransmits before injecting")
-	time.Sleep(100 * time.Millisecond)
-	for myControl.GetFromUDP(false) != nil {
-	}
-
-	t.Log("Feed my own msg1 back to me as if it came from someone else")
-	selfMsg := msg1.Copy()
-	selfMsg.From = netip.MustParseAddrPort("10.0.0.99:4242")
-	selfMsg.To = myUdpAddr
-	myControl.InjectUDPPacket(selfMsg)
-
-	t.Log("Verify no response was sent (self-connection rejected)")
-	time.Sleep(100 * time.Millisecond)
-	// Drain any further retransmits from the original handshake, then check
-	// that none of them are a handshake response (MessageCounter=2)
-	h := &header.H{}
-	for {
-		p := myControl.GetFromUDP(false)
-		if p == nil {
-			break
-		}
-		_ = h.Parse(p.Data)
-		assert.NotEqual(t, uint64(2), h.MessageCounter,
-			"should not send a stage 2 response to self-connection")
-	}
-
-	t.Log("Verify no tunnel to myself was created")
-	assert.Nil(t, myControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false))
-
-	myControl.Stop()
-}
-
-func TestHandshakeMessageCounter0Dropped(t *testing.T) {
-	// MessageCounter=0 is not a valid handshake message and should be dropped.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
-	_, _, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.Start()
-
-	t.Log("Inject handshake with MessageCounter=0")
-	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 0))
-
-	time.Sleep(100 * time.Millisecond)
-	assert.Empty(t, myControl.ListHostmapHosts(false))
-	assert.Empty(t, myControl.ListHostmapHosts(true))
-	assert.Nil(t, myControl.GetFromUDP(false))
-
-	myControl.Stop()
-}
-
-func TestHandshakeRemoteAllowList(t *testing.T) {
-	// Verify that a handshake from a blocked underlay IP is dropped with no
-	// response and no state changes. Then verify the same packet from an
-	// allowed IP succeeds.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{
-		"lighthouse": m{
-			"remote_allow_list": m{
-				"10.0.0.0/8": true,
-				"0.0.0.0/0":  false,
-			},
-		},
-	})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	t.Log("Trigger handshake from them")
-	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi"))
-	msg1 := theirControl.GetFromUDP(true)
-
-	t.Log("Rewrite the source to a blocked IP and inject")
-	blockedMsg := msg1.Copy()
-	blockedMsg.From = netip.MustParseAddrPort("192.168.1.1:4242")
-	myControl.InjectUDPPacket(blockedMsg)
-
-	t.Log("Verify no tunnel, no pending, no response from blocked source")
-	time.Sleep(100 * time.Millisecond)
-	assert.Empty(t, myControl.ListHostmapHosts(false))
-	assert.Empty(t, myControl.ListHostmapHosts(true))
-	assert.Nil(t, myControl.GetFromUDP(false), "should not respond to blocked source")
-
-	t.Log("Now inject the real packet from the allowed source")
-	myControl.InjectUDPPacket(msg1)
-
-	t.Log("Verify handshake completes from allowed source")
-	resp := myControl.GetFromUDP(true)
-	assert.NotNil(t, resp)
-	theirControl.InjectUDPPacket(resp)
-	theirControl.WaitForType(1, 0, myControl)
-
-	t.Log("Drain cached packet and verify tunnel works")
-	cachedPacket := myControl.GetFromTun(true)
-	assertUdpPacket(t, []byte("Hi"), cachedPacket, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), 80, 80)
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestHandshakeAlreadySeenPreferredRemote(t *testing.T) {
-	// When a duplicate msg1 arrives via ErrAlreadySeen, verify the tunnel
-	// remains functional and hostmap index count is stable.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	t.Log("Complete a normal handshake via the router")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
-	r.RouteForAllUntilTxTun(theirControl)
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	t.Log("Record hostmap state")
-	theirIndexes := len(theirControl.ListHostmapIndexes(false))
-	hi := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-	assert.NotNil(t, hi)
-	originalRemote := hi.CurrentRemote
-
-	t.Log("Re-trigger traffic to cause a new handshake attempt (ErrAlreadySeen)")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("roam"))
-	r.RouteForAllUntilTxTun(theirControl)
-
-	t.Log("Verify tunnel still works")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	t.Log("Verify remote is still valid and index count is stable")
-	hi2 := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-	assert.NotNil(t, hi2)
-	assert.Equal(t, originalRemote, hi2.CurrentRemote)
-	assert.Equal(t, theirIndexes, len(theirControl.ListHostmapIndexes(false)),
-		"no extra indexes should be created from ErrAlreadySeen")
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestHandshakeWrongResponderPacketStore(t *testing.T) {
-	// Verify that when the wrong host responds, the cached packets are
-	// transferred to the new handshake, the evil tunnel is closed, evil's
-	// address is blocked, and the correct tunnel is eventually established.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.100/24", nil)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.99/24", nil)
-	evilControl, evilVpnIpNet, evilUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "evil", "10.128.0.2/24", nil)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), evilUdpAddr)
-
-	r := router.NewR(t, myControl, theirControl, evilControl)
-	defer r.RenderFlow()
-
-	myControl.Start()
-	theirControl.Start()
-	evilControl.Start()
-
-	t.Log("Send multiple packets to them (cached during handshake)")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet1"))
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet2"))
-
-	t.Log("Route until evil tunnel is closed")
-	h := &header.H{}
-	r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType {
-		if err := h.Parse(p.Data); err != nil {
-			panic(err)
-		}
-		if h.Type == header.CloseTunnel && p.To == evilUdpAddr {
-			return router.RouteAndExit
-		}
-		return router.KeepRouting
-	})
-
-	t.Log("Verify evil's address is blocked in the new pending handshake")
-	pendingHI := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), true)
-	if pendingHI != nil {
-		assert.NotContains(t, pendingHI.RemoteAddrs, evilUdpAddr,
-			"evil's address should be blocked")
-	}
-
-	t.Log("Inject correct lighthouse addr for them")
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	t.Log("Route until cached packets arrive at the real them")
-	p := r.RouteForAllUntilTxTun(theirControl)
-	assert.NotNil(t, p, "a cached packet should be delivered to the correct host")
-
-	t.Log("Verify the correct host has a tunnel")
-	assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet, theirVpnIpNet, myControl, theirControl)
-
-	t.Log("Verify no hostinfo artifacts from evil remain")
-	assert.Nil(t, myControl.GetHostInfoByVpnAddr(evilVpnIpNet[0].Addr(), true),
-		"no pending hostinfo for evil")
-	assert.Nil(t, myControl.GetHostInfoByVpnAddr(evilVpnIpNet[0].Addr(), false),
-		"no main hostinfo for evil")
-
-	myControl.Stop()
-	theirControl.Stop()
-	evilControl.Stop()
-}
-
-func TestHandshakeRelayComplete(t *testing.T) {
-	// Verify that a relay handshake completes correctly and relay state is
-	// properly maintained on all three nodes.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
-	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
-
-	myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
-	myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
-	relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	r := router.NewR(t, myControl, relayControl, theirControl)
-	defer r.RenderFlow()
-
-	myControl.Start()
-	relayControl.Start()
-	theirControl.Start()
-
-	t.Log("Trigger handshake via relay")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi via relay"))
-
-	p := r.RouteForAllUntilTxTun(theirControl)
-	assertUdpPacket(t, []byte("Hi via relay"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
-
-	t.Log("Verify bidirectional tunnel via relay")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	t.Log("Verify relay state on my side shows relay-to-me")
-	myHI := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
-	assert.NotNil(t, myHI)
-	assert.NotEmpty(t, myHI.CurrentRelaysToMe, "should have relay-to-me for them")
-
-	t.Log("Verify relay state on their side shows relay-to-me")
-	theirHI := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-	assert.NotNil(t, theirHI)
-	assert.NotEmpty(t, theirHI.CurrentRelaysToMe, "should have relay-to-me for me")
-
-	t.Log("Verify relay node shows through-me relays")
-	relayHI := relayControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-	assert.NotNil(t, relayHI)
-
-	myControl.Stop()
-	relayControl.Stop()
-	theirControl.Stop()
-}
-
-// NOTE: Relay V1 cert + IPv6 rejection is not tested here because
-// InjectTunUDPPacket from a V4 node to a V6 address panics in the test
-// framework. The check is in handshake_manager.go handleOutbound relay
-// logic (lines ~304-313): if the relay host has a V1 cert and either
-// address is IPv6, the relay is skipped.
-
-// NOTE: Relay reestablishment (Disestablished state transition) is covered
-// by the existing TestReestablishRelays in handshakes_test.go.

e2e/handshakes_test.go

@@ -20,17 +20,16 @@ import (
 	"github.com/slackhq/nebula/udp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.yaml.in/yaml/v3"
+	"gopkg.in/yaml.v3"
 )
 
 func BenchmarkHotPath(b *testing.B) {
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
 	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
 
 	// Put their info in our lighthouse
 	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
 
 	// Start the servers
 	myControl.Start()
@@ -39,9 +38,6 @@ func BenchmarkHotPath(b *testing.B) {
 	r := router.NewR(b, myControl, theirControl)
 	r.CancelFlowLogs()
 
-	assertTunnel(b, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-	b.ResetTimer()
-
 	for n := 0; n < b.N; n++ {
 		myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 		_ = r.RouteForAllUntilTxTun(theirControl)
@@ -51,39 +47,6 @@ func BenchmarkHotPath(b *testing.B) {
 	theirControl.Stop()
 }
 
-func BenchmarkHotPathRelay(b *testing.B) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
-	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay  ", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them   ", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
-
-	// Teach my how to get to the relay and that their can be reached via the relay
-	myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
-	myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
-	relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	// Build a router so we don't have to reason who gets which packet
-	r := router.NewR(b, myControl, relayControl, theirControl)
-	r.CancelFlowLogs()
-
-	// Start the servers
-	myControl.Start()
-	relayControl.Start()
-	theirControl.Start()
-
-	assertTunnel(b, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
-	b.ResetTimer()
-
-	for n := 0; n < b.N; n++ {
-		myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-		_ = r.RouteForAllUntilTxTun(theirControl)
-	}
-
-	myControl.Stop()
-	theirControl.Stop()
-	relayControl.Stop()
-}
-
 func TestGoodHandshake(t *testing.T) {
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
@@ -134,41 +97,6 @@ func TestGoodHandshake(t *testing.T) {
 	theirControl.Stop()
 }
 
-func TestGoodHandshakeNoOverlap(t *testing.T) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.1/24", nil)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "2001::69/24", nil) //look ma, cross-stack!
-
-	// Put their info in our lighthouse
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	empty := []byte{}
-	t.Log("do something to cause a handshake")
-	myControl.GetF().SendMessageToVpnAddr(header.Test, header.MessageNone, theirVpnIpNet[0].Addr(), empty, empty, empty)
-
-	t.Log("Have them consume my stage 0 packet. They have a tunnel now")
-	theirControl.InjectUDPPacket(myControl.GetFromUDP(true))
-
-	t.Log("Get their stage 1 packet")
-	stage1Packet := theirControl.GetFromUDP(true)
-
-	t.Log("Have me consume their stage 1 packet. I have a tunnel now")
-	myControl.InjectUDPPacket(stage1Packet)
-
-	t.Log("Wait until we see a test packet come through to make sure we give the tunnel time to complete")
-	myControl.WaitForType(header.Test, 0, theirControl)
-
-	t.Log("Make sure our host infos are correct")
-	assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet, theirVpnIpNet, myControl, theirControl)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
 func TestWrongResponderHandshake(t *testing.T) {
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 
@@ -536,35 +464,6 @@ func TestRelays(t *testing.T) {
 	r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
 }
 
-func TestRelaysDontCareAboutIps(t *testing.T) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
-	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "relay  ", "2001::9999/24", m{"relay": m{"am_relay": true}})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them   ", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
-
-	// Teach my how to get to the relay and that their can be reached via the relay
-	myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
-	myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
-	relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	// Build a router so we don't have to reason who gets which packet
-	r := router.NewR(t, myControl, relayControl, theirControl)
-	defer r.RenderFlow()
-
-	// Start the servers
-	myControl.Start()
-	relayControl.Start()
-	theirControl.Start()
-
-	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-
-	p := r.RouteForAllUntilTxTun(theirControl)
-	r.Log("Assert the tunnel works")
-	assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
-	r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
-}
-
 func TestReestablishRelays(t *testing.T) {
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
@@ -1328,109 +1227,3 @@ func TestV2NonPrimaryWithLighthouse(t *testing.T) {
 	myControl.Stop()
 	theirControl.Stop()
 }
-
-func TestV2NonPrimaryWithOffNetLighthouse(t *testing.T) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh  ", "2001::1/64", m{"lighthouse": m{"am_lighthouse": true}})
-
-	o := m{
-		"static_host_map": m{
-			lhVpnIpNet[0].Addr().String(): []string{lhUdpAddr.String()},
-		},
-		"lighthouse": m{
-			"hosts": []string{lhVpnIpNet[0].Addr().String()},
-			"local_allow_list": m{
-				// Try and block our lighthouse updates from using the actual addresses assigned to this computer
-				// If we start discovering addresses the test router doesn't know about then test traffic cant flow
-				"10.0.0.0/24": true,
-				"::/0":        false,
-			},
-		},
-	}
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me  ", "10.128.0.2/24, ff::2/64", o)
-	theirControl, theirVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.3/24, ff::3/64", o)
-
-	// Build a router so we don't have to reason who gets which packet
-	r := router.NewR(t, lhControl, myControl, theirControl)
-	defer r.RenderFlow()
-
-	// Start the servers
-	lhControl.Start()
-	myControl.Start()
-	theirControl.Start()
-
-	t.Log("Stand up an ipv6 tunnel between me and them")
-	assert.True(t, myVpnIpNet[1].Addr().Is6())
-	assert.True(t, theirVpnIpNet[1].Addr().Is6())
-	assertTunnel(t, myVpnIpNet[1].Addr(), theirVpnIpNet[1].Addr(), myControl, theirControl, r)
-
-	lhControl.Stop()
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestGoodHandshakeUnsafeDest(t *testing.T) {
-	unsafePrefix := "192.168.6.0/24"
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServerWithUdpAndUnsafeNetworks(cert.Version2, ca, caKey, "spooky", "10.128.0.2/24", netip.MustParseAddrPort("10.64.0.2:4242"), unsafePrefix, nil)
-	route := m{"route": unsafePrefix, "via": theirVpnIpNet[0].Addr().String()}
-	myCfg := m{
-		"tun": m{
-			"unsafe_routes": []m{route},
-		},
-	}
-	myControl, myVpnIpNet, myUdpAddr, myConfig := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.1/24", myCfg)
-	t.Logf("my config %v", myConfig)
-	// Put their info in our lighthouse
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	spookyDest := netip.MustParseAddr("192.168.6.4")
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	t.Log("Send a udp packet through to begin standing up the tunnel, this should come out the other side")
-	myControl.InjectTunUDPPacket(spookyDest, 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-
-	t.Log("Have them consume my stage 0 packet. They have a tunnel now")
-	theirControl.InjectUDPPacket(myControl.GetFromUDP(true))
-
-	t.Log("Get their stage 1 packet so that we can play with it")
-	stage1Packet := theirControl.GetFromUDP(true)
-
-	t.Log("I consume a garbage packet with a proper nebula header for our tunnel")
-	// this should log a statement and get ignored, allowing the real handshake packet to complete the tunnel
-	badPacket := stage1Packet.Copy()
-	badPacket.Data = badPacket.Data[:len(badPacket.Data)-header.Len]
-	myControl.InjectUDPPacket(badPacket)
-
-	t.Log("Have me consume their real stage 1 packet. I have a tunnel now")
-	myControl.InjectUDPPacket(stage1Packet)
-
-	t.Log("Wait until we see my cached packet come through")
-	myControl.WaitForType(1, 0, theirControl)
-
-	t.Log("Make sure our host infos are correct")
-	assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet, theirVpnIpNet, myControl, theirControl)
-
-	t.Log("Get that cached packet and make sure it looks right")
-	myCachedPacket := theirControl.GetFromTun(true)
-	assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIpNet[0].Addr(), spookyDest, 80, 80)
-
-	//reply
-	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, spookyDest, 80, []byte("Hi from the spookyman"))
-	//wait for reply
-	theirControl.WaitForType(1, 0, myControl)
-	theirCachedPacket := myControl.GetFromTun(true)
-	assertUdpPacket(t, []byte("Hi from the spookyman"), theirCachedPacket, spookyDest, myVpnIpNet[0].Addr(), 80, 80)
-
-	t.Log("Do a bidirectional tunnel test")
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	r.RenderHostmaps("Final hostmaps", myControl, theirControl)
-	myControl.Stop()
-	theirControl.Stop()
-}

e2e/helpers_test.go

@@ -22,14 +22,15 @@ import (
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/e2e/router"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"go.yaml.in/yaml/v3"
+	"gopkg.in/yaml.v3"
 )
 
 type m = map[string]any
 
 // newSimpleServer creates a nebula instance with many assumptions
 func newSimpleServer(v cert.Version, caCrt cert.Certificate, caKey []byte, name string, sVpnNetworks string, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) {
+	l := NewTestLogger()
+
 	var vpnNetworks []netip.Prefix
 	for _, sn := range strings.Split(sVpnNetworks, ",") {
 		vpnIpNet, err := netip.ParsePrefix(strings.TrimSpace(sn))
@@ -55,54 +56,7 @@ func newSimpleServer(v cert.Version, caCrt cert.Certificate, caKey []byte, name
 		budpIp[3] = 239
 		udpAddr = netip.AddrPortFrom(netip.AddrFrom16(budpIp), 4242)
 	}
-	return newSimpleServerWithUdp(v, caCrt, caKey, name, sVpnNetworks, udpAddr, overrides)
-}
-
-func newSimpleServerWithUdp(v cert.Version, caCrt cert.Certificate, caKey []byte, name string, sVpnNetworks string, udpAddr netip.AddrPort, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) {
-	return newSimpleServerWithUdpAndUnsafeNetworks(v, caCrt, caKey, name, sVpnNetworks, udpAddr, "", overrides)
-}
-
-func newSimpleServerWithUdpAndUnsafeNetworks(v cert.Version, caCrt cert.Certificate, caKey []byte, name string, sVpnNetworks string, udpAddr netip.AddrPort, sUnsafeNetworks string, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) {
-	l := NewTestLogger()
-
-	var vpnNetworks []netip.Prefix
-	for _, sn := range strings.Split(sVpnNetworks, ",") {
-		vpnIpNet, err := netip.ParsePrefix(strings.TrimSpace(sn))
-		if err != nil {
-			panic(err)
-		}
-		vpnNetworks = append(vpnNetworks, vpnIpNet)
-	}
-
-	if len(vpnNetworks) == 0 {
-		panic("no vpn networks")
-	}
-
-	firewallInbound := []m{{
-		"proto": "any",
-		"port":  "any",
-		"host":  "any",
-	}}
-
-	var unsafeNetworks []netip.Prefix
-	if sUnsafeNetworks != "" {
-		firewallInbound = []m{{
-			"proto":      "any",
-			"port":       "any",
-			"host":       "any",
-			"local_cidr": "0.0.0.0/0",
-		}}
-
-		for _, sn := range strings.Split(sUnsafeNetworks, ",") {
-			x, err := netip.ParsePrefix(strings.TrimSpace(sn))
-			if err != nil {
-				panic(err)
-			}
-			unsafeNetworks = append(unsafeNetworks, x)
-		}
-	}
-
-	_, _, myPrivKey, myPEM := cert_test.NewTestCert(v, cert.Curve_CURVE25519, caCrt, caKey, name, time.Now(), time.Now().Add(5*time.Minute), vpnNetworks, unsafeNetworks, []string{})
+	_, _, myPrivKey, myPEM := cert_test.NewTestCert(v, cert.Curve_CURVE25519, caCrt, caKey, name, time.Now(), time.Now().Add(5*time.Minute), vpnNetworks, nil, []string{})
 
 	caB, err := caCrt.MarshalPEM()
 	if err != nil {
@@ -122,7 +76,11 @@ func newSimpleServerWithUdpAndUnsafeNetworks(v cert.Version, caCrt cert.Certific
 				"port":  "any",
 				"host":  "any",
 			}},
-			"inbound": firewallInbound,
+			"inbound": []m{{
+				"proto": "any",
+				"port":  "any",
+				"host":  "any",
+			}},
 		},
 		//"handshakes": m{
 		//	"try_interval": "1s",
@@ -292,7 +250,7 @@ func deadline(t *testing.T, seconds time.Duration) doneCb {
 	}
 }
 
-func assertTunnel(t testing.TB, vpnIpA, vpnIpB netip.Addr, controlA, controlB *nebula.Control, r *router.R) {
+func assertTunnel(t *testing.T, vpnIpA, vpnIpB netip.Addr, controlA, controlB *nebula.Control, r *router.R) {
 	// Send a packet from them to me
 	controlB.InjectTunUDPPacket(vpnIpA, 80, vpnIpB, 90, []byte("Hi from B"))
 	bPacket := r.RouteForAllUntilTxTun(controlA)
@@ -304,14 +262,14 @@ func assertTunnel(t testing.TB, vpnIpA, vpnIpB netip.Addr, controlA, controlB *n
 	assertUdpPacket(t, []byte("Hello from A"), aPacket, vpnIpA, vpnIpB, 90, 80)
 }
 
-func assertHostInfoPair(t testing.TB, addrA, addrB netip.AddrPort, vpnNetsA, vpnNetsB []netip.Prefix, controlA, controlB *nebula.Control) {
+func assertHostInfoPair(t *testing.T, addrA, addrB netip.AddrPort, vpnNetsA, vpnNetsB []netip.Prefix, controlA, controlB *nebula.Control) {
 	// Get both host infos
 	//TODO: CERT-V2 we may want to loop over each vpnAddr and assert all the things
 	hBinA := controlA.GetHostInfoByVpnAddr(vpnNetsB[0].Addr(), false)
-	require.NotNil(t, hBinA, "Host B was not found by vpnAddr in controlA")
+	assert.NotNil(t, hBinA, "Host B was not found by vpnAddr in controlA")
 
 	hAinB := controlB.GetHostInfoByVpnAddr(vpnNetsA[0].Addr(), false)
-	require.NotNil(t, hAinB, "Host A was not found by vpnAddr in controlB")
+	assert.NotNil(t, hAinB, "Host A was not found by vpnAddr in controlB")
 
 	// Check that both vpn and real addr are correct
 	assert.EqualValues(t, getAddrs(vpnNetsB), hBinA.VpnAddrs, "Host B VpnIp is wrong in control A")
@@ -325,7 +283,7 @@ func assertHostInfoPair(t testing.TB, addrA, addrB netip.AddrPort, vpnNetsA, vpn
 	assert.Equal(t, hBinA.RemoteIndex, hAinB.LocalIndex, "Host B remote index does not match host A local index")
 }
 
-func assertUdpPacket(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
+func assertUdpPacket(t *testing.T, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
 	if toIp.Is6() {
 		assertUdpPacket6(t, expected, b, fromIp, toIp, fromPort, toPort)
 	} else {
@@ -333,7 +291,7 @@ func assertUdpPacket(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr,
 	}
 }
 
-func assertUdpPacket6(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
+func assertUdpPacket6(t *testing.T, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
 	packet := gopacket.NewPacket(b, layers.LayerTypeIPv6, gopacket.Lazy)
 	v6 := packet.Layer(layers.LayerTypeIPv6).(*layers.IPv6)
 	assert.NotNil(t, v6, "No ipv6 data found")
@@ -352,7 +310,7 @@ func assertUdpPacket6(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr,
 	assert.Equal(t, expected, data.Payload(), "Data was incorrect")
 }
 
-func assertUdpPacket4(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
+func assertUdpPacket4(t *testing.T, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
 	packet := gopacket.NewPacket(b, layers.LayerTypeIPv4, gopacket.Lazy)
 	v4 := packet.Layer(layers.LayerTypeIPv4).(*layers.IPv4)
 	assert.NotNil(t, v4, "No ipv4 data found")

e2e/tunnels_test.go

@@ -12,8 +12,6 @@ import (
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/cert_test"
 	"github.com/slackhq/nebula/e2e/router"
-	"github.com/slackhq/nebula/header"
-	"github.com/slackhq/nebula/udp"
 	"github.com/stretchr/testify/assert"
 	"gopkg.in/yaml.v3"
 )
@@ -320,153 +318,3 @@ func TestCertMismatchCorrection(t *testing.T) {
 	myControl.Stop()
 	theirControl.Stop()
 }
-
-func TestCrossStackRelaysWork(t *testing.T) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me     ", "10.128.0.1/24,fc00::1/64", m{"relay": m{"use_relays": true}})
-	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "relay  ", "10.128.0.128/24,fc00::128/64", m{"relay": m{"am_relay": true}})
-	theirUdp := netip.MustParseAddrPort("10.0.0.2:4242")
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServerWithUdp(cert.Version2, ca, caKey, "them   ", "fc00::2/64", theirUdp, m{"relay": m{"use_relays": true}})
-
-	//myVpnV4 := myVpnIpNet[0]
-	myVpnV6 := myVpnIpNet[1]
-	relayVpnV4 := relayVpnIpNet[0]
-	relayVpnV6 := relayVpnIpNet[1]
-	theirVpnV6 := theirVpnIpNet[0]
-
-	// Teach my how to get to the relay and that their can be reached via the relay
-	myControl.InjectLightHouseAddr(relayVpnV4.Addr(), relayUdpAddr)
-	myControl.InjectLightHouseAddr(relayVpnV6.Addr(), relayUdpAddr)
-	myControl.InjectRelays(theirVpnV6.Addr(), []netip.Addr{relayVpnV6.Addr()})
-	relayControl.InjectLightHouseAddr(theirVpnV6.Addr(), theirUdpAddr)
-
-	// Build a router so we don't have to reason who gets which packet
-	r := router.NewR(t, myControl, relayControl, theirControl)
-	defer r.RenderFlow()
-
-	// Start the servers
-	myControl.Start()
-	relayControl.Start()
-	theirControl.Start()
-
-	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunUDPPacket(theirVpnV6.Addr(), 80, myVpnV6.Addr(), 80, []byte("Hi from me"))
-
-	p := r.RouteForAllUntilTxTun(theirControl)
-	r.Log("Assert the tunnel works")
-	assertUdpPacket(t, []byte("Hi from me"), p, myVpnV6.Addr(), theirVpnV6.Addr(), 80, 80)
-
-	t.Log("reply?")
-	theirControl.InjectTunUDPPacket(myVpnV6.Addr(), 80, theirVpnV6.Addr(), 80, []byte("Hi from them"))
-	p = r.RouteForAllUntilTxTun(myControl)
-	assertUdpPacket(t, []byte("Hi from them"), p, theirVpnV6.Addr(), myVpnV6.Addr(), 80, 80)
-
-	r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
-	//t.Log("finish up")
-	//myControl.Stop()
-	//theirControl.Stop()
-	//relayControl.Stop()
-}
-
-func TestCloseTunnelAuthenticated(t *testing.T) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "5s"}})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "10m"}})
-
-	// Share our underlay information
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-
-	r.Log("Assert the tunnel between me and them works")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	r.Log("Close the tunnel")
-	myControl.CloseTunnel(theirVpnIpNet[0].Addr(), false)
-	r.FlushAll()
-
-	waitStart := time.Now()
-	for {
-		myIndexes := len(myControl.GetHostmap().Indexes)
-		theirIndexes := len(theirControl.GetHostmap().Indexes)
-		if myIndexes == 0 && theirIndexes == 0 {
-			break
-		}
-
-		since := time.Since(waitStart)
-		r.Logf("my tunnels: %v; their tunnels: %v; duration: %v", myIndexes, theirIndexes, since)
-		if since > time.Second*6 {
-			t.Fatal("Tunnel should have been declared inactive after 2 seconds and before 6 seconds")
-		}
-
-		time.Sleep(1 * time.Second)
-		//r.FlushAll()
-	}
-
-	r.Logf("Happy path success, tunnels were dropped within %v", time.Since(waitStart))
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-	r.Log("Assert another tunnel between me and them works")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-	hi := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
-	if hi == nil {
-		t.Fatal("There is no hostinfo for this tunnel")
-	}
-	myHi := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-	if myHi == nil {
-		t.Fatal("There is no hostinfo for my tunnel")
-	}
-	r.Log("It does")
-
-	buf := make([]byte, 1024)
-	hdr := header.H{
-		Version:        1,
-		Type:           header.CloseTunnel,
-		Subtype:        0,
-		Reserved:       0,
-		RemoteIndex:    hi.RemoteIndex,
-		MessageCounter: 5,
-	}
-	out, err := hdr.Encode(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	pkt := &udp.Packet{
-		To:   hi.CurrentRemote,
-		From: myHi.CurrentRemote,
-		Data: out,
-	}
-	r.InjectUDPPacket(myControl, theirControl, pkt)
-	r.Log("Injected bogus close tunnel. Let's see!")
-	waitStart = time.Now()
-	for {
-		myIndexes := len(myControl.GetHostmap().Indexes)
-		theirIndexes := len(theirControl.GetHostmap().Indexes)
-		if myIndexes == 0 {
-			t.Fatal("myIndexes should not be 0")
-		}
-		if theirIndexes == 0 {
-			t.Fatal("theirIndexes should not be 0, they should have rejected this bogus packet")
-		}
-
-		since := time.Since(waitStart)
-		r.Logf("my tunnels: %v; their tunnels: %v; duration: %v", myIndexes, theirIndexes, since)
-		if since > time.Second*4 {
-			t.Log("The tunnel would have been gone by now")
-			break
-		}
-
-		time.Sleep(1 * time.Second)
-		r.FlushAll()
-	}
-
-	myControl.Stop()
-	theirControl.Stop()
-}

examples/config.yml

@@ -144,10 +144,6 @@ listen:
   # valid values: always, never, private
   # This setting is reloadable.
   #send_recv_error: always
-  # Similar to send_recv_error, this option lets you configure if you want to accept "recv_error" packets from remote hosts.
-  # valid values: always, never, private
-  # This setting is reloadable.
-  #accept_recv_error: always
   # The so_sock option is a Linux-specific feature that allows all outgoing Nebula packets to be tagged with a specific identifier.
   # This tagging enables IP rule-based filtering. For example, it supports 0.0.0.0/0 unsafe_routes,
   # allowing for more precise routing decisions based on the packet tags. Default is 0 meaning no mark is set.
@@ -204,12 +200,6 @@ punchy:
   # Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access.
   #trusted_cas:
     #- "ssh public key string"
-  # sandbox_dir restricts file paths for profiling commands (start-cpu-profile, save-heap-profile,
-  # save-mutex-profile) to the specified directory. Relative paths will be resolved within this directory,
-  # and absolute paths outside of it will be rejected. Default is $TMP/nebula-debug.
-  # The directory is NOT automatically created.
-  # Overriding this to "" is the same as "/" and will allow overwriting any path on the host.
-  #sandbox_dir: /var/tmp/nebula-debug
 
 # EXPERIMENTAL: relay support for networks that can't establish direct connections.
 relay:
@@ -388,14 +378,13 @@ firewall:
   # Rules are comprised of a protocol, port, and one or more of host, group, or CIDR
   # Logical evaluation is roughly: port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND (local cidr)
   # - port: Takes `0` or `any` as any, a single number `80`, a range `200-901`, or `fragment` to match second and further fragments of fragmented packets (since there is no port available).
+  #   code: same as port but makes more sense when talking about ICMP, TODO: this is not currently implemented in a way that works, use `any`
   #   proto: `any`, `tcp`, `udp`, or `icmp`
-  #          a port specification is ignored if proto is `icmp`
   #   host: `any` or a literal hostname, ie `test-host`
   #   group: `any` or a literal group name, ie `default-group`
   #   groups: Same as group but accepts a list of values. Multiple values are AND'd together and a certificate would have to contain all groups to pass
-  #   cidr: a remote CIDR, `0.0.0.0/0` is any ipv4 and `::/0` is any ipv6. `any` means any ip family and address.
-  #   local_cidr: a local CIDR, `0.0.0.0/0` is any ipv4 and `::/0` is any ipv6. `any` means any ip family and address.
-  #     This can be used to filter destinations when using unsafe_routes.
+  #   cidr: a remote CIDR, `0.0.0.0/0` is any ipv4 and `::/0` is any ipv6.
+  #   local_cidr: a local CIDR, `0.0.0.0/0` is any ipv4 and `::/0` is any ipv6. This can be used to filter destinations when using unsafe_routes.
   #     By default, this is set to only the VPN (overlay) networks assigned via the certificate networks field unless `default_local_cidr_any` is set to true.
   #     If there are unsafe_routes present in this config file, `local_cidr` should be set appropriately for the intended us case.
   #   ca_name: An issuing CA name

firewall.go

@@ -8,7 +8,6 @@ import (
 	"hash/fnv"
 	"net/netip"
 	"reflect"
-	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -20,11 +19,10 @@ import (
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/firewall"
-	"github.com/slackhq/nebula/firewall/events"
 )
 
 type FirewallInterface interface {
-	AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr string, caName string, caSha string) error
+	AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, addr, localAddr netip.Prefix, caName string, caSha string) error
 }
 
 type conn struct {
@@ -68,14 +66,6 @@ type Firewall struct {
 	incomingMetrics     firewallMetrics
 	outgoingMetrics     firewallMetrics
 
-	// reporter is the optional embedder-supplied event sink. Immutable for
-	// the lifetime of this Firewall; Control.SetFirewallEventReporter
-	// installs it by shallow-copying the Firewall under the conntrack lock
-	// and swapping the pointer, and reloadFirewall carries it forward.
-	// Read unsynchronized on the data path: the preceding Firewall-pointer
-	// read pins the field's value for the duration of that call.
-	reporter events.Reporter
-
 	l *logrus.Logger
 }
 
@@ -239,7 +229,7 @@ func NewFirewallFromConfig(l *logrus.Logger, cs *CertState, c *config.C) (*Firew
 	case "drop":
 		fw.OutSendReject = false
 	default:
-		l.WithField("action", outboundAction).Warn("invalid firewall.outbound_action, defaulting to `drop`")
+		l.WithField("action", inboundAction).Warn("invalid firewall.outbound_action, defaulting to `drop`")
 		fw.OutSendReject = false
 	}
 
@@ -257,7 +247,32 @@ func NewFirewallFromConfig(l *logrus.Logger, cs *CertState, c *config.C) (*Firew
 }
 
 // AddRule properly creates the in memory rule structure for a firewall table.
-func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error {
+func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, ip, localIp netip.Prefix, caName string, caSha string) error {
+	// Under gomobile, stringing a nil pointer with fmt causes an abort in debug mode for iOS
+	// https://github.com/golang/go/issues/14131
+	sIp := ""
+	if ip.IsValid() {
+		sIp = ip.String()
+	}
+	lIp := ""
+	if localIp.IsValid() {
+		lIp = localIp.String()
+	}
+
+	// We need this rule string because we generate a hash. Removing this will break firewall reload.
+	ruleString := fmt.Sprintf(
+		"incoming: %v, proto: %v, startPort: %v, endPort: %v, groups: %v, host: %v, ip: %v, localIp: %v, caName: %v, caSha: %s",
+		incoming, proto, startPort, endPort, groups, host, sIp, lIp, caName, caSha,
+	)
+	f.rules += ruleString + "\n"
+
+	direction := "incoming"
+	if !incoming {
+		direction = "outgoing"
+	}
+	f.l.WithField("firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "ip": sIp, "localIp": lIp, "caName": caName, "caSha": caSha}).
+		Info("Firewall rule added")
+
 	var (
 		ft *FirewallTable
 		fp firewallPort
@@ -275,12 +290,6 @@ func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort
 	case firewall.ProtoUDP:
 		fp = ft.UDP
 	case firewall.ProtoICMP, firewall.ProtoICMPv6:
-		//ICMP traffic doesn't have ports, so we always coerce to "any", even if a value is provided
-		if startPort != firewall.PortAny {
-			f.l.WithField("startPort", startPort).Warn("ignoring port specification for ICMP firewall rule")
-		}
-		startPort = firewall.PortAny
-		endPort = firewall.PortAny
 		fp = ft.ICMP
 	case firewall.ProtoAny:
 		fp = ft.AnyProto
@@ -288,21 +297,7 @@ func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort
 		return fmt.Errorf("unknown protocol %v", proto)
 	}
 
-	// We need this rule string because we generate a hash. Removing this will break firewall reload.
-	ruleString := fmt.Sprintf(
-		"incoming: %v, proto: %v, startPort: %v, endPort: %v, groups: %v, host: %v, ip: %v, localIp: %v, caName: %v, caSha: %s",
-		incoming, proto, startPort, endPort, groups, host, cidr, localCidr, caName, caSha,
-	)
-	f.rules += ruleString + "\n"
-
-	direction := "incoming"
-	if !incoming {
-		direction = "outgoing"
-	}
-	f.l.WithField("firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "cidr": cidr, "localCidr": localCidr, "caName": caName, "caSha": caSha}).
-		Info("Firewall rule added")
-
-	return fp.addRule(f, startPort, endPort, groups, host, cidr, localCidr, caName, caSha)
+	return fp.addRule(f, startPort, endPort, groups, host, ip, localIp, caName, caSha)
 }
 
 // GetRuleHash returns a hash representation of all inbound and outbound rules
@@ -342,6 +337,7 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw
 	}
 
 	for i, t := range rs {
+		var groups []string
 		r, err := convertRule(l, t, table, i)
 		if err != nil {
 			return fmt.Errorf("%s rule #%v; %s", table, i, err)
@@ -351,10 +347,23 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw
 			return fmt.Errorf("%s rule #%v; only one of port or code should be provided", table, i)
 		}
 
-		if r.Host == "" && len(r.Groups) == 0 && r.Cidr == "" && r.LocalCidr == "" && r.CAName == "" && r.CASha == "" {
+		if r.Host == "" && len(r.Groups) == 0 && r.Group == "" && r.Cidr == "" && r.LocalCidr == "" && r.CAName == "" && r.CASha == "" {
 			return fmt.Errorf("%s rule #%v; at least one of host, group, cidr, local_cidr, ca_name, or ca_sha must be provided", table, i)
 		}
 
+		if len(r.Groups) > 0 {
+			groups = r.Groups
+		}
+
+		if r.Group != "" {
+			// Check if we have both groups and group provided in the rule config
+			if len(groups) > 0 {
+				return fmt.Errorf("%s rule #%v; only one of group or groups should be defined, both provided", table, i)
+			}
+
+			groups = []string{r.Group}
+		}
+
 		var sPort, errPort string
 		if r.Code != "" {
 			errPort = "code"
@@ -364,51 +373,42 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw
 			sPort = r.Port
 		}
 
-		var proto uint8
-		var startPort, endPort int32
-		switch r.Proto {
-		case "any":
-			proto = firewall.ProtoAny
-			startPort, endPort, err = parsePort(sPort)
-		case "tcp":
-			proto = firewall.ProtoTCP
-			startPort, endPort, err = parsePort(sPort)
-		case "udp":
-			proto = firewall.ProtoUDP
-			startPort, endPort, err = parsePort(sPort)
-		case "icmp":
-			proto = firewall.ProtoICMP
-			startPort = firewall.PortAny
-			endPort = firewall.PortAny
-			if sPort != "" {
-				l.WithField("port", sPort).Warn("ignoring port specification for ICMP firewall rule")
-			}
-		default:
-			return fmt.Errorf("%s rule #%v; proto was not understood; `%s`", table, i, r.Proto)
-		}
+		startPort, endPort, err := parsePort(sPort)
 		if err != nil {
 			return fmt.Errorf("%s rule #%v; %s %s", table, i, errPort, err)
 		}
 
-		if r.Cidr != "" && r.Cidr != "any" {
-			_, err = netip.ParsePrefix(r.Cidr)
+		var proto uint8
+		switch r.Proto {
+		case "any":
+			proto = firewall.ProtoAny
+		case "tcp":
+			proto = firewall.ProtoTCP
+		case "udp":
+			proto = firewall.ProtoUDP
+		case "icmp":
+			proto = firewall.ProtoICMP
+		default:
+			return fmt.Errorf("%s rule #%v; proto was not understood; `%s`", table, i, r.Proto)
+		}
+
+		var cidr netip.Prefix
+		if r.Cidr != "" {
+			cidr, err = netip.ParsePrefix(r.Cidr)
 			if err != nil {
 				return fmt.Errorf("%s rule #%v; cidr did not parse; %s", table, i, err)
 			}
 		}
 
-		if r.LocalCidr != "" && r.LocalCidr != "any" {
-			_, err = netip.ParsePrefix(r.LocalCidr)
+		var localCidr netip.Prefix
+		if r.LocalCidr != "" {
+			localCidr, err = netip.ParsePrefix(r.LocalCidr)
 			if err != nil {
 				return fmt.Errorf("%s rule #%v; local_cidr did not parse; %s", table, i, err)
 			}
 		}
 
-		if warning := r.sanity(); warning != nil {
-			l.Warnf("%s rule #%v; %s", table, i, warning)
-		}
-
-		err = fw.AddRule(inbound, proto, startPort, endPort, r.Groups, r.Host, r.Cidr, r.LocalCidr, r.CAName, r.CASha)
+		err = fw.AddRule(inbound, proto, startPort, endPort, groups, r.Host, cidr, localCidr, r.CAName, r.CASha)
 		if err != nil {
 			return fmt.Errorf("%s rule #%v; `%s`", table, i, err)
 		}
@@ -417,57 +417,35 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw
 	return nil
 }
 
-var ErrUnknownNetworkType = errors.New("unknown network type")
-var ErrPeerRejected = errors.New("remote address is not within a network that we handle")
-var ErrInvalidRemoteIP = errors.New("remote address is not in remote certificate networks")
-var ErrInvalidLocalIP = errors.New("local address is not in list of handled local addresses")
+var ErrInvalidRemoteIP = errors.New("remote IP is not in remote certificate subnets")
+var ErrInvalidLocalIP = errors.New("local IP is not in list of handled local IPs")
 var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
 
 // Drop returns an error if the packet should be dropped, explaining why. It
 // returns nil if the packet should not be dropped.
-func (f *Firewall) Drop(fp firewall.Packet, ctx firewall.PacketContext, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error {
+func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache *firewall.ConntrackCache) error {
 	// Check if we spoke to this tuple, if we did then allow this packet
 	if f.inConns(fp, h, caPool, localCache) {
 		return nil
 	}
 
-	peerCert := h.ConnectionState.peerCert
-
-	// Make sure remote address matches nebula certificate, and determine how to treat it
-	if h.networks == nil {
-		// Simple case: Certificate has one address and no unsafe networks
-		if h.vpnAddrs[0] != fp.RemoteAddr {
+	// Make sure remote address matches nebula certificate
+	if h.networks != nil {
+		if !h.networks.Contains(fp.RemoteAddr) {
 			f.metrics(incoming).droppedRemoteAddr.Inc(1)
-			f.reportDrop(incoming, events.DropInvalidRemoteIP, fp, ctx, peerCert)
 			return ErrInvalidRemoteIP
 		}
 	} else {
-		nwType, ok := h.networks.Lookup(fp.RemoteAddr)
-		if !ok {
+		// Simple case: Certificate has one address and no unsafe networks
+		if h.vpnAddrs[0] != fp.RemoteAddr {
 			f.metrics(incoming).droppedRemoteAddr.Inc(1)
-			f.reportDrop(incoming, events.DropInvalidRemoteIP, fp, ctx, peerCert)
 			return ErrInvalidRemoteIP
 		}
-		switch nwType {
-		case NetworkTypeVPN:
-			break // nothing special
-		case NetworkTypeVPNPeer:
-			f.metrics(incoming).droppedRemoteAddr.Inc(1)
-			f.reportDrop(incoming, events.DropPeerRejected, fp, ctx, peerCert)
-			return ErrPeerRejected // reject for now, one day this may have different FW rules
-		case NetworkTypeUnsafe:
-			break // nothing special, one day this may have different FW rules
-		default:
-			f.metrics(incoming).droppedRemoteAddr.Inc(1)
-			f.reportDrop(incoming, events.DropUnknownNetwork, fp, ctx, peerCert)
-			return ErrUnknownNetworkType //should never happen
-		}
 	}
 
 	// Make sure we are supposed to be handling this local ip address
 	if !f.routableNetworks.Contains(fp.LocalAddr) {
 		f.metrics(incoming).droppedLocalAddr.Inc(1)
-		f.reportDrop(incoming, events.DropInvalidLocalIP, fp, ctx, peerCert)
 		return ErrInvalidLocalIP
 	}
 
@@ -477,14 +455,13 @@ func (f *Firewall) Drop(fp firewall.Packet, ctx firewall.PacketContext, incoming
 	}
 
 	// We now know which firewall table to check against
-	if !table.match(fp, incoming, peerCert, caPool) {
+	if !table.match(fp, incoming, h.ConnectionState.peerCert, caPool) {
 		f.metrics(incoming).droppedNoRule.Inc(1)
-		f.reportDrop(incoming, events.DropNoMatchingRule, fp, ctx, peerCert)
 		return ErrNoMatchingRule
 	}
 
 	// We always want to conntrack since it is a faster operation
-	f.addConn(fp, ctx, incoming, peerCert)
+	f.addConn(fp, incoming)
 
 	return nil
 }
@@ -497,65 +474,12 @@ func (f *Firewall) metrics(incoming bool) firewallMetrics {
 	}
 }
 
-// Destroy cleans up any known cyclical references so the object can be freed by GC. This should be called if a new
+// Destroy cleans up any known cyclical references so the object can be free'd my GC. This should be called if a new
 // firewall object is created
 func (f *Firewall) Destroy() {
 	//TODO: clean references if/when needed
 }
 
-func (f *Firewall) reportDrop(incoming bool, reason events.DropReason, fp firewall.Packet, ctx firewall.PacketContext, peerCert *cert.CachedCertificate) {
-	r := f.reporter
-	if r == nil {
-		return
-	}
-	r.ReportDrop(events.DropEvent{
-		Incoming:     incoming,
-		Reason:       reason,
-		Packet:       fp,
-		Context:      ctx,
-		PeerCert:     peerCert,
-		RulesVersion: f.rulesVersion,
-	})
-}
-
-func (f *Firewall) reportFlowCreate(incoming bool, fp firewall.Packet, ctx firewall.PacketContext, peerCert *cert.CachedCertificate) {
-	r := f.reporter
-	if r == nil {
-		return
-	}
-	r.ReportFlowCreate(events.FlowCreateEvent{
-		Incoming:     incoming,
-		Packet:       fp,
-		Context:      ctx,
-		PeerCert:     peerCert,
-		RulesVersion: f.rulesVersion,
-	})
-}
-
-func (f *Firewall) reportFlowEvict(incoming bool, fp firewall.Packet, rulesVersion uint16, expired bool) {
-	r := f.reporter
-	if r == nil {
-		return
-	}
-	r.ReportFlowEvict(events.FlowEvictEvent{
-		Incoming:     incoming,
-		Packet:       fp,
-		RulesVersion: rulesVersion,
-		Expired:      expired,
-	})
-}
-
-func (f *Firewall) reportRulesReload(oldVersion, newVersion uint16) {
-	r := f.reporter
-	if r == nil {
-		return
-	}
-	r.ReportRulesReload(events.RulesReloadEvent{
-		OldVersion: oldVersion,
-		NewVersion: newVersion,
-	})
-}
-
 func (f *Firewall) EmitStats() {
 	conntrack := f.Conntrack
 	conntrack.Lock()
@@ -566,11 +490,9 @@ func (f *Firewall) EmitStats() {
 	metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV()))
 }
 
-func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) bool {
-	if localCache != nil {
-		if _, ok := localCache[fp]; ok {
-			return true
-		}
+func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool, localCache *firewall.ConntrackCache) bool {
+	if localCache != nil && localCache.Has(fp) {
+		return true
 	}
 	conntrack := f.Conntrack
 	conntrack.Lock()
@@ -606,9 +528,7 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 					WithField("oldRulesVersion", c.rulesVersion).
 					Debugln("dropping old conntrack entry, does not match new ruleset")
 			}
-			oldRulesVersion := c.rulesVersion
 			delete(conntrack.Conns, fp)
-			f.reportFlowEvict(c.incoming, fp, oldRulesVersion, false)
 			conntrack.Unlock()
 			return false
 		}
@@ -637,13 +557,13 @@ func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.CAPool,
 	conntrack.Unlock()
 
 	if localCache != nil {
-		localCache[fp] = struct{}{}
+		localCache.Add(fp)
 	}
 
 	return true
 }
 
-func (f *Firewall) addConn(fp firewall.Packet, ctx firewall.PacketContext, incoming bool, peerCert *cert.CachedCertificate) {
+func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
 	var timeout time.Duration
 	c := &conn{}
 
@@ -658,8 +578,7 @@ func (f *Firewall) addConn(fp firewall.Packet, ctx firewall.PacketContext, incom
 
 	conntrack := f.Conntrack
 	conntrack.Lock()
-	_, existing := conntrack.Conns[fp]
-	if !existing {
+	if _, ok := conntrack.Conns[fp]; !ok {
 		conntrack.TimerWheel.Advance(time.Now())
 		conntrack.TimerWheel.Add(fp, timeout)
 	}
@@ -670,13 +589,6 @@ func (f *Firewall) addConn(fp firewall.Packet, ctx firewall.PacketContext, incom
 	c.rulesVersion = f.rulesVersion
 	c.Expires = time.Now().Add(timeout)
 	conntrack.Conns[fp] = c
-
-	// Report only when this represents a genuinely new flow. Fires under the
-	// conntrack lock so FlowCreate/FlowEvict events stay ordered relative to
-	// RulesReloadEvent, which also fires under this lock.
-	if !existing {
-		f.reportFlowCreate(incoming, fp, ctx, peerCert)
-	}
 	conntrack.Unlock()
 }
 
@@ -700,10 +612,7 @@ func (f *Firewall) evict(p firewall.Packet) {
 	}
 
 	// This conn is done
-	rulesVersion := t.rulesVersion
-	incoming := t.incoming
 	delete(conntrack.Conns, p)
-	f.reportFlowEvict(incoming, p, rulesVersion, true)
 }
 
 func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedCertificate, caPool *cert.CAPool) bool {
@@ -729,7 +638,7 @@ func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedC
 	return false
 }
 
-func (fp firewallPort) addRule(f *Firewall, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error {
+func (fp firewallPort) addRule(f *Firewall, startPort int32, endPort int32, groups []string, host string, ip, localIp netip.Prefix, caName string, caSha string) error {
 	if startPort > endPort {
 		return fmt.Errorf("start port was lower than end port")
 	}
@@ -742,7 +651,7 @@ func (fp firewallPort) addRule(f *Firewall, startPort int32, endPort int32, grou
 			}
 		}
 
-		if err := fp[i].addRule(f, groups, host, cidr, localCidr, caName, caSha); err != nil {
+		if err := fp[i].addRule(f, groups, host, ip, localIp, caName, caSha); err != nil {
 			return err
 		}
 	}
@@ -756,13 +665,6 @@ func (fp firewallPort) match(p firewall.Packet, incoming bool, c *cert.CachedCer
 		return false
 	}
 
-	// this branch is here to catch traffic from FirewallTable.Any.match and FirewallTable.ICMP.match
-	if p.Protocol == firewall.ProtoICMP || p.Protocol == firewall.ProtoICMPv6 {
-		// port numbers are re-used for connection tracking of ICMP,
-		// but we don't want to actually filter on them.
-		return fp[firewall.PortAny].match(p, c, caPool)
-	}
-
 	var port int32
 
 	if p.Fragment {
@@ -780,7 +682,7 @@ func (fp firewallPort) match(p firewall.Packet, incoming bool, c *cert.CachedCer
 	return fp[firewall.PortAny].match(p, c, caPool)
 }
 
-func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, cidr, localCidr, caName, caSha string) error {
+func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, ip, localIp netip.Prefix, caName, caSha string) error {
 	fr := func() *FirewallRule {
 		return &FirewallRule{
 			Hosts:  make(map[string]*firewallLocalCIDR),
@@ -794,14 +696,14 @@ func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, cidr, l
 			fc.Any = fr()
 		}
 
-		return fc.Any.addRule(f, groups, host, cidr, localCidr)
+		return fc.Any.addRule(f, groups, host, ip, localIp)
 	}
 
 	if caSha != "" {
 		if _, ok := fc.CAShas[caSha]; !ok {
 			fc.CAShas[caSha] = fr()
 		}
-		err := fc.CAShas[caSha].addRule(f, groups, host, cidr, localCidr)
+		err := fc.CAShas[caSha].addRule(f, groups, host, ip, localIp)
 		if err != nil {
 			return err
 		}
@@ -811,7 +713,7 @@ func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, cidr, l
 		if _, ok := fc.CANames[caName]; !ok {
 			fc.CANames[caName] = fr()
 		}
-		err := fc.CANames[caName].addRule(f, groups, host, cidr, localCidr)
+		err := fc.CANames[caName].addRule(f, groups, host, ip, localIp)
 		if err != nil {
 			return err
 		}
@@ -843,24 +745,24 @@ func (fc *FirewallCA) match(p firewall.Packet, c *cert.CachedCertificate, caPool
 	return fc.CANames[s.Certificate.Name()].match(p, c)
 }
 
-func (fr *FirewallRule) addRule(f *Firewall, groups []string, host, cidr, localCidr string) error {
+func (fr *FirewallRule) addRule(f *Firewall, groups []string, host string, ip, localCIDR netip.Prefix) error {
 	flc := func() *firewallLocalCIDR {
 		return &firewallLocalCIDR{
 			LocalCIDR: new(bart.Lite),
 		}
 	}
 
-	if fr.isAny(groups, host, cidr) {
+	if fr.isAny(groups, host, ip) {
 		if fr.Any == nil {
 			fr.Any = flc()
 		}
 
-		return fr.Any.addRule(f, localCidr)
+		return fr.Any.addRule(f, localCIDR)
 	}
 
 	if len(groups) > 0 {
 		nlc := flc()
-		err := nlc.addRule(f, localCidr)
+		err := nlc.addRule(f, localCIDR)
 		if err != nil {
 			return err
 		}
@@ -876,46 +778,44 @@ func (fr *FirewallRule) addRule(f *Firewall, groups []string, host, cidr, localC
 		if nlc == nil {
 			nlc = flc()
 		}
-		err := nlc.addRule(f, localCidr)
+		err := nlc.addRule(f, localCIDR)
 		if err != nil {
 			return err
 		}
 		fr.Hosts[host] = nlc
 	}
 
-	if cidr != "" {
-		c, err := netip.ParsePrefix(cidr)
-		if err != nil {
-			return err
-		}
-		nlc, _ := fr.CIDR.Get(c)
+	if ip.IsValid() {
+		nlc, _ := fr.CIDR.Get(ip)
 		if nlc == nil {
 			nlc = flc()
 		}
-		err = nlc.addRule(f, localCidr)
+		err := nlc.addRule(f, localCIDR)
 		if err != nil {
 			return err
 		}
-		fr.CIDR.Insert(c, nlc)
+		fr.CIDR.Insert(ip, nlc)
 	}
 
 	return nil
 }
 
-func (fr *FirewallRule) isAny(groups []string, host string, cidr string) bool {
-	if len(groups) == 0 && host == "" && cidr == "" {
+func (fr *FirewallRule) isAny(groups []string, host string, ip netip.Prefix) bool {
+	if len(groups) == 0 && host == "" && !ip.IsValid() {
 		return true
 	}
 
-	if slices.Contains(groups, "any") {
-		return true
+	for _, group := range groups {
+		if group == "any" {
+			return true
+		}
 	}
 
 	if host == "any" {
 		return true
 	}
 
-	if cidr == "any" {
+	if ip.IsValid() && ip.Bits() == 0 {
 		return true
 	}
 
@@ -967,13 +867,8 @@ func (fr *FirewallRule) match(p firewall.Packet, c *cert.CachedCertificate) bool
 	return false
 }
 
-func (flc *firewallLocalCIDR) addRule(f *Firewall, localCidr string) error {
-	if localCidr == "any" {
-		flc.Any = true
-		return nil
-	}
-
-	if localCidr == "" {
+func (flc *firewallLocalCIDR) addRule(f *Firewall, localIp netip.Prefix) error {
+	if !localIp.IsValid() {
 		if !f.hasUnsafeNetworks || f.defaultLocalCIDRAny {
 			flc.Any = true
 			return nil
@@ -984,13 +879,12 @@ func (flc *firewallLocalCIDR) addRule(f *Firewall, localCidr string) error {
 		}
 		return nil
 
+	} else if localIp.Bits() == 0 {
+		flc.Any = true
+		return nil
 	}
 
-	c, err := netip.ParsePrefix(localCidr)
-	if err != nil {
-		return err
-	}
-	flc.LocalCIDR.Insert(c)
+	flc.LocalCIDR.Insert(localIp)
 	return nil
 }
 
@@ -1011,6 +905,7 @@ type rule struct {
 	Code      string
 	Proto     string
 	Host      string
+	Group     string
 	Groups    []string
 	Cidr      string
 	LocalCidr string
@@ -1052,8 +947,7 @@ func convertRule(l *logrus.Logger, p any, table string, i int) (rule, error) {
 		l.Warnf("%s rule #%v; group was an array with a single value, converting to simple value", table, i)
 		m["group"] = v[0]
 	}
-
-	singleGroup := toString("group", m)
+	r.Group = toString("group", m)
 
 	if rg, ok := m["groups"]; ok {
 		switch reflect.TypeOf(rg).Kind() {
@@ -1070,105 +964,52 @@ func convertRule(l *logrus.Logger, p any, table string, i int) (rule, error) {
 		}
 	}
 
-	//flatten group vs groups
-	if singleGroup != "" {
-		// Check if we have both groups and group provided in the rule config
-		if len(r.Groups) > 0 {
-			return r, fmt.Errorf("only one of group or groups should be defined, both provided")
-		}
-		r.Groups = []string{singleGroup}
-	}
-
 	return r, nil
 }
 
-// sanity returns an error if the rule would be evaluated in a way that would short-circuit a configured check on a wildcard value
-// rules are evaluated as "port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND local_cidr"
-func (r *rule) sanity() error {
-	//port, proto, local_cidr are AND, no need to check here
-	//ca_sha and ca_name don't have a wildcard value, no need to check here
-	groupsEmpty := len(r.Groups) == 0
-	hostEmpty := r.Host == ""
-	cidrEmpty := r.Cidr == ""
-
-	if (groupsEmpty && hostEmpty && cidrEmpty) == true {
-		return nil //no content!
-	}
-
-	groupsHasAny := slices.Contains(r.Groups, "any")
-	if groupsHasAny && len(r.Groups) > 1 {
-		return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the other groups specified", r.Groups)
-	}
-
-	if r.Host == "any" {
-		if !groupsEmpty {
-			return fmt.Errorf("groups specified as %s, but host=any will match any host, regardless of groups", r.Groups)
-		}
-
-		if !cidrEmpty {
-			return fmt.Errorf("cidr specified as %s, but host=any will match any host, regardless of cidr", r.Cidr)
-		}
-	}
-
-	if groupsHasAny {
-		if !hostEmpty && r.Host != "any" {
-			return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified host %s", r.Groups, r.Host)
-		}
-		if !cidrEmpty {
-			return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified cidr %s", r.Groups, r.Cidr)
-		}
-	}
-
-	if r.Code != "" {
-		return fmt.Errorf("code specified as [%s]. Support for 'code' will be dropped in a future release, as it has never been functional", r.Code)
-	}
-
-	//todo alert on cidr-any
-
-	return nil
-}
-
-func parsePort(s string) (int32, int32, error) {
-	var err error
-	const notAPort int32 = -2
+func parsePort(s string) (startPort, endPort int32, err error) {
 	if s == "any" {
-		return firewall.PortAny, firewall.PortAny, nil
-	}
-	if s == "fragment" {
-		return firewall.PortFragment, firewall.PortFragment, nil
-	}
-	if !strings.Contains(s, `-`) {
+		startPort = firewall.PortAny
+		endPort = firewall.PortAny
+
+	} else if s == "fragment" {
+		startPort = firewall.PortFragment
+		endPort = firewall.PortFragment
+
+	} else if strings.Contains(s, `-`) {
+		sPorts := strings.SplitN(s, `-`, 2)
+		sPorts[0] = strings.Trim(sPorts[0], " ")
+		sPorts[1] = strings.Trim(sPorts[1], " ")
+
+		if len(sPorts) != 2 || sPorts[0] == "" || sPorts[1] == "" {
+			return 0, 0, fmt.Errorf("appears to be a range but could not be parsed; `%s`", s)
+		}
+
+		rStartPort, err := strconv.Atoi(sPorts[0])
+		if err != nil {
+			return 0, 0, fmt.Errorf("beginning range was not a number; `%s`", sPorts[0])
+		}
+
+		rEndPort, err := strconv.Atoi(sPorts[1])
+		if err != nil {
+			return 0, 0, fmt.Errorf("ending range was not a number; `%s`", sPorts[1])
+		}
+
+		startPort = int32(rStartPort)
+		endPort = int32(rEndPort)
+
+		if startPort == firewall.PortAny {
+			endPort = firewall.PortAny
+		}
+
+	} else {
 		rPort, err := strconv.Atoi(s)
 		if err != nil {
-			return notAPort, notAPort, fmt.Errorf("was not a number; `%s`", s)
+			return 0, 0, fmt.Errorf("was not a number; `%s`", s)
 		}
-		return int32(rPort), int32(rPort), nil
+		startPort = int32(rPort)
+		endPort = startPort
 	}
 
-	sPorts := strings.SplitN(s, `-`, 2)
-	for i := range sPorts {
-		sPorts[i] = strings.Trim(sPorts[i], " ")
-	}
-	if len(sPorts) != 2 || sPorts[0] == "" || sPorts[1] == "" {
-		return notAPort, notAPort, fmt.Errorf("appears to be a range but could not be parsed; `%s`", s)
-	}
-
-	rStartPort, err := strconv.Atoi(sPorts[0])
-	if err != nil {
-		return notAPort, notAPort, fmt.Errorf("beginning range was not a number; `%s`", sPorts[0])
-	}
-
-	rEndPort, err := strconv.Atoi(sPorts[1])
-	if err != nil {
-		return notAPort, notAPort, fmt.Errorf("ending range was not a number; `%s`", sPorts[1])
-	}
-
-	startPort := int32(rStartPort)
-	endPort := int32(rEndPort)
-
-	if startPort == firewall.PortAny {
-		endPort = firewall.PortAny
-	}
-
-	return startPort, endPort, nil
+	return
 }

firewall/cache.go

@@ -1,7 +1,7 @@
 package firewall
 
 import (
-	"context"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -10,55 +10,92 @@ import (
 
 // ConntrackCache is used as a local routine cache to know if a given flow
 // has been seen in the conntrack table.
-type ConntrackCache map[Packet]struct{}
+type ConntrackCache struct {
+	mu      sync.Mutex
+	entries map[Packet]struct{}
+}
+
+func newConntrackCache() *ConntrackCache {
+	return &ConntrackCache{entries: make(map[Packet]struct{})}
+}
+
+func (c *ConntrackCache) Has(p Packet) bool {
+	if c == nil {
+		return false
+	}
+	c.mu.Lock()
+	_, ok := c.entries[p]
+	c.mu.Unlock()
+	return ok
+}
+
+func (c *ConntrackCache) Add(p Packet) {
+	if c == nil {
+		return
+	}
+	c.mu.Lock()
+	c.entries[p] = struct{}{}
+	c.mu.Unlock()
+}
+
+func (c *ConntrackCache) Len() int {
+	if c == nil {
+		return 0
+	}
+	c.mu.Lock()
+	l := len(c.entries)
+	c.mu.Unlock()
+	return l
+}
+
+func (c *ConntrackCache) Reset(capHint int) {
+	if c == nil {
+		return
+	}
+	c.mu.Lock()
+	c.entries = make(map[Packet]struct{}, capHint)
+	c.mu.Unlock()
+}
 
 type ConntrackCacheTicker struct {
 	cacheV    uint64
 	cacheTick atomic.Uint64
 
-	cache ConntrackCache
+	cache *ConntrackCache
 }
 
-func NewConntrackCacheTicker(ctx context.Context, d time.Duration) *ConntrackCacheTicker {
+func NewConntrackCacheTicker(d time.Duration) *ConntrackCacheTicker {
 	if d == 0 {
 		return nil
 	}
 
-	c := &ConntrackCacheTicker{
-		cache: ConntrackCache{},
-	}
+	c := &ConntrackCacheTicker{cache: newConntrackCache()}
 
-	go c.tick(ctx, d)
+	go c.tick(d)
 
 	return c
 }
 
-func (c *ConntrackCacheTicker) tick(ctx context.Context, d time.Duration) {
-	t := time.NewTicker(d)
-	defer t.Stop()
+func (c *ConntrackCacheTicker) tick(d time.Duration) {
 	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-t.C:
-			c.cacheTick.Add(1)
-		}
+		time.Sleep(d)
+		c.cacheTick.Add(1)
 	}
 }
 
 // Get checks if the cache ticker has moved to the next version before returning
 // the map. If it has moved, we reset the map.
-func (c *ConntrackCacheTicker) Get(l *logrus.Logger) ConntrackCache {
+func (c *ConntrackCacheTicker) Get(l *logrus.Logger) *ConntrackCache {
 	if c == nil {
 		return nil
 	}
 	if tick := c.cacheTick.Load(); tick != c.cacheV {
 		c.cacheV = tick
-		if ll := len(c.cache); ll > 0 {
+		if ll := c.cache.Len(); ll > 0 {
 			if l.Level == logrus.DebugLevel {
 				l.WithField("len", ll).Debug("resetting conntrack cache")
 			}
-			c.cache = make(ConntrackCache, ll)
+			c.cache.Reset(ll)
 		}
 	}
 

firewall/events/events.go

@@ -1,102 +0,0 @@
-// Package events defines the opt-in firewall event reporting interface.
-//
-// Nebula emits raw packet-level events (drops, flow creations, flow evictions,
-// rule reloads) and does no aggregation, counting, batching, rule-description,
-// transport, or timestamping. Embedders correlate events back to yaml rules
-// out of band and capture whatever clock they need themselves. All Report*
-// methods are invoked while nebula holds internal locks and must be
-// non-blocking.
-//
-// Events are passed to Report* methods by value. Implementations must not
-// take the address of a received event: doing so forces Go's escape
-// analysis to move the event to the heap and costs one allocation per call.
-// To forward an event, either copy its fields into the reporter's own
-// pooled record or send it through a value-typed channel (chan DropEvent,
-// not chan *DropEvent).
-package events
-
-import (
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/firewall"
-)
-
-type DropReason uint8
-
-const (
-	DropInvalidLocalIP DropReason = iota
-	DropInvalidRemoteIP
-	DropPeerRejected
-	DropUnknownNetwork
-	DropNoMatchingRule
-)
-
-func (r DropReason) String() string {
-	switch r {
-	case DropInvalidLocalIP:
-		return "invalid_local_ip"
-	case DropInvalidRemoteIP:
-		return "invalid_remote_ip"
-	case DropPeerRejected:
-		return "peer_rejected"
-	case DropUnknownNetwork:
-		return "unknown_network"
-	case DropNoMatchingRule:
-		return "no_matching_rule"
-	default:
-		return "unknown"
-	}
-}
-
-// DropEvent is emitted for every packet that fails the firewall check. Drops
-// are not aggregated; every drop produces one event.
-type DropEvent struct {
-	Incoming     bool
-	Reason       DropReason
-	Packet       firewall.Packet
-	Context      firewall.PacketContext
-	PeerCert     *cert.CachedCertificate
-	RulesVersion uint16
-}
-
-// FlowCreateEvent is emitted when a packet is allowed and a new conntrack
-// entry is created. Subsequent packets in the same flow do not re-emit.
-type FlowCreateEvent struct {
-	Incoming     bool
-	Packet       firewall.Packet
-	Context      firewall.PacketContext
-	PeerCert     *cert.CachedCertificate
-	RulesVersion uint16
-}
-
-// FlowEvictEvent is emitted when a conntrack entry is removed. Context is
-// not carried: timer-wheel eviction has no packet in hand, and reload
-// revalidation evicts the OLD flow rather than the triggering packet.
-// RulesVersion is the version under which the flow was originally allowed,
-// which may differ from the current firewall version.
-type FlowEvictEvent struct {
-	Incoming     bool
-	Packet       firewall.Packet
-	RulesVersion uint16
-	// Expired is true when eviction was due to conntrack timeout; false when
-	// the entry was removed because it failed re-validation after a reload.
-	Expired bool
-}
-
-// RulesReloadEvent is emitted once after each successful firewall reload.
-// Reporters that bucket state by RulesVersion should close the old bucket
-// and open a new one on receipt.
-type RulesReloadEvent struct {
-	OldVersion uint16
-	NewVersion uint16
-}
-
-// Reporter is the embedder-supplied sink for firewall events. Implementations
-// that want a timestamp should call time.Now() themselves at the top of the
-// method; nebula does not provide one. See the package doc for the
-// do-not-take-address rule.
-type Reporter interface {
-	ReportDrop(DropEvent)
-	ReportFlowCreate(FlowCreateEvent)
-	ReportFlowEvict(FlowEvictEvent)
-	ReportRulesReload(RulesReloadEvent)
-}

firewall/packet.go

@@ -22,36 +22,12 @@ const (
 type Packet struct {
 	LocalAddr  netip.Addr
 	RemoteAddr netip.Addr
-	// LocalPort is the destination port for incoming traffic, or the source port for outgoing. Zero for ICMP.
-	LocalPort uint16
-	// RemotePort is the source port for incoming traffic, or the destination port for outgoing.
-	// For ICMP, it's the "identifier". This is only used for connection tracking, actual firewall rules will not filter on ICMP identifier
+	LocalPort  uint16
 	RemotePort uint16
 	Protocol   uint8
 	Fragment   bool
 }
 
-// PacketContext carries additional parsed details about a packet that are
-// useful for event reporting but deliberately kept out of Packet so Packet
-// can keep being used as a conntrack map key. Populated alongside Packet by
-// newPacket.
-//
-// Fields are interpreted based on Packet.Protocol:
-//   - ProtoTCP: TCPFlags is meaningful; ICMPType / ICMPCode are zero
-//   - ProtoICMP, ProtoICMPv6: ICMPType / ICMPCode are meaningful; TCPFlags is zero
-//   - ProtoUDP and others: only Length is meaningful
-type PacketContext struct {
-	// Length is the total IP packet length in bytes, including headers.
-	Length uint16
-	// TCPFlags is the flag byte from the TCP header (bits for FIN, SYN, RST,
-	// PSH, ACK, URG, ECE, CWR).
-	TCPFlags uint8
-	// ICMPType is the type field of the ICMP / ICMPv6 header.
-	ICMPType uint8
-	// ICMPCode is the code field of the ICMP / ICMPv6 header.
-	ICMPCode uint8
-}
-
 func (fp *Packet) Copy() *Packet {
 	return &Packet{
 		LocalAddr:  fp.LocalAddr,
@@ -70,8 +46,6 @@ func (fp Packet) MarshalJSON() ([]byte, error) {
 		proto = "tcp"
 	case ProtoICMP:
 		proto = "icmp"
-	case ProtoICMPv6:
-		proto = "icmpv6"
 	case ProtoUDP:
 		proto = "udp"
 	default:

firewall_events_test.go

@@ -1,731 +0,0 @@
-package nebula
-
-import (
-	"net"
-	"net/netip"
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/gaissmai/bart"
-	"github.com/google/gopacket"
-	"github.com/google/gopacket/layers"
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/firewall"
-	"github.com/slackhq/nebula/firewall/events"
-	"github.com/slackhq/nebula/test"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// recordingReporter captures every event fired against it. Its methods take
-// the conntrack lock implicitly (via the firewall code path that invokes
-// them), so we synchronize accumulator mutations with a small mutex to keep
-// the race detector happy across goroutines in case a test introduces any.
-type recordingReporter struct {
-	mu      sync.Mutex
-	drops   []recordedDrop
-	creates []recordedCreate
-	evicts  []recordedEvict
-	reloads []recordedReload
-}
-
-type recordedDrop struct {
-	incoming     bool
-	reason       events.DropReason
-	remote       netip.Addr
-	local        netip.Addr
-	peerName     string
-	rulesVersion uint16
-	ctx          firewall.PacketContext
-}
-
-type recordedCreate struct {
-	incoming     bool
-	remote       netip.Addr
-	local        netip.Addr
-	peerName     string
-	rulesVersion uint16
-	ctx          firewall.PacketContext
-}
-
-type recordedEvict struct {
-	incoming     bool
-	remote       netip.Addr
-	local        netip.Addr
-	rulesVersion uint16
-	expired      bool
-}
-
-type recordedReload struct {
-	oldVersion uint16
-	newVersion uint16
-}
-
-func (r *recordingReporter) ReportDrop(e events.DropEvent) {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-	name := ""
-	if e.PeerCert != nil && e.PeerCert.Certificate != nil {
-		name = e.PeerCert.Certificate.Name()
-	}
-	r.drops = append(r.drops, recordedDrop{
-		incoming:     e.Incoming,
-		reason:       e.Reason,
-		remote:       e.Packet.RemoteAddr,
-		local:        e.Packet.LocalAddr,
-		peerName:     name,
-		rulesVersion: e.RulesVersion,
-		ctx:          e.Context,
-	})
-}
-
-func (r *recordingReporter) ReportFlowCreate(e events.FlowCreateEvent) {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-	name := ""
-	if e.PeerCert != nil && e.PeerCert.Certificate != nil {
-		name = e.PeerCert.Certificate.Name()
-	}
-	r.creates = append(r.creates, recordedCreate{
-		incoming:     e.Incoming,
-		remote:       e.Packet.RemoteAddr,
-		local:        e.Packet.LocalAddr,
-		peerName:     name,
-		rulesVersion: e.RulesVersion,
-		ctx:          e.Context,
-	})
-}
-
-func (r *recordingReporter) ReportFlowEvict(e events.FlowEvictEvent) {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-	r.evicts = append(r.evicts, recordedEvict{
-		incoming:     e.Incoming,
-		remote:       e.Packet.RemoteAddr,
-		local:        e.Packet.LocalAddr,
-		rulesVersion: e.RulesVersion,
-		expired:      e.Expired,
-	})
-}
-
-func (r *recordingReporter) ReportRulesReload(e events.RulesReloadEvent) {
-	r.mu.Lock()
-	defer r.mu.Unlock()
-	r.reloads = append(r.reloads, recordedReload{
-		oldVersion: e.OldVersion,
-		newVersion: e.NewVersion,
-	})
-}
-
-// eventFixture builds a Firewall wired to a Control plus a packet/hostinfo
-// pair that a test can reuse. By default the ruleset allows the packet;
-// callers mutate fw / p / h as needed before invoking Drop.
-type eventFixture struct {
-	ctl *Control
-	fw  *Firewall
-	p   firewall.Packet
-	h   *HostInfo
-	cp  *cert.CAPool
-}
-
-func newEventFixture(t *testing.T) *eventFixture {
-	t.Helper()
-	l := test.NewLogger()
-
-	// myVpnNetworksTable covers our single peer address so buildNetworks takes
-	// the "simple case" path (h.networks stays nil); tests that want a populated
-	// BART table overwrite h.networks directly.
-	vpnNetworks := new(bart.Lite)
-	vpnNetworks.Insert(netip.MustParsePrefix("1.2.3.0/24"))
-
-	// Use the same cert for "peer" and "local" endpoints, matching the
-	// TestFirewall_Drop fixture style: LocalAddr == RemoteAddr == peer vpn addr.
-	c := &dummyCert{
-		name:     "host1",
-		networks: []netip.Prefix{netip.MustParsePrefix("1.2.3.4/24")},
-		groups:   []string{"default-group"},
-		issuer:   "signer-shasum",
-	}
-	h := &HostInfo{
-		ConnectionState: &ConnectionState{
-			peerCert: &cert.CachedCertificate{
-				Certificate:    c,
-				InvertedGroups: map[string]struct{}{"default-group": {}},
-			},
-		},
-		vpnAddrs: []netip.Addr{netip.MustParseAddr("1.2.3.4")},
-	}
-	h.buildNetworks(vpnNetworks, c)
-
-	fw := NewFirewall(l, time.Minute, time.Minute, time.Minute, c)
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
-	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
-
-	ctl := &Control{
-		f: &Interface{firewall: fw},
-		l: l,
-	}
-
-	return &eventFixture{
-		ctl: ctl,
-		fw:  fw,
-		p: firewall.Packet{
-			LocalAddr:  netip.MustParseAddr("1.2.3.4"),
-			RemoteAddr: netip.MustParseAddr("1.2.3.4"),
-			LocalPort:  10,
-			RemotePort: 90,
-			Protocol:   firewall.ProtoUDP,
-		},
-		h:  h,
-		cp: cert.NewCAPool(),
-	}
-}
-
-// firewall() returns the currently-installed firewall. Needed because
-// SetFirewallEventReporter replaces it via shallow-copy swap.
-func (f *eventFixture) firewall() *Firewall {
-	return f.ctl.f.firewall
-}
-
-func TestEvents_ReportDrop_InvalidRemoteIP(t *testing.T) {
-	f := newEventFixture(t)
-	r := &recordingReporter{}
-	f.ctl.SetFirewallEventReporter(r)
-
-	// Packet to an address not in the cert's networks.
-	f.p.RemoteAddr = netip.MustParseAddr("9.9.9.9")
-	assert.Equal(t, ErrInvalidRemoteIP, f.firewall().Drop(f.p, firewall.PacketContext{}, false, f.h, f.cp, nil))
-
-	require.Len(t, r.drops, 1)
-	assert.Equal(t, events.DropInvalidRemoteIP, r.drops[0].reason)
-	assert.False(t, r.drops[0].incoming)
-	assert.Equal(t, "host1", r.drops[0].peerName)
-	assert.Empty(t, r.creates)
-	assert.Empty(t, r.evicts)
-}
-
-func TestEvents_ReportDrop_InvalidLocalIP(t *testing.T) {
-	f := newEventFixture(t)
-	r := &recordingReporter{}
-	f.ctl.SetFirewallEventReporter(r)
-
-	// LocalAddr outside our routable networks.
-	f.p.LocalAddr = netip.MustParseAddr("9.9.9.9")
-	assert.Equal(t, ErrInvalidLocalIP, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
-
-	require.Len(t, r.drops, 1)
-	assert.Equal(t, events.DropInvalidLocalIP, r.drops[0].reason)
-	assert.True(t, r.drops[0].incoming)
-}
-
-func TestEvents_ReportDrop_NoMatchingRule(t *testing.T) {
-	f := newEventFixture(t)
-	// Reset to a firewall with no matching rule.
-	l := test.NewLogger()
-	fw := NewFirewall(l, time.Minute, time.Minute, time.Minute, f.h.ConnectionState.peerCert.Certificate)
-	// Rule that won't match (group not in peer's groups).
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", ""))
-	require.NoError(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"nope"}, "", "", "", "", ""))
-	f.ctl.f.firewall = fw
-
-	r := &recordingReporter{}
-	f.ctl.SetFirewallEventReporter(r)
-
-	assert.Equal(t, ErrNoMatchingRule, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
-	require.Len(t, r.drops, 1)
-	assert.Equal(t, events.DropNoMatchingRule, r.drops[0].reason)
-}
-
-func TestEvents_ReportDrop_PeerRejected(t *testing.T) {
-	f := newEventFixture(t)
-	// Re-classify the remote as VPNPeer so it triggers DropPeerRejected.
-	f.h.networks = new(bart.Table[NetworkType])
-	f.h.networks.Insert(netip.MustParsePrefix("1.2.3.0/24"), NetworkTypeVPNPeer)
-
-	r := &recordingReporter{}
-	f.ctl.SetFirewallEventReporter(r)
-
-	assert.Equal(t, ErrPeerRejected, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
-	require.Len(t, r.drops, 1)
-	assert.Equal(t, events.DropPeerRejected, r.drops[0].reason)
-}
-
-func TestEvents_ReportDrop_UnknownNetwork(t *testing.T) {
-	f := newEventFixture(t)
-	// Insert an unrecognized NetworkType value to hit the default branch.
-	f.h.networks = new(bart.Table[NetworkType])
-	f.h.networks.Insert(netip.MustParsePrefix("1.2.3.0/24"), NetworkTypeUnknown)
-
-	r := &recordingReporter{}
-	f.ctl.SetFirewallEventReporter(r)
-
-	assert.Equal(t, ErrUnknownNetworkType, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
-	require.Len(t, r.drops, 1)
-	assert.Equal(t, events.DropUnknownNetwork, r.drops[0].reason)
-}
-
-func TestEvents_ReportFlowCreate_OnceOnly(t *testing.T) {
-	f := newEventFixture(t)
-	r := &recordingReporter{}
-	f.ctl.SetFirewallEventReporter(r)
-
-	// First allowed packet creates the conntrack entry.
-	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
-	// Second matching packet on the same tuple is short-circuited by conntrack
-	// and must not fire another FlowCreate.
-	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
-
-	require.Len(t, r.creates, 1)
-	assert.True(t, r.creates[0].incoming)
-	assert.Equal(t, f.p.RemoteAddr, r.creates[0].remote)
-	assert.Empty(t, r.drops)
-}
-
-func TestEvents_ReportFlowEvict_OnReloadPurge(t *testing.T) {
-	f := newEventFixture(t)
-	r := &recordingReporter{}
-	f.ctl.SetFirewallEventReporter(r)
-
-	// Create a flow under the current rules.
-	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
-	require.Len(t, r.creates, 1)
-
-	// Simulate a reload that produces rules the existing flow no longer
-	// matches. Bump rulesVersion and replace InRules with an empty table so
-	// revalidation fails.
-	fw := f.firewall()
-	fw.Conntrack.Lock()
-	fw.rulesVersion++
-	fw.InRules = newFirewallTable()
-	fw.Conntrack.Unlock()
-
-	// Next packet triggers re-validation, which fails and evicts the entry.
-	err := fw.Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
-	assert.Equal(t, ErrNoMatchingRule, err)
-
-	require.Len(t, r.evicts, 1)
-	assert.False(t, r.evicts[0].expired, "evict from reload purge is not expiration")
-	assert.True(t, r.evicts[0].incoming)
-}
-
-func TestEvents_ReportFlowEvict_OnTimeout(t *testing.T) {
-	f := newEventFixture(t)
-	r := &recordingReporter{}
-	f.ctl.SetFirewallEventReporter(r)
-
-	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
-	require.Len(t, r.creates, 1)
-
-	// Force expiration by rewinding the entry's deadline.
-	fw := f.firewall()
-	fw.Conntrack.Lock()
-	c := fw.Conntrack.Conns[f.p]
-	require.NotNil(t, c)
-	c.Expires = time.Now().Add(-time.Hour)
-	fw.evict(f.p)
-	fw.Conntrack.Unlock()
-
-	require.Len(t, r.evicts, 1)
-	assert.True(t, r.evicts[0].expired)
-}
-
-func TestEvents_SetNil_Clears(t *testing.T) {
-	f := newEventFixture(t)
-	r := &recordingReporter{}
-	f.ctl.SetFirewallEventReporter(r)
-	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
-	require.Len(t, r.creates, 1)
-
-	f.ctl.SetFirewallEventReporter(nil)
-	resetConntrack(f.firewall())
-	require.NoError(t, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
-	// No second create should be recorded.
-	assert.Len(t, r.creates, 1)
-}
-
-func TestEvents_ReporterSurvivesSwap(t *testing.T) {
-	f := newEventFixture(t)
-	r := &recordingReporter{}
-	f.ctl.SetFirewallEventReporter(r)
-
-	// Simulate a reload by swapping in a fresh Firewall that carries the
-	// reporter forward. Mirrors what reloadFirewall does with the shared
-	// conntrack pointer.
-	l := test.NewLogger()
-	oldFw := f.firewall()
-	newFw := NewFirewall(l, time.Minute, time.Minute, time.Minute, f.h.ConnectionState.peerCert.Certificate)
-	require.NoError(t, newFw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
-	require.NoError(t, newFw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
-	newFw.Conntrack = oldFw.Conntrack
-	newFw.rulesVersion = oldFw.rulesVersion + 1
-	newFw.reporter = oldFw.reporter
-	f.ctl.f.firewall = newFw
-	newFw.reportRulesReload(oldFw.rulesVersion, newFw.rulesVersion)
-
-	require.Len(t, r.reloads, 1)
-	assert.Equal(t, oldFw.rulesVersion, r.reloads[0].oldVersion)
-	assert.Equal(t, newFw.rulesVersion, r.reloads[0].newVersion)
-
-	// Events on the new firewall should still reach the same reporter.
-	require.NoError(t, newFw.Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
-	require.Len(t, r.creates, 1)
-	assert.Equal(t, newFw.rulesVersion, r.creates[0].rulesVersion)
-}
-
-func TestEvents_InstallDoesNotMutateOldFirewall(t *testing.T) {
-	f := newEventFixture(t)
-	before := f.firewall()
-
-	r := &recordingReporter{}
-	f.ctl.SetFirewallEventReporter(r)
-
-	after := f.firewall()
-	assert.NotSame(t, before, after, "SetFirewallEventReporter must replace the Firewall pointer")
-	assert.Nil(t, before.reporter, "the pre-install Firewall must remain untouched")
-	assert.NotNil(t, after.reporter)
-}
-
-// --- PacketContext parse tests --------------------------------------------
-
-func mustSerialize(t *testing.T, lrs ...gopacket.SerializableLayer) []byte {
-	t.Helper()
-	buf := gopacket.NewSerializeBuffer()
-	opt := gopacket.SerializeOptions{ComputeChecksums: false, FixLengths: true}
-	require.NoError(t, gopacket.SerializeLayers(buf, opt, lrs...))
-	return buf.Bytes()
-}
-
-func TestPacketContext_IPv4_TCPFlags(t *testing.T) {
-	ip := &layers.IPv4{
-		Version: 4, TTL: 64, Protocol: layers.IPProtocolTCP,
-		SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
-	}
-	tcp := &layers.TCP{SrcPort: 1234, DstPort: 80, SYN: true, ACK: true}
-	require.NoError(t, tcp.SetNetworkLayerForChecksum(ip))
-	data := mustSerialize(t, ip, tcp, gopacket.Payload([]byte("hello")))
-
-	var fp firewall.Packet
-	var ctx firewall.PacketContext
-	require.NoError(t, newPacket(data, true, &fp, &ctx))
-
-	assert.Equal(t, uint8(firewall.ProtoTCP), fp.Protocol)
-	// SYN (0x02) + ACK (0x10) = 0x12
-	assert.Equal(t, uint8(0x12), ctx.TCPFlags)
-	assert.Equal(t, uint16(len(data)), ctx.Length)
-	assert.Equal(t, uint8(0), ctx.ICMPType)
-	assert.Equal(t, uint8(0), ctx.ICMPCode)
-}
-
-func TestPacketContext_IPv4_ICMPTypeCode(t *testing.T) {
-	ip := &layers.IPv4{
-		Version: 4, TTL: 64, Protocol: layers.IPProtocolICMPv4,
-		SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
-	}
-	// Destination Unreachable, code 3 (port unreachable)
-	icmp := &layers.ICMPv4{
-		TypeCode: layers.CreateICMPv4TypeCode(layers.ICMPv4TypeDestinationUnreachable, layers.ICMPv4CodePort),
-	}
-	data := mustSerialize(t, ip, icmp, gopacket.Payload([]byte{0, 0, 0, 0}))
-
-	var fp firewall.Packet
-	var ctx firewall.PacketContext
-	require.NoError(t, newPacket(data, true, &fp, &ctx))
-
-	assert.Equal(t, uint8(firewall.ProtoICMP), fp.Protocol)
-	assert.Equal(t, uint8(layers.ICMPv4TypeDestinationUnreachable), ctx.ICMPType)
-	assert.Equal(t, uint8(layers.ICMPv4CodePort), ctx.ICMPCode)
-	assert.Equal(t, uint16(len(data)), ctx.Length)
-	assert.Equal(t, uint8(0), ctx.TCPFlags)
-}
-
-func TestPacketContext_IPv4_UDPLengthOnly(t *testing.T) {
-	ip := &layers.IPv4{
-		Version: 4, TTL: 64, Protocol: layers.IPProtocolUDP,
-		SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
-	}
-	udp := &layers.UDP{SrcPort: 1234, DstPort: 53}
-	require.NoError(t, udp.SetNetworkLayerForChecksum(ip))
-	data := mustSerialize(t, ip, udp, gopacket.Payload([]byte("query")))
-
-	var fp firewall.Packet
-	var ctx firewall.PacketContext
-	require.NoError(t, newPacket(data, true, &fp, &ctx))
-
-	assert.Equal(t, uint8(firewall.ProtoUDP), fp.Protocol)
-	assert.Equal(t, uint16(len(data)), ctx.Length)
-	assert.Zero(t, ctx.TCPFlags)
-	assert.Zero(t, ctx.ICMPType)
-	assert.Zero(t, ctx.ICMPCode)
-}
-
-func TestPacketContext_IPv6_TCPFlags(t *testing.T) {
-	ip := &layers.IPv6{
-		Version: 6, HopLimit: 64, NextHeader: layers.IPProtocolTCP,
-		SrcIP: net.ParseIP("fd00::1"), DstIP: net.ParseIP("fd00::2"),
-	}
-	tcp := &layers.TCP{SrcPort: 1234, DstPort: 443, FIN: true, ACK: true}
-	require.NoError(t, tcp.SetNetworkLayerForChecksum(ip))
-	data := mustSerialize(t, ip, tcp, gopacket.Payload([]byte("bye")))
-
-	var fp firewall.Packet
-	var ctx firewall.PacketContext
-	require.NoError(t, newPacket(data, true, &fp, &ctx))
-
-	assert.Equal(t, uint8(firewall.ProtoTCP), fp.Protocol)
-	// FIN (0x01) + ACK (0x10) = 0x11
-	assert.Equal(t, uint8(0x11), ctx.TCPFlags)
-	assert.Equal(t, uint16(len(data)), ctx.Length)
-}
-
-func TestPacketContext_IPv6_ICMPv6TypeCode(t *testing.T) {
-	ip := &layers.IPv6{
-		Version: 6, HopLimit: 64, NextHeader: layers.IPProtocolICMPv6,
-		SrcIP: net.ParseIP("fd00::1"), DstIP: net.ParseIP("fd00::2"),
-	}
-	icmp := &layers.ICMPv6{
-		TypeCode: layers.CreateICMPv6TypeCode(layers.ICMPv6TypeDestinationUnreachable, layers.ICMPv6CodePortUnreachable),
-	}
-	require.NoError(t, icmp.SetNetworkLayerForChecksum(ip))
-	data := mustSerialize(t, ip, icmp, gopacket.Payload([]byte{0, 0, 0, 0, 0, 0, 0, 0}))
-
-	var fp firewall.Packet
-	var ctx firewall.PacketContext
-	require.NoError(t, newPacket(data, true, &fp, &ctx))
-
-	assert.Equal(t, uint8(firewall.ProtoICMPv6), fp.Protocol)
-	assert.Equal(t, uint8(layers.ICMPv6TypeDestinationUnreachable), ctx.ICMPType)
-	assert.Equal(t, uint8(layers.ICMPv6CodePortUnreachable), ctx.ICMPCode)
-	assert.Equal(t, uint16(len(data)), ctx.Length)
-}
-
-// TestPacketContext_NilOK confirms a nil context pointer is accepted by
-// newPacket (the hot path may elect not to pass one).
-func TestPacketContext_NilOK(t *testing.T) {
-	ip := &layers.IPv4{
-		Version: 4, TTL: 64, Protocol: layers.IPProtocolUDP,
-		SrcIP: net.IPv4(10, 0, 0, 1), DstIP: net.IPv4(10, 0, 0, 2),
-	}
-	udp := &layers.UDP{SrcPort: 1, DstPort: 2}
-	require.NoError(t, udp.SetNetworkLayerForChecksum(ip))
-	data := mustSerialize(t, ip, udp)
-
-	var fp firewall.Packet
-	require.NoError(t, newPacket(data, true, &fp, nil))
-}
-
-// TestPacketContext_FlowCreateCarriesContext exercises the full Drop -> addConn
-// -> ReportFlowCreate path with a realistic TCP packet and confirms the
-// context makes it into the reporter.
-func TestPacketContext_FlowCreateCarriesContext(t *testing.T) {
-	f := newEventFixture(t)
-	r := &recordingReporter{}
-	f.ctl.SetFirewallEventReporter(r)
-
-	// Hand-construct a matching TCP packet.
-	ctx := firewall.PacketContext{Length: 1500, TCPFlags: 0x12}
-	p := f.p
-	p.Protocol = firewall.ProtoTCP
-	require.NoError(t, f.firewall().Drop(p, ctx, true, f.h, f.cp, nil))
-
-	require.Len(t, r.creates, 1)
-	assert.Equal(t, uint16(1500), r.creates[0].ctx.Length)
-	assert.Equal(t, uint8(0x12), r.creates[0].ctx.TCPFlags)
-}
-
-// --- benchmarks ------------------------------------------------------------
-
-// noopReporter is the cheapest possible reporter. Methods discard the event.
-type noopReporter struct{}
-
-func (noopReporter) ReportDrop(events.DropEvent)             {}
-func (noopReporter) ReportFlowCreate(events.FlowCreateEvent) {}
-func (noopReporter) ReportFlowEvict(events.FlowEvictEvent)   {}
-func (noopReporter) ReportRulesReload(events.RulesReloadEvent) {
-}
-
-// bufferedReporter demonstrates a realistic zero-alloc reporter: each event
-// is forwarded to a value-typed channel. The channel send is a memcpy into
-// the channel's pre-allocated ring buffer -- no heap traffic. A background
-// goroutine would drain these; the bench skips draining to keep the report
-// path pure.
-type bufferedReporter struct {
-	drops  chan events.DropEvent
-	flows  chan events.FlowCreateEvent
-	evicts chan events.FlowEvictEvent
-}
-
-func newBufferedReporter(cap int) *bufferedReporter {
-	return &bufferedReporter{
-		drops:  make(chan events.DropEvent, cap),
-		flows:  make(chan events.FlowCreateEvent, cap),
-		evicts: make(chan events.FlowEvictEvent, cap),
-	}
-}
-
-func (r *bufferedReporter) ReportDrop(e events.DropEvent) {
-	select {
-	case r.drops <- e:
-	default:
-	}
-}
-
-func (r *bufferedReporter) ReportFlowCreate(e events.FlowCreateEvent) {
-	select {
-	case r.flows <- e:
-	default:
-	}
-}
-
-func (r *bufferedReporter) ReportFlowEvict(e events.FlowEvictEvent) {
-	select {
-	case r.evicts <- e:
-	default:
-	}
-}
-
-func (r *bufferedReporter) ReportRulesReload(events.RulesReloadEvent) {}
-
-// pointerReporter is the anti-pattern: it takes the address of the incoming
-// event struct, which forces the callee-side copy onto the heap. Kept for
-// comparison so we can see the alloc cost an unwary reporter would incur.
-type pointerReporter struct {
-	last *events.DropEvent
-}
-
-func (r *pointerReporter) ReportDrop(e events.DropEvent) {
-	r.last = &e
-}
-
-func (r *pointerReporter) ReportFlowCreate(events.FlowCreateEvent) {}
-func (r *pointerReporter) ReportFlowEvict(events.FlowEvictEvent)   {}
-func (r *pointerReporter) ReportRulesReload(events.RulesReloadEvent) {
-}
-
-func newBenchFixture(b *testing.B) *eventFixture {
-	b.Helper()
-	l := test.NewLogger()
-
-	vpnNetworks := new(bart.Lite)
-	vpnNetworks.Insert(netip.MustParsePrefix("1.2.3.0/24"))
-
-	c := &dummyCert{
-		name:     "host1",
-		networks: []netip.Prefix{netip.MustParsePrefix("1.2.3.4/24")},
-		groups:   []string{"default-group"},
-		issuer:   "signer-shasum",
-	}
-	h := &HostInfo{
-		ConnectionState: &ConnectionState{
-			peerCert: &cert.CachedCertificate{
-				Certificate:    c,
-				InvertedGroups: map[string]struct{}{"default-group": {}},
-			},
-		},
-		vpnAddrs: []netip.Addr{netip.MustParseAddr("1.2.3.4")},
-	}
-	h.buildNetworks(vpnNetworks, c)
-
-	fw := NewFirewall(l, time.Minute, time.Minute, time.Minute, c)
-	// Inbound rule that matches our packet; outbound has no match so we can
-	// also benchmark the no-rule drop path.
-	if err := fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""); err != nil {
-		b.Fatal(err)
-	}
-
-	ctl := &Control{f: &Interface{firewall: fw}, l: l}
-	return &eventFixture{
-		ctl: ctl,
-		fw:  fw,
-		p: firewall.Packet{
-			LocalAddr:  netip.MustParseAddr("1.2.3.4"),
-			RemoteAddr: netip.MustParseAddr("1.2.3.4"),
-			LocalPort:  10,
-			RemotePort: 90,
-			Protocol:   firewall.ProtoUDP,
-		},
-		h:  h,
-		cp: cert.NewCAPool(),
-	}
-}
-
-// BenchmarkFirewallDropPath measures the cost of Firewall.Drop on a packet
-// that reaches the no-matching-rule branch (the longest drop path). Compare
-// reporter shapes:
-//
-//	nilReporter      -- no reporter installed (feature cost when off)
-//	noopReporter     -- reporter installed, methods discard args (minimum on-cost)
-//	bufferedReporter -- realistic zero-alloc reporter: value-typed channels
-//	pointerReporter  -- anti-pattern that takes &composite-literal (allocates)
-func BenchmarkFirewallDropPath(b *testing.B) {
-	run := func(b *testing.B, install func(*Control)) {
-		f := newBenchFixture(b)
-		install(f.ctl)
-		b.ReportAllocs()
-		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
-			_ = f.firewall().Drop(f.p, firewall.PacketContext{}, false, f.h, f.cp, nil)
-		}
-	}
-
-	b.Run("nilReporter", func(b *testing.B) { run(b, func(*Control) {}) })
-	b.Run("noopReporter", func(b *testing.B) {
-		run(b, func(c *Control) { c.SetFirewallEventReporter(noopReporter{}) })
-	})
-	b.Run("bufferedReporter", func(b *testing.B) {
-		run(b, func(c *Control) { c.SetFirewallEventReporter(newBufferedReporter(1024)) })
-	})
-	b.Run("pointerReporter", func(b *testing.B) {
-		run(b, func(c *Control) { c.SetFirewallEventReporter(&pointerReporter{}) })
-	})
-}
-
-// BenchmarkConntrackCreate measures Firewall.Drop for an allowed inbound
-// packet on a fresh conntrack (so addConn fires each iteration).
-func BenchmarkConntrackCreate(b *testing.B) {
-	run := func(b *testing.B, install func(*Control)) {
-		f := newBenchFixture(b)
-		install(f.ctl)
-		b.ReportAllocs()
-		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
-			resetConntrack(f.firewall())
-			_ = f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
-		}
-	}
-	b.Run("nilReporter", func(b *testing.B) { run(b, func(*Control) {}) })
-	b.Run("noopReporter", func(b *testing.B) {
-		run(b, func(c *Control) { c.SetFirewallEventReporter(noopReporter{}) })
-	})
-	b.Run("bufferedReporter", func(b *testing.B) {
-		run(b, func(c *Control) { c.SetFirewallEventReporter(newBufferedReporter(1024)) })
-	})
-}
-
-// BenchmarkConntrackHit measures the hot path where a flow is already in
-// conntrack and short-circuits rule evaluation. The reporter slot is checked
-// only on create/evict, so this bench should show the reporter having zero
-// impact regardless of install state.
-func BenchmarkConntrackHit(b *testing.B) {
-	b.Run("nilReporter", func(b *testing.B) {
-		f := newBenchFixture(b)
-		// Prime conntrack.
-		require.NoError(b, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
-		b.ReportAllocs()
-		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
-			_ = f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
-		}
-	})
-
-	b.Run("noopReporter", func(b *testing.B) {
-		f := newBenchFixture(b)
-		f.ctl.SetFirewallEventReporter(noopReporter{})
-		require.NoError(b, f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil))
-		b.ReportAllocs()
-		b.ResetTimer()
-		for i := 0; i < b.N; i++ {
-			_ = f.firewall().Drop(f.p, firewall.PacketContext{}, true, f.h, f.cp, nil)
-		}
-	})
-}

go.mod

@@ -1,39 +1,37 @@
 module github.com/slackhq/nebula
 
-go 1.25.0
+go 1.25
 
 require (
 	dario.cat/mergo v1.0.2
-	filippo.io/bigmod v0.1.0
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
 	github.com/armon/go-radix v1.0.0
 	github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432
 	github.com/flynn/noise v1.1.0
-	github.com/gaissmai/bart v0.26.0
+	github.com/gaissmai/bart v0.25.0
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/gopacket v1.1.19
 	github.com/kardianos/service v1.2.4
-	github.com/miekg/dns v1.1.72
-	github.com/miekg/pkcs11 v1.1.2
+	github.com/miekg/dns v1.1.68
+	github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b
 	github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f
 	github.com/prometheus/client_golang v1.23.2
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
-	github.com/sirupsen/logrus v1.9.4
+	github.com/sirupsen/logrus v1.9.3
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
 	github.com/stretchr/testify v1.11.1
 	github.com/vishvananda/netlink v1.3.1
-	go.yaml.in/yaml/v3 v3.0.4
-	golang.org/x/crypto v0.50.0
+	golang.org/x/crypto v0.43.0
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
-	golang.org/x/net v0.52.0
-	golang.org/x/sync v0.20.0
-	golang.org/x/sys v0.43.0
-	golang.org/x/term v0.42.0
+	golang.org/x/net v0.45.0
+	golang.org/x/sync v0.17.0
+	golang.org/x/sys v0.37.0
+	golang.org/x/term v0.36.0
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
 	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
-	golang.zx2c4.com/wireguard/windows v0.6.1
-	google.golang.org/protobuf v1.36.11
+	golang.zx2c4.com/wireguard/windows v0.5.3
+	google.golang.org/protobuf v1.36.8
 	gopkg.in/yaml.v3 v3.0.1
 	gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
 )
@@ -50,7 +48,7 @@ require (
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/vishvananda/netns v0.0.5 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
-	golang.org/x/mod v0.34.0 // indirect
+	golang.org/x/mod v0.24.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.43.0 // indirect
+	golang.org/x/tools v0.33.0 // indirect
 )

go.sum

@@ -1,8 +1,6 @@
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
 dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
-filippo.io/bigmod v0.1.0 h1:UNzDk7y9ADKST+axd9skUpBQeW7fG2KrTZyOE4uGQy8=
-filippo.io/bigmod v0.1.0/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -26,8 +24,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg=
 github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
-github.com/gaissmai/bart v0.26.0 h1:xOZ57E9hJLBiQaSyeZa9wgWhGuzfGACgqp4BE77OkO0=
-github.com/gaissmai/bart v0.26.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
+github.com/gaissmai/bart v0.25.0 h1:eqiokVPqM3F94vJ0bTHXHtH91S8zkKL+bKh+BsGOsJM=
+github.com/gaissmai/bart v0.25.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -85,10 +83,10 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
-github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
-github.com/miekg/pkcs11 v1.1.2 h1:/VxmeAX5qU6Q3EwafypogwWbYryHFmF2RpkJmw3m4MQ=
-github.com/miekg/pkcs11 v1.1.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/miekg/dns v1.1.68 h1:jsSRkNozw7G/mnmXULynzMNIsgY2dHC8LO6U6Ij2JEA=
+github.com/miekg/dns v1.1.68/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps=
+github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b h1:J/AzCvg5z0Hn1rqZUJjpbzALUmkKX0Zwbc/i4fw7Sfk=
+github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -133,8 +131,8 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
-github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
 github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 h1:pnnLyeX7o/5aX8qUQ69P/mLojDqwda8hFOCBTmP/6hw=
@@ -157,23 +155,21 @@ go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
-go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
-go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
-golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
+golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
+golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
-golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
+golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU=
+golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -184,8 +180,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
-golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
+golang.org/x/net v0.45.0 h1:RLBg5JKixCy82FtLJpeNlVM0nrSqpCRYzVU1n8kj0tM=
+golang.org/x/net v0.45.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -193,8 +189,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
-golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
+golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -208,13 +204,14 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
-golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
+golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
-golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
+golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q=
+golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -225,8 +222,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
-golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
+golang.org/x/tools v0.33.0 h1:4qz2S3zmRxbGIhDIAgjxvFutSvH5EfnsYrRBj0UI0bc=
+golang.org/x/tools v0.33.0/go.mod h1:CIJMaWEY88juyUfo7UbgPqbC8rU2OqfAV1h2Qp0oMYI=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -235,8 +232,8 @@ golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeu
 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
 golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo=
 golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
-golang.zx2c4.com/wireguard/windows v0.6.1 h1:XMaKojH1Hs/raMrmnir4n35nTvzvWj7NmSYzHn2F4qU=
-golang.zx2c4.com/wireguard/windows v0.6.1/go.mod h1:04aqInu5GYuTFvMuDw/rKBAF7mHrltW/3rekpfbbZDM=
+golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE=
+golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -245,8 +242,8 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
-google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc=
+google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

handshake_ix.go

@@ -1,8 +1,8 @@
 package nebula
 
 import (
-	"bytes"
 	"net/netip"
+	"slices"
 	"time"
 
 	"github.com/flynn/noise"
@@ -100,11 +100,11 @@ func ixHandshakeStage0(f *Interface, hh *HandshakeHostInfo) bool {
 	return true
 }
 
-func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H) {
+func ixHandshakeStage1(f *Interface, addr netip.AddrPort, via *ViaSender, packet []byte, h *header.H) {
 	cs := f.pki.getCertState()
 	crt := cs.GetDefaultCertificate()
 	if crt == nil {
-		f.l.WithField("from", via).
+		f.l.WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).
 			WithField("certVersion", cs.initiatingVersion).
 			Error("Unable to handshake with host because no certificate is available")
@@ -113,7 +113,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 
 	ci, err := NewConnectionState(f.l, cs, crt, false, noise.HandshakeIX)
 	if err != nil {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
 			Error("Failed to create connection state")
 		return
@@ -124,7 +124,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 
 	msg, _, _, err := ci.H.ReadMessage(nil, packet[header.Len:])
 	if err != nil {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
 			Error("Failed to call noise.ReadMessage")
 		return
@@ -133,7 +133,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 	hs := &NebulaHandshake{}
 	err = hs.Unmarshal(msg)
 	if err != nil || hs.Details == nil {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
 			Error("Failed unmarshal handshake message")
 		return
@@ -141,7 +141,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 
 	rc, err := cert.Recombine(cert.Version(hs.Details.CertVersion), hs.Details.Cert, ci.H.PeerStatic(), ci.Curve())
 	if err != nil {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
 			Info("Handshake did not contain a certificate")
 		return
@@ -154,7 +154,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 			fp = "<error generating certificate fingerprint>"
 		}
 
-		e := f.l.WithError(err).WithField("from", via).
+		e := f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
 			WithField("certVpnNetworks", rc.Networks()).
 			WithField("certFingerprint", fp)
@@ -167,20 +167,13 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 		return
 	}
 
-	if !bytes.Equal(remoteCert.Certificate.PublicKey(), ci.H.PeerStatic()) {
-		f.l.WithField("from", via).
-			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
-			WithField("cert", remoteCert).Info("public key mismatch between certificate and handshake")
-		return
-	}
-
 	if remoteCert.Certificate.Version() != ci.myCert.Version() {
 		// We started off using the wrong certificate version, lets see if we can match the version that was sent to us
 		myCertOtherVersion := cs.getCertificate(remoteCert.Certificate.Version())
 		if myCertOtherVersion == nil {
 			if f.l.Level >= logrus.DebugLevel {
 				f.l.WithError(err).WithFields(m{
-					"from":      via,
+					"udpAddr":   addr,
 					"handshake": m{"stage": 1, "style": "ix_psk0"},
 					"cert":      remoteCert,
 				}).Debug("Might be unable to handshake with host due to missing certificate version")
@@ -192,24 +185,24 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 	}
 
 	if len(remoteCert.Certificate.Networks()) == 0 {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("cert", remoteCert).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
 			Info("No networks in certificate")
 		return
 	}
 
+	var vpnAddrs []netip.Addr
+	var filteredNetworks []netip.Prefix
 	certName := remoteCert.Certificate.Name()
 	certVersion := remoteCert.Certificate.Version()
 	fingerprint := remoteCert.Fingerprint
 	issuer := remoteCert.Certificate.Issuer()
-	vpnNetworks := remoteCert.Certificate.Networks()
 
-	anyVpnAddrsInCommon := false
-	vpnAddrs := make([]netip.Addr, len(vpnNetworks))
-	for i, network := range vpnNetworks {
-		if f.myVpnAddrsTable.Contains(network.Addr()) {
-			f.l.WithField("vpnNetworks", vpnNetworks).WithField("from", via).
+	for _, network := range remoteCert.Certificate.Networks() {
+		vpnAddr := network.Addr()
+		if f.myVpnAddrsTable.Contains(vpnAddr) {
+			f.l.WithField("vpnAddr", vpnAddr).WithField("udpAddr", addr).
 				WithField("certName", certName).
 				WithField("certVersion", certVersion).
 				WithField("fingerprint", fingerprint).
@@ -217,24 +210,38 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Refusing to handshake with myself")
 			return
 		}
-		vpnAddrs[i] = network.Addr()
-		if f.myVpnNetworksTable.Contains(network.Addr()) {
-			anyVpnAddrsInCommon = true
+
+		// vpnAddrs outside our vpn networks are of no use to us, filter them out
+		if !f.myVpnNetworksTable.Contains(vpnAddr) {
+			continue
 		}
+
+		filteredNetworks = append(filteredNetworks, network)
+		vpnAddrs = append(vpnAddrs, vpnAddr)
 	}
 
-	if !via.IsRelayed {
+	if len(vpnAddrs) == 0 {
+		f.l.WithError(err).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("certVersion", certVersion).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("No usable vpn addresses from host, refusing handshake")
+		return
+	}
+
+	if addr.IsValid() {
+		// addr can be invalid when the tunnel is being relayed.
 		// We only want to apply the remote allow list for direct tunnels here
-		if !f.lightHouse.GetRemoteAllowList().AllowAll(vpnAddrs, via.UdpAddr.Addr()) {
-			f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
-				Debug("lighthouse.remote_allow_list denied incoming handshake")
+		if !f.lightHouse.GetRemoteAllowList().AllowAll(vpnAddrs, addr.Addr()) {
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake")
 			return
 		}
 	}
 
 	myIndex, err := generateIndex(f.l)
 	if err != nil {
-		f.l.WithError(err).WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+		f.l.WithError(err).WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
 			WithField("certName", certName).
 			WithField("certVersion", certVersion).
 			WithField("fingerprint", fingerprint).
@@ -257,30 +264,26 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 		},
 	}
 
-	msgRxL := f.l.WithFields(m{
-		"vpnAddrs":       vpnAddrs,
-		"from":           via,
-		"certName":       certName,
-		"certVersion":    certVersion,
-		"fingerprint":    fingerprint,
-		"issuer":         issuer,
-		"initiatorIndex": hs.Details.InitiatorIndex,
-		"responderIndex": hs.Details.ResponderIndex,
-		"remoteIndex":    h.RemoteIndex,
-		"handshake":      m{"stage": 1, "style": "ix_psk0"},
-	})
-
-	if anyVpnAddrsInCommon {
-		msgRxL.Info("Handshake message received")
-	} else {
-		//todo warn if not lighthouse or relay?
-		msgRxL.Info("Handshake message received, but no vpnNetworks in common.")
-	}
+	f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
+		WithField("certName", certName).
+		WithField("certVersion", certVersion).
+		WithField("fingerprint", fingerprint).
+		WithField("issuer", issuer).
+		WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+		WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+		Info("Handshake message received")
 
 	hs.Details.ResponderIndex = myIndex
 	hs.Details.Cert = cs.getHandshakeBytes(ci.myCert.Version())
 	if hs.Details.Cert == nil {
-		msgRxL.WithField("myCertVersion", ci.myCert.Version()).
+		f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("certVersion", certVersion).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+			WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+			WithField("certVersion", ci.myCert.Version()).
 			Error("Unable to handshake with host because no certificate handshake bytes is available")
 		return
 	}
@@ -291,7 +294,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 
 	hsBytes, err := hs.Marshal()
 	if err != nil {
-		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("certName", certName).
 			WithField("certVersion", certVersion).
 			WithField("fingerprint", fingerprint).
@@ -303,7 +306,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 	nh := header.Encode(make([]byte, header.Len), header.Version, header.Handshake, header.HandshakeIXPSK0, hs.Details.InitiatorIndex, 2)
 	msg, dKey, eKey, err := ci.H.WriteMessage(nh, hsBytes)
 	if err != nil {
-		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("certName", certName).
 			WithField("certVersion", certVersion).
 			WithField("fingerprint", fingerprint).
@@ -311,7 +314,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to call noise.WriteMessage")
 		return
 	} else if dKey == nil || eKey == nil {
-		f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("certName", certName).
 			WithField("certVersion", certVersion).
 			WithField("fingerprint", fingerprint).
@@ -337,17 +340,15 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 	ci.eKey = NewNebulaCipherState(eKey)
 
 	hostinfo.remotes = f.lightHouse.QueryCache(vpnAddrs)
-	if !via.IsRelayed {
-		hostinfo.SetRemote(via.UdpAddr)
-	}
-	hostinfo.buildNetworks(f.myVpnNetworksTable, remoteCert.Certificate)
+	hostinfo.SetRemote(addr)
+	hostinfo.buildNetworks(filteredNetworks, remoteCert.Certificate.UnsafeNetworks())
 
 	existing, err := f.handshakeManager.CheckAndComplete(hostinfo, 0, f)
 	if err != nil {
 		switch err {
 		case ErrAlreadySeen:
 			// Update remote if preferred
-			if existing.SetRemoteIfPreferred(f.hostMap, via) {
+			if existing.SetRemoteIfPreferred(f.hostMap, addr) {
 				// Send a test packet to ensure the other side has also switched to
 				// the preferred remote
 				f.SendMessageToVpnAddr(header.Test, header.TestRequest, vpnAddrs[0], []byte(""), make([]byte, 12, 12), make([]byte, mtu))
@@ -355,21 +356,21 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 
 			msg = existing.HandshakePacket[2]
 			f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1)
-			if !via.IsRelayed {
-				err := f.outside.WriteTo(msg, via.UdpAddr)
+			if addr.IsValid() {
+				err := f.outside.WriteTo(msg, addr)
 				if err != nil {
-					f.l.WithField("vpnAddrs", existing.vpnAddrs).WithField("from", via).
+					f.l.WithField("vpnAddrs", existing.vpnAddrs).WithField("udpAddr", addr).
 						WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
 						WithError(err).Error("Failed to send handshake message")
 				} else {
-					f.l.WithField("vpnAddrs", existing.vpnAddrs).WithField("from", via).
+					f.l.WithField("vpnAddrs", existing.vpnAddrs).WithField("udpAddr", addr).
 						WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
 						Info("Handshake message sent")
 				}
 				return
 			} else {
-				if via.relay == nil {
-					f.l.Error("Handshake send failed: both addr and via.relay are nil.")
+				if via == nil {
+					f.l.Error("Handshake send failed: both addr and via are nil.")
 					return
 				}
 				hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
@@ -381,7 +382,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 			}
 		case ErrExistingHostInfo:
 			// This means there was an existing tunnel and this handshake was older than the one we are currently based on
-			f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
 				WithField("certName", certName).
 				WithField("certVersion", certVersion).
 				WithField("oldHandshakeTime", existing.lastHandshakeTime).
@@ -397,7 +398,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 			return
 		case ErrLocalIndexCollision:
 			// This means we failed to insert because of collision on localIndexId. Just let the next handshake packet retry
-			f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
 				WithField("certName", certName).
 				WithField("certVersion", certVersion).
 				WithField("fingerprint", fingerprint).
@@ -410,7 +411,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 		default:
 			// Shouldn't happen, but just in case someone adds a new error type to CheckAndComplete
 			// And we forget to update it here
-			f.l.WithError(err).WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+			f.l.WithError(err).WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
 				WithField("certName", certName).
 				WithField("certVersion", certVersion).
 				WithField("fingerprint", fingerprint).
@@ -424,23 +425,30 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 
 	// Do the send
 	f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1)
-	if !via.IsRelayed {
-		err = f.outside.WriteTo(msg, via.UdpAddr)
-		log := f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
-			WithField("certName", certName).
-			WithField("certVersion", certVersion).
-			WithField("fingerprint", fingerprint).
-			WithField("issuer", issuer).
-			WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
-			WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"})
+	if addr.IsValid() {
+		err = f.outside.WriteTo(msg, addr)
 		if err != nil {
-			log.WithError(err).Error("Failed to send handshake")
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
+				WithField("certName", certName).
+				WithField("certVersion", certVersion).
+				WithField("fingerprint", fingerprint).
+				WithField("issuer", issuer).
+				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+				WithError(err).Error("Failed to send handshake")
 		} else {
-			log.Info("Handshake message sent")
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
+				WithField("certName", certName).
+				WithField("certVersion", certVersion).
+				WithField("fingerprint", fingerprint).
+				WithField("issuer", issuer).
+				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+				Info("Handshake message sent")
 		}
 	} else {
-		if via.relay == nil {
-			f.l.Error("Handshake send failed: both addr and via.relay are nil.")
+		if via == nil {
+			f.l.Error("Handshake send failed: both addr and via are nil.")
 			return
 		}
 		hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
@@ -465,7 +473,7 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 	return
 }
 
-func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packet []byte, h *header.H) bool {
+func ixHandshakeStage2(f *Interface, addr netip.AddrPort, via *ViaSender, hh *HandshakeHostInfo, packet []byte, h *header.H) bool {
 	if hh == nil {
 		// Nothing here to tear down, got a bogus stage 2 packet
 		return true
@@ -475,10 +483,10 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 	defer hh.Unlock()
 
 	hostinfo := hh.hostinfo
-	if !via.IsRelayed {
+	if addr.IsValid() {
 		// The vpnAddr we know about is the one we tried to handshake with, use it to apply the remote allow list.
-		if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, via.UdpAddr.Addr()) {
-			f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).Debug("lighthouse.remote_allow_list denied incoming handshake")
+		if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, addr.Addr()) {
+			f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake")
 			return false
 		}
 	}
@@ -486,7 +494,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 	ci := hostinfo.ConnectionState
 	msg, eKey, dKey, err := ci.H.ReadMessage(nil, packet[header.Len:])
 	if err != nil {
-		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("header", h).
 			Error("Failed to call noise.ReadMessage")
 
@@ -495,7 +503,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 		// near future
 		return false
 	} else if dKey == nil || eKey == nil {
-		f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
 			Error("Noise did not arrive at a key")
 
@@ -507,7 +515,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 	hs := &NebulaHandshake{}
 	err = hs.Unmarshal(msg)
 	if err != nil || hs.Details == nil {
-		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).Error("Failed unmarshal handshake message")
 
 		// The handshake state machine is complete, if things break now there is no chance to recover. Tear down and start again
@@ -516,7 +524,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 
 	rc, err := cert.Recombine(cert.Version(hs.Details.CertVersion), hs.Details.Cert, ci.H.PeerStatic(), ci.Curve())
 	if err != nil {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("vpnAddrs", hostinfo.vpnAddrs).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
 			Info("Handshake did not contain a certificate")
@@ -530,7 +538,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 			fp = "<error generating certificate fingerprint>"
 		}
 
-		e := f.l.WithError(err).WithField("from", via).
+		e := f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("vpnAddrs", hostinfo.vpnAddrs).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
 			WithField("certFingerprint", fp).
@@ -543,15 +551,9 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 		e.Info("Invalid certificate from host")
 		return true
 	}
-	if !bytes.Equal(remoteCert.Certificate.PublicKey(), ci.H.PeerStatic()) {
-		f.l.WithField("from", via).
-			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
-			WithField("cert", remoteCert).Info("public key mismatch between certificate and handshake")
-		return true
-	}
 
 	if len(remoteCert.Certificate.Networks()) == 0 {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("vpnAddrs", hostinfo.vpnAddrs).
 			WithField("cert", remoteCert).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
@@ -574,30 +576,39 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 	ci.eKey = NewNebulaCipherState(eKey)
 
 	// Make sure the current udpAddr being used is set for responding
-	if !via.IsRelayed {
-		hostinfo.SetRemote(via.UdpAddr)
+	if addr.IsValid() {
+		hostinfo.SetRemote(addr)
 	} else {
 		hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
 	}
 
-	correctHostResponded := false
-	anyVpnAddrsInCommon := false
-	vpnAddrs := make([]netip.Addr, len(vpnNetworks))
-	for i, network := range vpnNetworks {
-		vpnAddrs[i] = network.Addr()
-		if f.myVpnNetworksTable.Contains(network.Addr()) {
-			anyVpnAddrsInCommon = true
-		}
-		if hostinfo.vpnAddrs[0] == network.Addr() {
-			// todo is it more correct to see if any of hostinfo.vpnAddrs are in the cert? it should have len==1, but one day it might not?
-			correctHostResponded = true
+	var vpnAddrs []netip.Addr
+	var filteredNetworks []netip.Prefix
+	for _, network := range vpnNetworks {
+		// vpnAddrs outside our vpn networks are of no use to us, filter them out
+		vpnAddr := network.Addr()
+		if !f.myVpnNetworksTable.Contains(vpnAddr) {
+			continue
 		}
+
+		filteredNetworks = append(filteredNetworks, network)
+		vpnAddrs = append(vpnAddrs, vpnAddr)
+	}
+
+	if len(vpnAddrs) == 0 {
+		f.l.WithError(err).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("certVersion", certVersion).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).Error("No usable vpn addresses from host, refusing handshake")
+		return true
 	}
 
 	// Ensure the right host responded
-	if !correctHostResponded {
+	if !slices.Contains(vpnAddrs, hostinfo.vpnAddrs[0]) {
 		f.l.WithField("intendedVpnAddrs", hostinfo.vpnAddrs).WithField("haveVpnNetworks", vpnNetworks).
-			WithField("from", via).
+			WithField("udpAddr", addr).
 			WithField("certName", certName).
 			WithField("certVersion", certVersion).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
@@ -607,11 +618,10 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 		f.handshakeManager.DeleteHostInfo(hostinfo)
 
 		// Create a new hostinfo/handshake for the intended vpn ip
-		//TODO is hostinfo.vpnAddrs[0] always the address to use?
 		f.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], func(newHH *HandshakeHostInfo) {
 			// Block the current used address
 			newHH.hostinfo.remotes = hostinfo.remotes
-			newHH.hostinfo.remotes.BlockRemote(via)
+			newHH.hostinfo.remotes.BlockRemote(addr)
 
 			f.l.WithField("blockedUdpAddrs", newHH.hostinfo.remotes.CopyBlockedRemotes()).
 				WithField("vpnNetworks", vpnNetworks).
@@ -634,7 +644,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 	ci.window.Update(f.l, 2)
 
 	duration := time.Since(hh.startTime).Nanoseconds()
-	msgRxL := f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+	f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
 		WithField("certName", certName).
 		WithField("certVersion", certVersion).
 		WithField("fingerprint", fingerprint).
@@ -642,17 +652,12 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 		WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
 		WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
 		WithField("durationNs", duration).
-		WithField("sentCachedPackets", len(hh.packetStore))
-	if anyVpnAddrsInCommon {
-		msgRxL.Info("Handshake message received")
-	} else {
-		//todo warn if not lighthouse or relay?
-		msgRxL.Info("Handshake message received, but no vpnNetworks in common.")
-	}
+		WithField("sentCachedPackets", len(hh.packetStore)).
+		Info("Handshake message received")
 
 	// Build up the radix for the firewall if we have subnets in the cert
 	hostinfo.vpnAddrs = vpnAddrs
-	hostinfo.buildNetworks(f.myVpnNetworksTable, remoteCert.Certificate)
+	hostinfo.buildNetworks(filteredNetworks, remoteCert.Certificate.UnsafeNetworks())
 
 	// Complete our handshake and update metrics, this will replace any existing tunnels for the vpnAddrs here
 	f.handshakeManager.Complete(hostinfo, f)

handshake_manager.go

@@ -136,11 +136,11 @@ func (hm *HandshakeManager) Run(ctx context.Context) {
 	}
 }
 
-func (hm *HandshakeManager) HandleIncoming(via ViaSender, packet []byte, h *header.H) {
+func (hm *HandshakeManager) HandleIncoming(addr netip.AddrPort, via *ViaSender, packet []byte, h *header.H) {
 	// First remote allow list check before we know the vpnIp
-	if !via.IsRelayed {
-		if !hm.lightHouse.GetRemoteAllowList().AllowUnknownVpnAddr(via.UdpAddr.Addr()) {
-			hm.l.WithField("from", via).Debug("lighthouse.remote_allow_list denied incoming handshake")
+	if addr.IsValid() {
+		if !hm.lightHouse.GetRemoteAllowList().AllowUnknownVpnAddr(addr.Addr()) {
+			hm.l.WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake")
 			return
 		}
 	}
@@ -149,11 +149,11 @@ func (hm *HandshakeManager) HandleIncoming(via ViaSender, packet []byte, h *head
 	case header.HandshakeIXPSK0:
 		switch h.MessageCounter {
 		case 1:
-			ixHandshakeStage1(hm.f, via, packet, h)
+			ixHandshakeStage1(hm.f, addr, via, packet, h)
 
 		case 2:
 			newHostinfo := hm.queryIndex(h.RemoteIndex)
-			tearDown := ixHandshakeStage2(hm.f, via, newHostinfo, packet, h)
+			tearDown := ixHandshakeStage2(hm.f, addr, via, newHostinfo, packet, h)
 			if tearDown && newHostinfo != nil {
 				hm.DeleteHostInfo(newHostinfo.hostinfo)
 			}
@@ -269,12 +269,12 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 		hostinfo.logger(hm.l).WithField("relays", hostinfo.remotes.relays).Info("Attempt to relay through hosts")
 		// Send a RelayRequest to all known Relay IP's
 		for _, relay := range hostinfo.remotes.relays {
-			// Don't relay through the host I'm trying to connect to
+			// Don't relay to myself
 			if relay == vpnIp {
 				continue
 			}
 
-			// Don't relay to myself
+			// Don't relay through the host I'm trying to connect to
 			if hm.f.myVpnAddrsTable.Contains(relay) {
 				continue
 			}
@@ -590,7 +590,7 @@ func (hm *HandshakeManager) allocateIndex(hh *HandshakeHostInfo) error {
 	hm.Lock()
 	defer hm.Unlock()
 
-	for range 32 {
+	for i := 0; i < 32; i++ {
 		index, err := generateIndex(hm.l)
 		if err != nil {
 			return err

hostmap.go

@@ -1,9 +1,7 @@
 package nebula
 
 import (
-	"encoding/json"
 	"errors"
-	"fmt"
 	"net"
 	"net/netip"
 	"slices"
@@ -214,18 +212,6 @@ func (rs *RelayState) InsertRelay(ip netip.Addr, idx uint32, r *Relay) {
 	rs.relayForByIdx[idx] = r
 }
 
-type NetworkType uint8
-
-const (
-	NetworkTypeUnknown NetworkType = iota
-	// NetworkTypeVPN is a network that overlaps one or more of the vpnNetworks in our certificate
-	NetworkTypeVPN
-	// NetworkTypeVPNPeer is a network that does not overlap one of our networks
-	NetworkTypeVPNPeer
-	// NetworkTypeUnsafe is a network from Certificate.UnsafeNetworks()
-	NetworkTypeUnsafe
-)
-
 type HostInfo struct {
 	remote          netip.AddrPort
 	remotes         *RemoteList
@@ -239,8 +225,8 @@ type HostInfo struct {
 	// vpn networks but were removed because they are not usable
 	vpnAddrs []netip.Addr
 
-	// networks is a combination of specific vpn addresses (not prefixes!) and full unsafe networks assigned to this host.
-	networks   *bart.Table[NetworkType]
+	// networks are both all vpn and unsafe networks assigned to this host
+	networks   *bart.Lite
 	relayState RelayState
 
 	// HandshakePacket records the packets used to create this hostinfo
@@ -278,25 +264,9 @@ type HostInfo struct {
 }
 
 type ViaSender struct {
-	UdpAddr   netip.AddrPort
 	relayHI   *HostInfo // relayHI is the host info object of the relay
 	remoteIdx uint32    // remoteIdx is the index included in the header of the received packet
 	relay     *Relay    // relay contains the rest of the relay information, including the PeerIP of the host trying to communicate with us.
-	IsRelayed bool      // IsRelayed is true if the packet was sent through a relay
-}
-
-func (v ViaSender) String() string {
-	if v.IsRelayed {
-		return fmt.Sprintf("%s (relayed)", v.UdpAddr)
-	}
-	return v.UdpAddr.String()
-}
-
-func (v ViaSender) MarshalJSON() ([]byte, error) {
-	if v.IsRelayed {
-		return json.Marshal(m{"relay": v.UdpAddr})
-	}
-	return json.Marshal(m{"direct": v.UdpAddr})
 }
 
 type cachedPacket struct {
@@ -604,9 +574,9 @@ func (hm *HostMap) queryVpnAddr(vpnIp netip.Addr, promoteIfce *Interface) *HostI
 // unlockedAddHostInfo assumes you have a write-lock and will add a hostinfo object to the hostmap Indexes and RemoteIndexes maps.
 // If an entry exists for the Hosts table (vpnIp -> hostinfo) then the provided hostinfo will be made primary
 func (hm *HostMap) unlockedAddHostInfo(hostinfo *HostInfo, f *Interface) {
-	if f.dnsServer != nil {
+	if f.serveDns {
 		remoteCert := hostinfo.ConnectionState.peerCert
-		f.dnsServer.Add(remoteCert.Certificate.Name()+".", hostinfo.vpnAddrs)
+		dnsR.Add(remoteCert.Certificate.Name()+".", hostinfo.vpnAddrs)
 	}
 	for _, addr := range hostinfo.vpnAddrs {
 		hm.unlockedInnerAddHostInfo(addr, hostinfo, f)
@@ -712,7 +682,6 @@ func (i *HostInfo) GetCert() *cert.CachedCertificate {
 	return nil
 }
 
-// TODO: Maybe use ViaSender here?
 func (i *HostInfo) SetRemote(remote netip.AddrPort) {
 	// We copy here because we likely got this remote from a source that reuses the object
 	if i.remote != remote {
@@ -723,14 +692,14 @@ func (i *HostInfo) SetRemote(remote netip.AddrPort) {
 
 // SetRemoteIfPreferred returns true if the remote was changed. The lastRoam
 // time on the HostInfo will also be updated.
-func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, via ViaSender) bool {
-	if via.IsRelayed {
+func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, newRemote netip.AddrPort) bool {
+	if !newRemote.IsValid() {
+		// relays have nil udp Addrs
 		return false
 	}
-
 	currentRemote := i.remote
 	if !currentRemote.IsValid() {
-		i.SetRemote(via.UdpAddr)
+		i.SetRemote(newRemote)
 		return true
 	}
 
@@ -743,7 +712,7 @@ func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, via ViaSender) bool {
 			return false
 		}
 
-		if l.Contains(via.UdpAddr.Addr()) {
+		if l.Contains(newRemote.Addr()) {
 			newIsPreferred = true
 		}
 	}
@@ -753,7 +722,7 @@ func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, via ViaSender) bool {
 		i.lastRoam = time.Now()
 		i.lastRoamRemote = currentRemote
 
-		i.SetRemote(via.UdpAddr)
+		i.SetRemote(newRemote)
 
 		return true
 	}
@@ -761,26 +730,20 @@ func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, via ViaSender) bool {
 	return false
 }
 
-// buildNetworks fills in the networks field of HostInfo. It accepts a cert.Certificate so you never ever mix the network types up.
-func (i *HostInfo) buildNetworks(myVpnNetworksTable *bart.Lite, c cert.Certificate) {
-	if len(c.Networks()) == 1 && len(c.UnsafeNetworks()) == 0 {
-		if myVpnNetworksTable.Contains(c.Networks()[0].Addr()) {
-			return // Simple case, no BART needed
-		}
+func (i *HostInfo) buildNetworks(networks, unsafeNetworks []netip.Prefix) {
+	if len(networks) == 1 && len(unsafeNetworks) == 0 {
+		// Simple case, no CIDRTree needed
+		return
 	}
 
-	i.networks = new(bart.Table[NetworkType])
-	for _, network := range c.Networks() {
+	i.networks = new(bart.Lite)
+	for _, network := range networks {
 		nprefix := netip.PrefixFrom(network.Addr(), network.Addr().BitLen())
-		if myVpnNetworksTable.Contains(network.Addr()) {
-			i.networks.Insert(nprefix, NetworkTypeVPN)
-		} else {
-			i.networks.Insert(nprefix, NetworkTypeVPNPeer)
-		}
+		i.networks.Insert(nprefix)
 	}
 
-	for _, network := range c.UnsafeNetworks() {
-		i.networks.Insert(network, NetworkTypeUnsafe)
+	for _, network := range unsafeNetworks {
+		i.networks.Insert(network)
 	}
 }
 

hostmap_tester.go

@@ -1,4 +1,5 @@
 //go:build e2e_testing
+// +build e2e_testing
 
 package nebula
 

inside.go

@@ -2,17 +2,19 @@ package nebula
 
 import (
 	"net/netip"
+	"unsafe"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/iputil"
 	"github.com/slackhq/nebula/noiseutil"
+	"github.com/slackhq/nebula/overlay"
 	"github.com/slackhq/nebula/routing"
 )
 
-func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, fwCtx *firewall.PacketContext, nb, out []byte, q int, localCache firewall.ConntrackCache) {
-	err := newPacket(packet, false, fwPacket, fwCtx)
+func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache *firewall.ConntrackCache) {
+	err := newPacket(packet, false, fwPacket)
 	if err != nil {
 		if f.l.Level >= logrus.DebugLevel {
 			f.l.WithField("packet", packet).Debugf("Error while validating outbound packet: %s", err)
@@ -66,7 +68,7 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 		return
 	}
 
-	dropReason := f.firewall.Drop(*fwPacket, *fwCtx, false, hostinfo, f.pki.GetCAPool(), localCache)
+	dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason == nil {
 		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
 
@@ -120,10 +122,9 @@ func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *
 	f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, out, nb, packet, q)
 }
 
-// Handshake will attempt to initiate a tunnel with the provided vpn address. This is a no-op if the tunnel is already established or being established
-// it does not check if it is within our vpn networks!
+// Handshake will attempt to initiate a tunnel with the provided vpn address if it is within our vpn networks. This is a no-op if the tunnel is already established or being established
 func (f *Interface) Handshake(vpnAddr netip.Addr) {
-	f.handshakeManager.GetOrHandshake(vpnAddr, nil)
+	f.getOrHandshakeNoRouting(vpnAddr, nil)
 }
 
 // getOrHandshakeNoRouting returns nil if the vpnAddr is not routable.
@@ -139,6 +140,7 @@ func (f *Interface) getOrHandshakeNoRouting(vpnAddr netip.Addr, cacheCallback fu
 // getOrHandshakeConsiderRouting will try to find the HostInfo to handle this packet, starting a handshake if necessary.
 // If the 2nd return var is false then the hostinfo is not ready to be used in a tunnel.
 func (f *Interface) getOrHandshakeConsiderRouting(fwPacket *firewall.Packet, cacheCallback func(*HandshakeHostInfo)) (*HostInfo, bool) {
+
 	destinationAddr := fwPacket.RemoteAddr
 
 	hostinfo, ready := f.getOrHandshakeNoRouting(destinationAddr, cacheCallback)
@@ -211,15 +213,14 @@ func (f *Interface) getOrHandshakeConsiderRouting(fwPacket *firewall.Packet, cac
 
 func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte) {
 	fp := &firewall.Packet{}
-	ctx := &firewall.PacketContext{}
-	err := newPacket(p, false, fp, ctx)
+	err := newPacket(p, false, fp)
 	if err != nil {
 		f.l.Warnf("error while parsing outgoing packet for firewall check; %v", err)
 		return
 	}
 
 	// check if packet is in outbound fw rules
-	dropReason := f.firewall.Drop(*fp, *ctx, false, hostinfo, f.pki.GetCAPool(), nil)
+	dropReason := f.firewall.Drop(*fp, false, hostinfo, f.pki.GetCAPool(), nil)
 	if dropReason != nil {
 		if f.l.Level >= logrus.DebugLevel {
 			f.l.WithField("fwPacket", fp).
@@ -232,10 +233,9 @@ func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubTyp
 	f.sendNoMetrics(header.Message, st, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, p, nb, out, 0)
 }
 
-// SendMessageToVpnAddr handles real addr:port lookup and sends to the current best known address for vpnAddr.
-// This function ignores myVpnNetworksTable, and will always attempt to treat the address as a vpnAddr
+// SendMessageToVpnAddr handles real addr:port lookup and sends to the current best known address for vpnAddr
 func (f *Interface) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p, nb, out []byte) {
-	hostInfo, ready := f.handshakeManager.GetOrHandshake(vpnAddr, func(hh *HandshakeHostInfo) {
+	hostInfo, ready := f.getOrHandshakeNoRouting(vpnAddr, func(hh *HandshakeHostInfo) {
 		hh.cachePacket(f.l, t, st, p, f.SendMessageToHostInfo, f.cachedPacketMetrics)
 	})
 
@@ -337,9 +337,21 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 	if ci.eKey == nil {
 		return
 	}
-	useRelay := !remote.IsValid() && !hostinfo.remote.IsValid()
+	target := remote
+	if !target.IsValid() {
+		target = hostinfo.remote
+	}
+	useRelay := !target.IsValid()
 	fullOut := out
 
+	var pkt *overlay.Packet
+	if !useRelay && f.batches.Enabled() {
+		pkt = f.batches.newPacket()
+		if pkt != nil {
+			out = pkt.Payload()[:0]
+		}
+	}
+
 	if useRelay {
 		if len(out) < header.Len {
 			// out always has a capacity of mtu, but not always a length greater than the header.Len.
@@ -373,41 +385,85 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 	}
 
 	var err error
+	if len(p) > 0 && slicesOverlap(out, p) {
+		tmp := make([]byte, len(p))
+		copy(tmp, p)
+		p = tmp
+	}
 	out, err = ci.eKey.EncryptDanger(out, out, p, c, nb)
 	if noiseutil.EncryptLockNeeded {
 		ci.writeLock.Unlock()
 	}
 	if err != nil {
+		if pkt != nil {
+			pkt.Release()
+		}
 		hostinfo.logger(f.l).WithError(err).
-			WithField("udpAddr", remote).WithField("counter", c).
+			WithField("udpAddr", target).WithField("counter", c).
 			WithField("attemptedCounter", c).
 			Error("Failed to encrypt outgoing packet")
 		return
 	}
 
-	if remote.IsValid() {
-		err = f.writers[q].WriteTo(out, remote)
-		if err != nil {
-			hostinfo.logger(f.l).WithError(err).
-				WithField("udpAddr", remote).Error("Failed to write outgoing packet")
-		}
-	} else if hostinfo.remote.IsValid() {
-		err = f.writers[q].WriteTo(out, hostinfo.remote)
-		if err != nil {
-			hostinfo.logger(f.l).WithError(err).
-				WithField("udpAddr", remote).Error("Failed to write outgoing packet")
-		}
-	} else {
-		// Try to send via a relay
-		for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
-			relayHostInfo, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
-			if err != nil {
-				hostinfo.relayState.DeleteRelay(relayIP)
-				hostinfo.logger(f.l).WithField("relay", relayIP).WithError(err).Info("sendNoMetrics failed to find HostInfo")
-				continue
+	if target.IsValid() {
+		if pkt != nil {
+			pkt.Len = len(out)
+			if f.l.Level >= logrus.DebugLevel {
+				f.l.WithFields(logrus.Fields{
+					"queue":        q,
+					"dest":         target,
+					"payload_len":  pkt.Len,
+					"use_batches":  true,
+					"remote_index": hostinfo.remoteIndexId,
+				}).Debug("enqueueing packet to UDP batch queue")
 			}
-			f.SendVia(relayHostInfo, relay, out, nb, fullOut[:header.Len+len(out)], true)
-			break
+			if f.tryQueuePacket(q, pkt, target) {
+				return
+			}
+			if f.l.Level >= logrus.DebugLevel {
+				f.l.WithFields(logrus.Fields{
+					"queue": q,
+					"dest":  target,
+				}).Debug("failed to enqueue packet; falling back to immediate send")
+			}
+			f.writeImmediatePacket(q, pkt, target, hostinfo)
+			return
 		}
+		if f.tryQueueDatagram(q, out, target) {
+			return
+		}
+		f.writeImmediate(q, out, target, hostinfo)
+		return
+	}
+
+	// fall back to relay path
+	if pkt != nil {
+		pkt.Release()
+	}
+
+	// Try to send via a relay
+	for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
+		relayHostInfo, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
+		if err != nil {
+			hostinfo.relayState.DeleteRelay(relayIP)
+			hostinfo.logger(f.l).WithField("relay", relayIP).WithError(err).Info("sendNoMetrics failed to find HostInfo")
+			continue
+		}
+		f.SendVia(relayHostInfo, relay, out, nb, fullOut[:header.Len+len(out)], true)
+		break
 	}
 }
+
+// slicesOverlap reports whether the two byte slices share any portion of memory.
+// cipher.AEAD.Seal requires plaintext and dst to live in disjoint regions.
+func slicesOverlap(a, b []byte) bool {
+	if len(a) == 0 || len(b) == 0 {
+		return false
+	}
+
+	aStart := uintptr(unsafe.Pointer(&a[0]))
+	aEnd := aStart + uintptr(len(a))
+	bStart := uintptr(unsafe.Pointer(&b[0]))
+	bEnd := bStart + uintptr(len(b))
+	return aStart < bEnd && bStart < aEnd
+}

inside_bsd.go

@@ -1,4 +1,5 @@
 //go:build darwin || dragonfly || freebsd || netbsd || openbsd
+// +build darwin dragonfly freebsd netbsd openbsd
 
 package nebula
 

inside_generic.go

@@ -1,4 +1,5 @@
 //go:build !darwin && !dragonfly && !freebsd && !netbsd && !openbsd
+// +build !darwin,!dragonfly,!freebsd,!netbsd,!openbsd
 
 package nebula
 

lighthouse.go

@@ -360,8 +360,7 @@ func (lh *LightHouse) parseLighthouses(c *config.C) ([]netip.Addr, error) {
 		}
 
 		if !lh.myVpnNetworksTable.Contains(addr) {
-			lh.l.WithFields(m{"vpnAddr": addr, "networks": lh.myVpnNetworks}).
-				Warn("lighthouse host is not within our networks, lighthouse functionality will work but layer 3 network traffic to the lighthouse will not")
+			return nil, util.NewContextualError("lighthouse host is not in our networks, invalid", m{"vpnAddr": addr, "networks": lh.myVpnNetworks}, nil)
 		}
 		out[i] = addr
 	}
@@ -432,8 +431,7 @@ func (lh *LightHouse) loadStaticMap(c *config.C, staticList map[netip.Addr]struc
 		}
 
 		if !lh.myVpnNetworksTable.Contains(vpnAddr) {
-			lh.l.WithFields(m{"vpnAddr": vpnAddr, "networks": lh.myVpnNetworks, "entry": i + 1}).
-				Warn("static_host_map key is not within our networks, layer 3 network traffic to this host will not work")
+			return util.NewContextualError("static_host_map key is not in our network, invalid", m{"vpnAddr": vpnAddr, "networks": lh.myVpnNetworks, "entry": i + 1}, nil)
 		}
 
 		vals, ok := v.([]any)
@@ -713,14 +711,21 @@ func (lh *LightHouse) unlockedShouldAddV6(vpnAddr netip.Addr, to *V6AddrPort) bo
 
 func (lh *LightHouse) IsLighthouseAddr(vpnAddr netip.Addr) bool {
 	l := lh.GetLighthouses()
-	return slices.Contains(l, vpnAddr)
+	for i := range l {
+		if l[i] == vpnAddr {
+			return true
+		}
+	}
+	return false
 }
 
 func (lh *LightHouse) IsAnyLighthouseAddr(vpnAddrs []netip.Addr) bool {
 	l := lh.GetLighthouses()
 	for i := range vpnAddrs {
-		if slices.Contains(l, vpnAddrs[i]) {
-			return true
+		for j := range l {
+			if l[j] == vpnAddrs[i] {
+				return true
+			}
 		}
 	}
 	return false
@@ -1332,19 +1337,12 @@ func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, fromVpn
 		}
 	}
 
-	remoteAllowList := lhh.lh.GetRemoteAllowList()
 	for _, a := range n.Details.V4AddrPorts {
-		b := protoV4AddrPortToNetAddrPort(a)
-		if remoteAllowList.Allow(detailsVpnAddr, b.Addr()) {
-			punch(b, detailsVpnAddr)
-		}
+		punch(protoV4AddrPortToNetAddrPort(a), detailsVpnAddr)
 	}
 
 	for _, a := range n.Details.V6AddrPorts {
-		b := protoV6AddrPortToNetAddrPort(a)
-		if remoteAllowList.Allow(detailsVpnAddr, b.Addr()) {
-			punch(b, detailsVpnAddr)
-		}
+		punch(protoV6AddrPortToNetAddrPort(a), detailsVpnAddr)
 	}
 
 	// This sends a nebula test packet to the host trying to contact us. In the case

lighthouse_test.go

@@ -1,6 +1,7 @@
 package nebula
 
 import (
+	"context"
 	"encoding/binary"
 	"fmt"
 	"net/netip"
@@ -13,7 +14,7 @@ import (
 	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.yaml.in/yaml/v3"
+	"gopkg.in/yaml.v3"
 )
 
 func TestOldIPv4Only(t *testing.T) {
@@ -41,14 +42,14 @@ func Test_lhStaticMapping(t *testing.T) {
 	c := config.NewC(l)
 	c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1}}
 	c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}}
-	_, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
+	_, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
 	require.NoError(t, err)
 
 	lh2 := "10.128.0.3"
 	c = config.NewC(l)
 	c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1, lh2}}
 	c.Settings["static_host_map"] = map[string]any{lh1: []any{"100.1.1.1:4242"}}
-	_, err = NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
+	_, err = NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
 	require.EqualError(t, err, "lighthouse 10.128.0.3 does not have a static_host_map entry")
 }
 
@@ -70,7 +71,7 @@ func TestReloadLighthouseInterval(t *testing.T) {
 	}
 
 	c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}}
-	lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
+	lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
 	require.NoError(t, err)
 	lh.ifce = &mockEncWriter{}
 
@@ -98,7 +99,7 @@ func BenchmarkLighthouseHandleRequest(b *testing.B) {
 	}
 
 	c := config.NewC(l)
-	lh, err := NewLightHouseFromConfig(b.Context(), l, c, cs, nil, nil)
+	lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
 	require.NoError(b, err)
 
 	hAddr := netip.MustParseAddrPort("4.5.6.7:12345")
@@ -201,7 +202,7 @@ func TestLighthouse_Memory(t *testing.T) {
 		myVpnNetworks:      []netip.Prefix{myVpnNet},
 		myVpnNetworksTable: nt,
 	}
-	lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
+	lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
 	lh.ifce = &mockEncWriter{}
 	require.NoError(t, err)
 	lhh := lh.NewRequestHandler()
@@ -287,7 +288,7 @@ func TestLighthouse_reload(t *testing.T) {
 		myVpnNetworksTable: nt,
 	}
 
-	lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
+	lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
 	require.NoError(t, err)
 
 	nc := map[string]any{
@@ -522,7 +523,7 @@ func TestLighthouse_Dont_Delete_Static_Hosts(t *testing.T) {
 		myVpnNetworks:      []netip.Prefix{myVpnNet},
 		myVpnNetworksTable: nt,
 	}
-	lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
+	lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
 	require.NoError(t, err)
 	lh.ifce = &mockEncWriter{}
 
@@ -588,7 +589,7 @@ func TestLighthouse_DeletesWork(t *testing.T) {
 		myVpnNetworks:      []netip.Prefix{myVpnNet},
 		myVpnNetworksTable: nt,
 	}
-	lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
+	lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
 	require.NoError(t, err)
 	lh.ifce = &mockEncWriter{}
 

main.go

@@ -5,8 +5,7 @@ import (
 	"fmt"
 	"net"
 	"net/netip"
-	"runtime/debug"
-	"strings"
+	"runtime"
 	"time"
 
 	"github.com/sirupsen/logrus"
@@ -15,7 +14,7 @@ import (
 	"github.com/slackhq/nebula/sshd"
 	"github.com/slackhq/nebula/udp"
 	"github.com/slackhq/nebula/util"
-	"go.yaml.in/yaml/v3"
+	"gopkg.in/yaml.v3"
 )
 
 type m = map[string]any
@@ -29,10 +28,6 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		}
 	}()
 
-	if buildVersion == "" {
-		buildVersion = moduleVersion()
-	}
-
 	l := logger
 	l.Formatter = &logrus.TextFormatter{
 		FullTimestamp: true,
@@ -81,8 +76,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	if c.GetBool("sshd.enabled", false) {
 		sshStart, err = configSSH(l, ssh, c)
 		if err != nil {
-			l.WithError(err).Warn("Failed to configure sshd, ssh debugging will not be available")
-			sshStart = nil
+			return nil, util.ContextualizeIfNeeded("Error while configuring the sshd", err)
 		}
 	}
 
@@ -105,7 +99,11 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		// deprecated and undocumented
 		tunQueues := c.GetInt("tun.routines", 1)
 		udpQueues := c.GetInt("listen.routines", 1)
-		routines = max(tunQueues, udpQueues)
+		if tunQueues > udpQueues {
+			routines = tunQueues
+		} else {
+			routines = udpQueues
+		}
 		if routines != 1 {
 			l.WithField("routines", routines).Warn("Setting tun.routines and listen.routines is deprecated. Use `routines` instead")
 		}
@@ -146,6 +144,20 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	// set up our UDP listener
 	udpConns := make([]udp.Conn, routines)
 	port := c.GetInt("listen.port", 0)
+	enableGSO := c.GetBool("listen.enable_gso", true)
+	enableGRO := c.GetBool("listen.enable_gro", true)
+	gsoMaxSegments := c.GetInt("listen.gso_max_segments", defaultGSOMaxSegments)
+	if gsoMaxSegments <= 0 {
+		gsoMaxSegments = defaultGSOMaxSegments
+	}
+	if gsoMaxSegments > maxKernelGSOSegments {
+		gsoMaxSegments = maxKernelGSOSegments
+	}
+	gsoFlushTimeout := c.GetDuration("listen.gso_flush_timeout", defaultGSOFlushInterval)
+	if gsoFlushTimeout < 0 {
+		gsoFlushTimeout = 0
+	}
+	batchQueueDepth := c.GetInt("batch.queue_depth", 0)
 
 	if !configTest {
 		rawListenHost := c.GetString("listen.host", "0.0.0.0")
@@ -165,13 +177,27 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 			listenHost = ips[0].Unmap()
 		}
 
+		useWGDefault := runtime.GOOS == "linux"
+		useWG := c.GetBool("listen.use_wireguard_stack", useWGDefault)
+		var mkListener func(*logrus.Logger, netip.Addr, int, bool, int) (udp.Conn, error)
+		if useWG {
+			mkListener = udp.NewWireguardListener
+		} else {
+			mkListener = udp.NewListener
+		}
+
 		for i := 0; i < routines; i++ {
 			l.Infof("listening on %v", netip.AddrPortFrom(listenHost, uint16(port)))
-			udpServer, err := udp.NewListener(l, listenHost, port, routines > 1, c.GetInt("listen.batch", 64))
+			udpServer, err := mkListener(l, listenHost, port, routines > 1, c.GetInt("listen.batch", 64))
 			if err != nil {
 				return nil, util.NewContextualError("Failed to open udp listener", m{"queue": i}, err)
 			}
 			udpServer.ReloadConfig(c)
+			if cfg, ok := udpServer.(interface {
+				ConfigureOffload(bool, bool, int)
+			}); ok {
+				cfg.ConfigureOffload(enableGSO, enableGRO, gsoMaxSegments)
+			}
 			udpConns[i] = udpServer
 
 			// If port is dynamic, discover it before the next pass through the for loop
@@ -215,9 +241,13 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	handshakeManager := NewHandshakeManager(l, hostMap, lightHouse, udpConns[0], handshakeConfig)
 	lightHouse.handshakeTrigger = handshakeManager.trigger
 
-	ds, err := newDnsServerFromConfig(ctx, l, pki.getCertState(), hostMap, c)
-	if err != nil {
-		l.WithError(err).Warn("Failed to start DNS responder")
+	serveDns := false
+	if c.GetBool("lighthouse.serve_dns", false) {
+		if c.GetBool("lighthouse.am_lighthouse", false) {
+			serveDns = true
+		} else {
+			l.Warn("DNS server refusing to run because this host is not a lighthouse.")
+		}
 	}
 
 	ifConfig := &InterfaceConfig{
@@ -226,7 +256,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		Outside:               udpConns[0],
 		pki:                   pki,
 		Firewall:              fw,
-		DnsServer:             ds,
+		ServeDns:              serveDns,
 		HandshakeManager:      handshakeManager,
 		connectionManager:     connManager,
 		lightHouse:            lightHouse,
@@ -235,12 +265,17 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		reQueryWait:           c.GetDuration("timers.requery_wait_duration", defaultReQueryWait),
 		DropLocalBroadcast:    c.GetBool("tun.drop_local_broadcast", false),
 		DropMulticast:         c.GetBool("tun.drop_multicast", false),
+		EnableGSO:             enableGSO,
+		EnableGRO:             enableGRO,
+		GSOMaxSegments:        gsoMaxSegments,
 		routines:              routines,
 		MessageMetrics:        messageMetrics,
 		version:               buildVersion,
 		relayManager:          NewRelayManager(ctx, l, hostMap, c),
 		punchy:                punchy,
 		ConntrackCacheTimeout: conntrackCacheTimeout,
+		BatchFlushInterval:    gsoFlushTimeout,
+		BatchQueueDepth:       batchQueueDepth,
 		l:                     l,
 	}
 
@@ -252,12 +287,12 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		}
 
 		ifce.writers = udpConns
+		ifce.applyOffloadConfig(enableGSO, enableGRO, gsoMaxSegments)
 		lightHouse.ifce = ifce
 
 		ifce.RegisterConfigChangeCallbacks(c)
 		ifce.reloadDisconnectInvalid(c)
 		ifce.reloadSendRecvError(c)
-		ifce.reloadAcceptRecvError(c)
 
 		handshakeManager.f = ifce
 		go handshakeManager.Run(ctx)
@@ -276,31 +311,22 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 
 	attachCommands(l, c, ssh, ifce)
 
+	// Start DNS server last to allow using the nebula IP as lighthouse.dns.host
+	var dnsStart func()
+	if lightHouse.amLighthouse && serveDns {
+		l.Debugln("Starting dns server")
+		dnsStart = dnsMain(l, pki.getCertState(), hostMap, c)
+	}
+
 	return &Control{
-		state:                  StateReady,
-		f:                      ifce,
-		l:                      l,
-		ctx:                    ctx,
-		cancel:                 cancel,
-		sshStart:               sshStart,
-		statsStart:             statsStart,
-		dnsStart:               ds.Start,
-		lighthouseStart:        lightHouse.StartUpdateWorker,
-		connectionManagerStart: connManager.Start,
+		ifce,
+		l,
+		ctx,
+		cancel,
+		sshStart,
+		statsStart,
+		dnsStart,
+		lightHouse.StartUpdateWorker,
+		connManager.Start,
 	}, nil
 }
-
-func moduleVersion() string {
-	info, ok := debug.ReadBuildInfo()
-	if !ok {
-		return ""
-	}
-
-	for _, dep := range info.Deps {
-		if dep.Path == "github.com/slackhq/nebula" {
-			return strings.TrimPrefix(dep.Version, "v")
-		}
-	}
-
-	return ""
-}

noiseutil/boring.go

@@ -22,7 +22,7 @@ const EncryptLockNeeded = true
 // NewGCMTLS is no longer exposed in go1.19+, so we need to link it in
 // See: https://github.com/golang/go/issues/56326
 //
-// NewGCMTLS is the internal method used with boringcrypto that provides a
+// NewGCMTLS is the internal method used with boringcrypto that provices a
 // validated mode of AES-GCM which enforces the nonce is strictly
 // monotonically increasing.  This is the TLS 1.2 specification for nonce
 // generation (which also matches the method used by the Noise Protocol)

notboring.go

@@ -1,4 +1,5 @@
 //go:build !boringcrypto
+// +build !boringcrypto
 
 package nebula
 

outside.go

@@ -12,6 +12,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
+	"github.com/slackhq/nebula/overlay"
 	"golang.org/x/net/ipv4"
 )
 
@@ -19,21 +20,21 @@ const (
 	minFwPacketLen = 4
 )
 
-func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, fwCtx *firewall.PacketContext, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
+func (f *Interface) readOutsidePackets(ip netip.AddrPort, via *ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache *firewall.ConntrackCache) {
 	err := h.Parse(packet)
 	if err != nil {
 		// Hole punch packets are 0 or 1 byte big, so lets ignore printing those errors
 		if len(packet) > 1 {
-			f.l.WithField("packet", packet).Infof("Error while parsing inbound packet from %s: %s", via, err)
+			f.l.WithField("packet", packet).Infof("Error while parsing inbound packet from %s: %s", ip, err)
 		}
 		return
 	}
 
 	//l.Error("in packet ", header, packet[HeaderLen:])
-	if !via.IsRelayed {
-		if f.myVpnNetworksTable.Contains(via.UdpAddr.Addr()) {
+	if ip.IsValid() {
+		if f.myVpnNetworksTable.Contains(ip.Addr()) {
 			if f.l.Level >= logrus.DebugLevel {
-				f.l.WithField("from", via).Debug("Refusing to process double encrypted packet")
+				f.l.WithField("udpAddr", ip).Debug("Refusing to process double encrypted packet")
 			}
 			return
 		}
@@ -54,13 +55,14 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 
 	switch h.Type {
 	case header.Message:
-		if !f.handleEncrypted(ci, via, h) {
+		// TODO handleEncrypted sends directly to addr on error. Handle this in the tunneling case.
+		if !f.handleEncrypted(ci, ip, h) {
 			return
 		}
 
 		switch h.Subtype {
 		case header.MessageNone:
-			if !f.decryptToTun(hostinfo, h.MessageCounter, out, packet, fwPacket, fwCtx, nb, q, localCache) {
+			if !f.decryptToTun(hostinfo, h.MessageCounter, out, packet, fwPacket, nb, q, localCache, ip, h.RemoteIndex) {
 				return
 			}
 		case header.MessageRelay:
@@ -78,7 +80,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 			// Successfully validated the thing. Get rid of the Relay header.
 			signedPayload = signedPayload[header.Len:]
 			// Pull the Roaming parts up here, and return in all call paths.
-			f.handleHostRoaming(hostinfo, via)
+			f.handleHostRoaming(hostinfo, ip)
 			// Track usage of both the HostInfo and the Relay for the received & authenticated packet
 			f.connectionManager.In(hostinfo)
 			f.connectionManager.RelayUsed(h.RemoteIndex)
@@ -95,14 +97,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 			case TerminalType:
 				// If I am the target of this relay, process the unwrapped packet
 				// From this recursive point, all these variables are 'burned'. We shouldn't rely on them again.
-				via = ViaSender{
-					UdpAddr:   via.UdpAddr,
-					relayHI:   hostinfo,
-					remoteIdx: relay.RemoteIndex,
-					relay:     relay,
-					IsRelayed: true,
-				}
-				f.readOutsidePackets(via, out[:0], signedPayload, h, fwPacket, fwCtx, lhf, nb, q, localCache)
+				f.readOutsidePackets(netip.AddrPort{}, &ViaSender{relayHI: hostinfo, remoteIdx: relay.RemoteIndex, relay: relay}, out[:0], signedPayload, h, fwPacket, lhf, nb, q, localCache)
 				return
 			case ForwardingType:
 				// Find the target HostInfo relay object
@@ -132,32 +127,31 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 
 	case header.LightHouse:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		if !f.handleEncrypted(ci, via, h) {
+		if !f.handleEncrypted(ci, ip, h) {
 			return
 		}
 
 		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("from", via).
+			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", ip).
 				WithField("packet", packet).
 				Error("Failed to decrypt lighthouse packet")
 			return
 		}
 
-		//TODO: assert via is not relayed
-		lhf.HandleRequest(via.UdpAddr, hostinfo.vpnAddrs, d, f)
+		lhf.HandleRequest(ip, hostinfo.vpnAddrs, d, f)
 
 		// Fallthrough to the bottom to record incoming traffic
 
 	case header.Test:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		if !f.handleEncrypted(ci, via, h) {
+		if !f.handleEncrypted(ci, ip, h) {
 			return
 		}
 
 		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("from", via).
+			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", ip).
 				WithField("packet", packet).
 				Error("Failed to decrypt test packet")
 			return
@@ -166,7 +160,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 		if h.Subtype == header.TestRequest {
 			// This testRequest might be from TryPromoteBest, so we should roam
 			// to the new IP address before responding
-			f.handleHostRoaming(hostinfo, via)
+			f.handleHostRoaming(hostinfo, ip)
 			f.send(header.Test, header.TestReply, ci, hostinfo, d, nb, out)
 		}
 
@@ -177,41 +171,34 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 
 	case header.Handshake:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		f.handshakeManager.HandleIncoming(via, packet, h)
+		f.handshakeManager.HandleIncoming(ip, via, packet, h)
 		return
 
 	case header.RecvError:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		f.handleRecvError(via.UdpAddr, h)
+		f.handleRecvError(ip, h)
 		return
 
 	case header.CloseTunnel:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		if !f.handleEncrypted(ci, via, h) {
-			return
-		}
-		_, err = f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
-		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("from", via).
-				WithField("packet", packet).
-				Error("Failed to decrypt CloseTunnel packet")
+		if !f.handleEncrypted(ci, ip, h) {
 			return
 		}
 
-		hostinfo.logger(f.l).WithField("from", via).
+		hostinfo.logger(f.l).WithField("udpAddr", ip).
 			Info("Close tunnel received, tearing down.")
 
 		f.closeTunnel(hostinfo)
 		return
 
 	case header.Control:
-		if !f.handleEncrypted(ci, via, h) {
+		if !f.handleEncrypted(ci, ip, h) {
 			return
 		}
 
 		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("from", via).
+			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", ip).
 				WithField("packet", packet).
 				Error("Failed to decrypt Control packet")
 			return
@@ -221,11 +208,11 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 
 	default:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		hostinfo.logger(f.l).Debugf("Unexpected packet received from %s", via)
+		hostinfo.logger(f.l).Debugf("Unexpected packet received from %s", ip)
 		return
 	}
 
-	f.handleHostRoaming(hostinfo, via)
+	f.handleHostRoaming(hostinfo, ip)
 
 	f.connectionManager.In(hostinfo)
 }
@@ -244,36 +231,36 @@ func (f *Interface) sendCloseTunnel(h *HostInfo) {
 	f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, make([]byte, 12, 12), make([]byte, mtu))
 }
 
-func (f *Interface) handleHostRoaming(hostinfo *HostInfo, via ViaSender) {
-	if !via.IsRelayed && hostinfo.remote != via.UdpAddr {
-		if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, via.UdpAddr.Addr()) {
-			hostinfo.logger(f.l).WithField("newAddr", via.UdpAddr).Debug("lighthouse.remote_allow_list denied roaming")
+func (f *Interface) handleHostRoaming(hostinfo *HostInfo, udpAddr netip.AddrPort) {
+	if udpAddr.IsValid() && hostinfo.remote != udpAddr {
+		if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, udpAddr.Addr()) {
+			hostinfo.logger(f.l).WithField("newAddr", udpAddr).Debug("lighthouse.remote_allow_list denied roaming")
 			return
 		}
 
-		if !hostinfo.lastRoam.IsZero() && via.UdpAddr == hostinfo.lastRoamRemote && time.Since(hostinfo.lastRoam) < RoamingSuppressSeconds*time.Second {
+		if !hostinfo.lastRoam.IsZero() && udpAddr == hostinfo.lastRoamRemote && time.Since(hostinfo.lastRoam) < RoamingSuppressSeconds*time.Second {
 			if f.l.Level >= logrus.DebugLevel {
-				hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", via.UdpAddr).
+				hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", udpAddr).
 					Debugf("Suppressing roam back to previous remote for %d seconds", RoamingSuppressSeconds)
 			}
 			return
 		}
 
-		hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", via.UdpAddr).
+		hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", udpAddr).
 			Info("Host roamed to new udp ip/port.")
 		hostinfo.lastRoam = time.Now()
 		hostinfo.lastRoamRemote = hostinfo.remote
-		hostinfo.SetRemote(via.UdpAddr)
+		hostinfo.SetRemote(udpAddr)
 	}
 
 }
 
 // handleEncrypted returns true if a packet should be processed, false otherwise
-func (f *Interface) handleEncrypted(ci *ConnectionState, via ViaSender, h *header.H) bool {
+func (f *Interface) handleEncrypted(ci *ConnectionState, addr netip.AddrPort, h *header.H) bool {
 	// If connectionstate does not exist, send a recv error, if possible, to encourage a fast reconnect
 	if ci == nil {
-		if !via.IsRelayed {
-			f.maybeSendRecvError(via.UdpAddr, h.RemoteIndex)
+		if addr.IsValid() {
+			f.maybeSendRecvError(addr, h.RemoteIndex)
 		}
 		return false
 	}
@@ -295,10 +282,7 @@ var (
 )
 
 // newPacket validates and parses the interesting bits for the firewall out of the ip and sub protocol headers
-func newPacket(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.PacketContext) error {
-	if ctx != nil {
-		*ctx = firewall.PacketContext{}
-	}
+func newPacket(data []byte, incoming bool, fp *firewall.Packet) error {
 	if len(data) < 1 {
 		return ErrPacketTooShort
 	}
@@ -306,14 +290,14 @@ func newPacket(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.Pa
 	version := int((data[0] >> 4) & 0x0f)
 	switch version {
 	case ipv4.Version:
-		return parseV4(data, incoming, fp, ctx)
+		return parseV4(data, incoming, fp)
 	case ipv6.Version:
-		return parseV6(data, incoming, fp, ctx)
+		return parseV6(data, incoming, fp)
 	}
 	return ErrUnknownIPVersion
 }
 
-func parseV6(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.PacketContext) error {
+func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 	dataLen := len(data)
 	if dataLen < ipv6.HeaderLen {
 		return ErrIPv6PacketTooShort
@@ -337,34 +321,13 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.Pack
 		proto := layers.IPProtocol(data[protoAt])
 
 		switch proto {
-		case layers.IPProtocolESP, layers.IPProtocolNoNextHeader:
+		case layers.IPProtocolICMPv6, layers.IPProtocolESP, layers.IPProtocolNoNextHeader:
 			fp.Protocol = uint8(proto)
 			fp.RemotePort = 0
 			fp.LocalPort = 0
 			fp.Fragment = false
 			return nil
 
-		case layers.IPProtocolICMPv6:
-			if dataLen < offset+6 {
-				return ErrIPv6PacketTooShort
-			}
-			fp.Protocol = uint8(proto)
-			fp.LocalPort = 0 //incoming vs outgoing doesn't matter for icmpv6
-			icmptype := data[offset+1]
-			switch icmptype {
-			case layers.ICMPv6TypeEchoRequest, layers.ICMPv6TypeEchoReply:
-				fp.RemotePort = binary.BigEndian.Uint16(data[offset+4 : offset+6]) //identifier
-			default:
-				fp.RemotePort = 0
-			}
-			fp.Fragment = false
-			if ctx != nil {
-				ctx.Length = binary.BigEndian.Uint16(data[4:6]) + uint16(ipv6.HeaderLen)
-				ctx.ICMPType = data[offset]
-				ctx.ICMPCode = data[offset+1]
-			}
-			return nil
-
 		case layers.IPProtocolTCP, layers.IPProtocolUDP:
 			if dataLen < offset+4 {
 				return ErrIPv6PacketTooShort
@@ -380,12 +343,6 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.Pack
 			}
 
 			fp.Fragment = false
-			if ctx != nil {
-				ctx.Length = binary.BigEndian.Uint16(data[4:6]) + uint16(ipv6.HeaderLen)
-				if proto == layers.IPProtocolTCP && dataLen >= offset+14 {
-					ctx.TCPFlags = data[offset+13]
-				}
-			}
 			return nil
 
 		case layers.IPProtocolIPv6Fragment:
@@ -437,7 +394,7 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.Pack
 	return ErrIPv6CouldNotFindPayload
 }
 
-func parseV4(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.PacketContext) error {
+func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
 	// Do we at least have an ipv4 header worth of data?
 	if len(data) < ipv4.HeaderLen {
 		return ErrIPv4PacketTooShort
@@ -460,52 +417,33 @@ func parseV4(data []byte, incoming bool, fp *firewall.Packet, ctx *firewall.Pack
 
 	// Accounting for a variable header length, do we have enough data for our src/dst tuples?
 	minLen := ihl
-	if !fp.Fragment {
-		if fp.Protocol == firewall.ProtoICMP {
-			minLen += minFwPacketLen + 2
-		} else {
-			minLen += minFwPacketLen
-		}
+	if !fp.Fragment && fp.Protocol != firewall.ProtoICMP {
+		minLen += minFwPacketLen
 	}
-
 	if len(data) < minLen {
 		return ErrIPv4InvalidHeaderLength
 	}
 
-	if incoming { // Firewall packets are locally oriented
+	// Firewall packets are locally oriented
+	if incoming {
 		fp.RemoteAddr, _ = netip.AddrFromSlice(data[12:16])
 		fp.LocalAddr, _ = netip.AddrFromSlice(data[16:20])
+		if fp.Fragment || fp.Protocol == firewall.ProtoICMP {
+			fp.RemotePort = 0
+			fp.LocalPort = 0
+		} else {
+			fp.RemotePort = binary.BigEndian.Uint16(data[ihl : ihl+2])
+			fp.LocalPort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4])
+		}
 	} else {
 		fp.LocalAddr, _ = netip.AddrFromSlice(data[12:16])
 		fp.RemoteAddr, _ = netip.AddrFromSlice(data[16:20])
-	}
-
-	if fp.Fragment {
-		fp.RemotePort = 0
-		fp.LocalPort = 0
-	} else if fp.Protocol == firewall.ProtoICMP { //note that orientation doesn't matter on ICMP
-		fp.RemotePort = binary.BigEndian.Uint16(data[ihl+4 : ihl+6]) //identifier
-		fp.LocalPort = 0                                             //code would be uint16(data[ihl+1])
-	} else if incoming {
-		fp.RemotePort = binary.BigEndian.Uint16(data[ihl : ihl+2])  //src port
-		fp.LocalPort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
-	} else {
-		fp.LocalPort = binary.BigEndian.Uint16(data[ihl : ihl+2])    //src port
-		fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
-	}
-
-	if ctx != nil {
-		ctx.Length = binary.BigEndian.Uint16(data[2:4])
-		if !fp.Fragment {
-			switch fp.Protocol {
-			case firewall.ProtoICMP:
-				ctx.ICMPType = data[ihl]
-				ctx.ICMPCode = data[ihl+1]
-			case firewall.ProtoTCP:
-				if len(data) >= ihl+14 {
-					ctx.TCPFlags = data[ihl+13]
-				}
-			}
+		if fp.Fragment || fp.Protocol == firewall.ProtoICMP {
+			fp.RemotePort = 0
+			fp.LocalPort = 0
+		} else {
+			fp.LocalPort = binary.BigEndian.Uint16(data[ihl : ihl+2])
+			fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4])
 		}
 	}
 
@@ -528,30 +466,55 @@ func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, out []byte, packet []
 	return out, nil
 }
 
-func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out []byte, packet []byte, fwPacket *firewall.Packet, fwCtx *firewall.PacketContext, nb []byte, q int, localCache firewall.ConntrackCache) bool {
-	var err error
+func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache *firewall.ConntrackCache, addr netip.AddrPort, recvIndex uint32) bool {
+	var (
+		err error
+		pkt *overlay.Packet
+	)
+
+	if f.batches.tunQueue(q) != nil {
+		pkt = f.batches.newPacket()
+		if pkt != nil {
+			out = pkt.Payload()[:0]
+		}
+	}
 
 	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
 	if err != nil {
+		if pkt != nil {
+			pkt.Release()
+		}
 		hostinfo.logger(f.l).WithError(err).Error("Failed to decrypt packet")
+		if addr.IsValid() {
+			f.maybeSendRecvError(addr, recvIndex)
+		}
 		return false
 	}
 
-	err = newPacket(out, true, fwPacket, fwCtx)
+	err = newPacket(out, true, fwPacket)
 	if err != nil {
+		if pkt != nil {
+			pkt.Release()
+		}
 		hostinfo.logger(f.l).WithError(err).WithField("packet", out).
 			Warnf("Error while validating inbound packet")
 		return false
 	}
 
 	if !hostinfo.ConnectionState.window.Update(f.l, messageCounter) {
+		if pkt != nil {
+			pkt.Release()
+		}
 		hostinfo.logger(f.l).WithField("fwPacket", fwPacket).
 			Debugln("dropping out of window packet")
 		return false
 	}
 
-	dropReason := f.firewall.Drop(*fwPacket, *fwCtx, true, hostinfo, f.pki.GetCAPool(), localCache)
+	dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason != nil {
+		if pkt != nil {
+			pkt.Release()
+		}
 		// NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore
 		// This gives us a buffer to build the reject packet in
 		f.rejectOutside(out, hostinfo.ConnectionState, hostinfo, nb, packet, q)
@@ -564,15 +527,24 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
 	}
 
 	f.connectionManager.In(hostinfo)
-	_, err = f.readers[q].Write(out)
-	if err != nil {
+	if pkt != nil {
+		pkt.Len = len(out)
+		if f.batches.enqueueTun(q, pkt) {
+			f.observeTunQueueLen(q)
+			return true
+		}
+		f.writePacketToTun(q, pkt)
+		return true
+	}
+
+	if _, err = f.readers[q].Write(out); err != nil {
 		f.l.WithError(err).Error("Failed to write to tun")
 	}
 	return true
 }
 
 func (f *Interface) maybeSendRecvError(endpoint netip.AddrPort, index uint32) {
-	if f.sendRecvErrorConfig.ShouldRecvError(endpoint) {
+	if f.sendRecvErrorConfig.ShouldSendRecvError(endpoint) {
 		f.sendRecvError(endpoint, index)
 	}
 }
@@ -590,13 +562,6 @@ func (f *Interface) sendRecvError(endpoint netip.AddrPort, index uint32) {
 }
 
 func (f *Interface) handleRecvError(addr netip.AddrPort, h *header.H) {
-	if !f.acceptRecvErrorConfig.ShouldRecvError(addr) {
-		f.l.WithField("index", h.RemoteIndex).
-			WithField("udpAddr", addr).
-			Debug("Recv error received, ignoring")
-		return
-	}
-
 	if f.l.Level >= logrus.DebugLevel {
 		f.l.WithField("index", h.RemoteIndex).
 			WithField("udpAddr", addr).

outside_test.go

@@ -20,13 +20,13 @@ func Test_newPacket(t *testing.T) {
 	p := &firewall.Packet{}
 
 	// length fails
-	err := newPacket([]byte{}, true, p, nil)
+	err := newPacket([]byte{}, true, p)
 	require.ErrorIs(t, err, ErrPacketTooShort)
 
-	err = newPacket([]byte{0x40}, true, p, nil)
+	err = newPacket([]byte{0x40}, true, p)
 	require.ErrorIs(t, err, ErrIPv4PacketTooShort)
 
-	err = newPacket([]byte{0x60}, true, p, nil)
+	err = newPacket([]byte{0x60}, true, p)
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 
 	// length fail with ip options
@@ -39,15 +39,15 @@ func Test_newPacket(t *testing.T) {
 	}
 
 	b, _ := h.Marshal()
-	err = newPacket(b, true, p, nil)
+	err = newPacket(b, true, p)
 	require.ErrorIs(t, err, ErrIPv4InvalidHeaderLength)
 
 	// not an ipv4 packet
-	err = newPacket([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p, nil)
+	err = newPacket([]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p)
 	require.ErrorIs(t, err, ErrUnknownIPVersion)
 
 	// invalid ihl
-	err = newPacket([]byte{4<<4 | (8 >> 2 & 0x0f), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p, nil)
+	err = newPacket([]byte{4<<4 | (8 >> 2 & 0x0f), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}, true, p)
 	require.ErrorIs(t, err, ErrIPv4InvalidHeaderLength)
 
 	// account for variable ip header length - incoming
@@ -62,7 +62,7 @@ func Test_newPacket(t *testing.T) {
 
 	b, _ = h.Marshal()
 	b = append(b, []byte{0, 3, 0, 4}...)
-	err = newPacket(b, true, p, nil)
+	err = newPacket(b, true, p)
 
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol)
@@ -84,7 +84,7 @@ func Test_newPacket(t *testing.T) {
 
 	b, _ = h.Marshal()
 	b = append(b, []byte{0, 5, 0, 6}...)
-	err = newPacket(b, false, p, nil)
+	err = newPacket(b, false, p)
 
 	require.NoError(t, err)
 	assert.Equal(t, uint8(2), p.Protocol)
@@ -114,7 +114,7 @@ func Test_newPacket_v6(t *testing.T) {
 	err := gopacket.SerializeLayers(buffer, opt, &ip)
 	require.NoError(t, err)
 
-	err = newPacket(buffer.Bytes(), true, p, nil)
+	err = newPacket(buffer.Bytes(), true, p)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 
 	// A v6 packet with a hop-by-hop extension
@@ -148,14 +148,13 @@ func Test_newPacket_v6(t *testing.T) {
 
 	// A full IPv6 header and 1 byte in the first extension, but missing
 	// the length byte.
-	err = newPacket(buffer.Bytes()[:41], true, p, nil)
+	err = newPacket(buffer.Bytes()[:41], true, p)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 
 	// A full IPv6 header plus 1 full extension, but only 1 byte of the
 	// next layer, missing length byte
-	err = newPacket(buffer.Bytes()[:49], true, p, nil)
+	err = newPacket(buffer.Bytes()[:49], true, p)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
-	err = nil
 
 	// A good ICMP packet
 	ip = layers.IPv6{
@@ -166,33 +165,27 @@ func Test_newPacket_v6(t *testing.T) {
 		DstIP:      net.IPv6linklocalallnodes,
 	}
 
-	icmp := layers.ICMPv6{
-		TypeCode: layers.ICMPv6TypeEchoRequest,
-		Checksum: 0x1234,
-	}
+	icmp := layers.ICMPv6{}
 
 	buffer.Clear()
-	require.NoError(t, gopacket.SerializeLayers(buffer, opt, &ip, &icmp))
-	require.Error(t, newPacket(buffer.Bytes(), true, p, nil))
-
-	buffer.Clear()
-	echo := layers.ICMPv6Echo{
-		Identifier: 0xabcd,
-		SeqNumber:  1234,
+	err = gopacket.SerializeLayers(buffer, opt, &ip, &icmp)
+	if err != nil {
+		panic(err)
 	}
-	require.NoError(t, gopacket.SerializeLayers(buffer, opt, &ip, &icmp, &echo))
-	require.NoError(t, newPacket(buffer.Bytes(), true, p, nil))
+
+	err = newPacket(buffer.Bytes(), true, p)
+	require.NoError(t, err)
 	assert.Equal(t, uint8(layers.IPProtocolICMPv6), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr)
-	assert.Equal(t, uint16(0xabcd), p.RemotePort)
+	assert.Equal(t, uint16(0), p.RemotePort)
 	assert.Equal(t, uint16(0), p.LocalPort)
 	assert.False(t, p.Fragment)
 
 	// A good ESP packet
 	b := buffer.Bytes()
 	b[6] = byte(layers.IPProtocolESP)
-	err = newPacket(b, true, p, nil)
+	err = newPacket(b, true, p)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(layers.IPProtocolESP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -204,7 +197,7 @@ func Test_newPacket_v6(t *testing.T) {
 	// A good None packet
 	b = buffer.Bytes()
 	b[6] = byte(layers.IPProtocolNoNextHeader)
-	err = newPacket(b, true, p, nil)
+	err = newPacket(b, true, p)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(layers.IPProtocolNoNextHeader), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -216,7 +209,7 @@ func Test_newPacket_v6(t *testing.T) {
 	// An unknown protocol packet
 	b = buffer.Bytes()
 	b[6] = 255 // 255 is a reserved protocol number
-	err = newPacket(b, true, p, nil)
+	err = newPacket(b, true, p)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 
 	// A good UDP packet
@@ -243,7 +236,7 @@ func Test_newPacket_v6(t *testing.T) {
 	b = buffer.Bytes()
 
 	// incoming
-	err = newPacket(b, true, p, nil)
+	err = newPacket(b, true, p)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -253,7 +246,7 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// outgoing
-	err = newPacket(b, false, p, nil)
+	err = newPacket(b, false, p)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
@@ -263,14 +256,14 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// Too short UDP packet
-	err = newPacket(b[:len(b)-10], false, p, nil) // pull off the last 10 bytes
+	err = newPacket(b[:len(b)-10], false, p) // pull off the last 10 bytes
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 
 	// A good TCP packet
 	b[6] = byte(layers.IPProtocolTCP)
 
 	// incoming
-	err = newPacket(b, true, p, nil)
+	err = newPacket(b, true, p)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -280,7 +273,7 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// outgoing
-	err = newPacket(b, false, p, nil)
+	err = newPacket(b, false, p)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
@@ -290,7 +283,7 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// Too short TCP packet
-	err = newPacket(b[:len(b)-10], false, p, nil) // pull off the last 10 bytes
+	err = newPacket(b[:len(b)-10], false, p) // pull off the last 10 bytes
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 
 	// A good UDP packet with an AH header
@@ -325,7 +318,7 @@ func Test_newPacket_v6(t *testing.T) {
 	b = append(b, ahb...)
 	b = append(b, udpHeader...)
 
-	err = newPacket(b, true, p, nil)
+	err = newPacket(b, true, p)
 	require.NoError(t, err)
 	assert.Equal(t, uint8(firewall.ProtoUDP), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
@@ -335,12 +328,12 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// Ensure buffer bounds checking during processing
-	err = newPacket(b[:41], true, p, nil)
+	err = newPacket(b[:41], true, p)
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 
 	// Invalid AH header
 	b = buffer.Bytes()
-	err = newPacket(b, true, p, nil)
+	err = newPacket(b, true, p)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 }
 
@@ -388,7 +381,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	firstFrag = append(firstFrag, []byte{0xde, 0xad, 0xbe, 0xef}...)
 
 	// Test first fragment incoming
-	err = newPacket(firstFrag, true, p, nil)
+	err = newPacket(firstFrag, true, p)
 	require.NoError(t, err)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr)
@@ -398,7 +391,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	assert.False(t, p.Fragment)
 
 	// Test first fragment outgoing
-	err = newPacket(firstFrag, false, p, nil)
+	err = newPacket(firstFrag, false, p)
 	require.NoError(t, err)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.RemoteAddr)
@@ -427,7 +420,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	secondFrag = append(secondFrag, []byte{0xde, 0xad, 0xbe, 0xef}...)
 
 	// Test second fragment incoming
-	err = newPacket(secondFrag, true, p, nil)
+	err = newPacket(secondFrag, true, p)
 	require.NoError(t, err)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr)
@@ -437,7 +430,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	assert.True(t, p.Fragment)
 
 	// Test second fragment outgoing
-	err = newPacket(secondFrag, false, p, nil)
+	err = newPacket(secondFrag, false, p)
 	require.NoError(t, err)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.LocalAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.RemoteAddr)
@@ -447,7 +440,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
 	assert.True(t, p.Fragment)
 
 	// Too short of a fragment packet
-	err = newPacket(secondFrag[:len(secondFrag)-10], false, p, nil)
+	err = newPacket(secondFrag[:len(secondFrag)-10], false, p)
 	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
 }
 
@@ -529,7 +522,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("Normal", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(normalPacket, true, fp, nil); err != nil {
+			if err = parseV6(normalPacket, true, fp); err != nil {
 				b.Fatal(err)
 			}
 		}
@@ -537,7 +530,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("FirstFragment", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(firstFrag, true, fp, nil); err != nil {
+			if err = parseV6(firstFrag, true, fp); err != nil {
 				b.Fatal(err)
 			}
 		}
@@ -545,7 +538,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("SecondFragment", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(secondFrag, true, fp, nil); err != nil {
+			if err = parseV6(secondFrag, true, fp); err != nil {
 				b.Fatal(err)
 			}
 		}
@@ -581,7 +574,7 @@ func BenchmarkParseV6(b *testing.B) {
 	}
 
 	evilBytes := buffer.Bytes()
-	for range 200 {
+	for i := 0; i < 200; i++ {
 		evilBytes = append(evilBytes, hopHeader...)
 	}
 	evilBytes = append(evilBytes, lastHopHeader...)
@@ -590,7 +583,7 @@ func BenchmarkParseV6(b *testing.B) {
 
 	b.Run("200 HopByHop headers", func(b *testing.B) {
 		for i := 0; i < b.N; i++ {
-			if err = parseV6(evilBytes, false, fp, nil); err != nil {
+			if err = parseV6(evilBytes, false, fp); err != nil {
 				b.Fatal(err)
 			}
 		}

overlay/device.go

@@ -3,6 +3,7 @@ package overlay
 import (
 	"io"
 	"net/netip"
+	"sync"
 
 	"github.com/slackhq/nebula/routing"
 )
@@ -13,6 +14,86 @@ type Device interface {
 	Networks() []netip.Prefix
 	Name() string
 	RoutesFor(netip.Addr) routing.Gateways
-	SupportsMultiqueue() bool
 	NewMultiQueueReader() (io.ReadWriteCloser, error)
 }
+
+// Packet represents a single packet buffer with optional headroom to carry
+// metadata (for example virtio-net headers).
+type Packet struct {
+	Buf     []byte
+	Offset  int
+	Len     int
+	release func()
+}
+
+func (p *Packet) Payload() []byte {
+	return p.Buf[p.Offset : p.Offset+p.Len]
+}
+
+func (p *Packet) Reset() {
+	p.Len = 0
+	p.Offset = 0
+	p.release = nil
+}
+
+func (p *Packet) Release() {
+	if p.release != nil {
+		p.release()
+		p.release = nil
+	}
+}
+
+func (p *Packet) Capacity() int {
+	return len(p.Buf) - p.Offset
+}
+
+// PacketPool manages reusable buffers with headroom.
+type PacketPool struct {
+	headroom int
+	blksz    int
+	pool     sync.Pool
+}
+
+func NewPacketPool(headroom, payload int) *PacketPool {
+	p := &PacketPool{headroom: headroom, blksz: headroom + payload}
+	p.pool.New = func() any {
+		buf := make([]byte, p.blksz)
+		return &Packet{Buf: buf, Offset: headroom}
+	}
+	return p
+}
+
+func (p *PacketPool) Get() *Packet {
+	pkt := p.pool.Get().(*Packet)
+	pkt.Offset = p.headroom
+	pkt.Len = 0
+	pkt.release = func() { p.put(pkt) }
+	return pkt
+}
+
+func (p *PacketPool) put(pkt *Packet) {
+	pkt.Reset()
+	p.pool.Put(pkt)
+}
+
+// BatchReader allows reading multiple packets into a shared pool with
+// preallocated headroom (e.g. virtio-net headers).
+type BatchReader interface {
+	ReadIntoBatch(pool *PacketPool) ([]*Packet, error)
+}
+
+// BatchWriter writes a slice of packets that carry their own metadata.
+type BatchWriter interface {
+	WriteBatch(packets []*Packet) (int, error)
+}
+
+// BatchCapableDevice describes a device that can efficiently read and write
+// batches of packets with virtio headroom.
+type BatchCapableDevice interface {
+	Device
+	BatchReader
+	BatchWriter
+	BatchHeadroom() int
+	BatchPayloadCap() int
+	BatchSize() int
+}

overlay/tun.go

@@ -12,15 +12,6 @@ import (
 
 const DefaultMTU = 1300
 
-type NameError struct {
-	Name       string
-	Underlying error
-}
-
-func (e *NameError) Error() string {
-	return fmt.Sprintf("could not set tun device name: %s because %s", e.Name, e.Underlying)
-}
-
 // TODO: We may be able to remove routines
 type DeviceFactory func(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error)
 

overlay/tun_android.go

@@ -95,10 +95,6 @@ func (t *tun) Name() string {
 	return "android"
 }
 
-func (t *tun) SupportsMultiqueue() bool {
-	return false
-}
-
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for android")
 }

overlay/tun_darwin.go

@@ -549,10 +549,6 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) SupportsMultiqueue() bool {
-	return false
-}
-
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
 }

overlay/tun_disabled.go

@@ -105,10 +105,6 @@ func (t *disabledTun) Write(b []byte) (int, error) {
 	return len(b), nil
 }
 
-func (t *disabledTun) SupportsMultiqueue() bool {
-	return true
-}
-
 func (t *disabledTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return t, nil
 }

overlay/tun_file_linux_test.go

@@ -1,120 +0,0 @@
-//go:build linux && !android && !e2e_testing
-// +build linux,!android,!e2e_testing
-
-package overlay
-
-import (
-	"errors"
-	"os"
-	"sync"
-	"testing"
-	"time"
-
-	"golang.org/x/sys/unix"
-)
-
-// newReadPipe returns a read fd. The matching write fd is registered for cleanup.
-// The caller takes ownership of the read fd (pass it to newTunFd / newFriend).
-func newReadPipe(t *testing.T) int {
-	t.Helper()
-	var fds [2]int
-	if err := unix.Pipe2(fds[:], unix.O_CLOEXEC); err != nil {
-		t.Fatalf("pipe2: %v", err)
-	}
-	t.Cleanup(func() { _ = unix.Close(fds[1]) })
-	return fds[0]
-}
-
-func TestTunFile_WakeForShutdown_UnblocksRead(t *testing.T) {
-	tf, err := newTunFd(newReadPipe(t))
-	if err != nil {
-		t.Fatalf("newTunFd: %v", err)
-	}
-	t.Cleanup(func() { _ = tf.Close() })
-
-	done := make(chan error, 1)
-	go func() {
-		_, err := tf.Read(make([]byte, 64))
-		done <- err
-	}()
-
-	// Verify Read is actually blocked in poll.
-	select {
-	case err := <-done:
-		t.Fatalf("Read returned before shutdown signal: %v", err)
-	case <-time.After(50 * time.Millisecond):
-	}
-
-	if err := tf.wakeForShutdown(); err != nil {
-		t.Fatalf("wakeForShutdown: %v", err)
-	}
-
-	select {
-	case err := <-done:
-		if !errors.Is(err, os.ErrClosed) {
-			t.Fatalf("expected os.ErrClosed, got %v", err)
-		}
-	case <-time.After(2 * time.Second):
-		t.Fatal("Read did not wake on shutdown")
-	}
-}
-
-func TestTunFile_WakeForShutdown_WakesFriends(t *testing.T) {
-	parent, err := newTunFd(newReadPipe(t))
-	if err != nil {
-		t.Fatalf("newTunFd: %v", err)
-	}
-	friend, err := parent.newFriend(newReadPipe(t))
-	if err != nil {
-		_ = parent.Close()
-		t.Fatalf("newFriend: %v", err)
-	}
-	t.Cleanup(func() {
-		_ = friend.Close()
-		_ = parent.Close()
-	})
-
-	readers := []*tunFile{parent, friend}
-	errs := make([]error, len(readers))
-	var wg sync.WaitGroup
-	for i, r := range readers {
-		wg.Add(1)
-		go func(i int, r *tunFile) {
-			defer wg.Done()
-			_, errs[i] = r.Read(make([]byte, 64))
-		}(i, r)
-	}
-
-	time.Sleep(50 * time.Millisecond)
-
-	if err := parent.wakeForShutdown(); err != nil {
-		t.Fatalf("wakeForShutdown: %v", err)
-	}
-
-	done := make(chan struct{})
-	go func() { wg.Wait(); close(done) }()
-	select {
-	case <-done:
-	case <-time.After(2 * time.Second):
-		t.Fatal("readers did not wake")
-	}
-
-	for i, err := range errs {
-		if !errors.Is(err, os.ErrClosed) {
-			t.Errorf("reader %d: expected os.ErrClosed, got %v", i, err)
-		}
-	}
-}
-
-func TestTunFile_Close_Idempotent(t *testing.T) {
-	tf, err := newTunFd(newReadPipe(t))
-	if err != nil {
-		t.Fatalf("newTunFd: %v", err)
-	}
-	if err := tf.Close(); err != nil {
-		t.Fatalf("first Close: %v", err)
-	}
-	if err := tf.Close(); err != nil {
-		t.Fatalf("second Close should be a no-op, got %v", err)
-	}
-}

overlay/tun_freebsd.go

@@ -10,7 +10,6 @@ import (
 	"io"
 	"io/fs"
 	"net/netip"
-	"os"
 	"sync/atomic"
 	"syscall"
 	"time"
@@ -94,184 +93,107 @@ type tun struct {
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
 	linkAddr    *netroute.LinkAddr
 	l           *logrus.Logger
-
-	fd        int
-	shutdownR int // read end of the shutdown pipe; closing the write end wakes blocked polls
-	shutdownW int // write end of the shutdown pipe; closing this signals shutdown to any blocked reader/writer
-	readPoll  [2]unix.PollFd
-	writePoll [2]unix.PollFd
-	closed    atomic.Bool
-}
-
-// blockOnRead waits until the tun fd is readable or shutdown has been signaled.
-// Returns os.ErrClosed if Close was called.
-func (t *tun) blockOnRead() error {
-	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
-	var err error
-	for {
-		_, err = unix.Poll(t.readPoll[:], -1)
-		if err != unix.EINTR {
-			break
-		}
-	}
-	tunEvents := t.readPoll[0].Revents
-	shutdownEvents := t.readPoll[1].Revents
-	t.readPoll[0].Revents = 0
-	t.readPoll[1].Revents = 0
-	if err != nil {
-		return err
-	}
-	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
-		return os.ErrClosed
-	}
-	if tunEvents&problemFlags != 0 {
-		return os.ErrClosed
-	}
-	return nil
-}
-
-func (t *tun) blockOnWrite() error {
-	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
-	var err error
-	for {
-		_, err = unix.Poll(t.writePoll[:], -1)
-		if err != unix.EINTR {
-			break
-		}
-	}
-	tunEvents := t.writePoll[0].Revents
-	shutdownEvents := t.writePoll[1].Revents
-	t.writePoll[0].Revents = 0
-	t.writePoll[1].Revents = 0
-	if err != nil {
-		return err
-	}
-	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
-		return os.ErrClosed
-	}
-	if tunEvents&problemFlags != 0 {
-		return os.ErrClosed
-	}
-	return nil
+	devFd       int
 }
 
 func (t *tun) Read(to []byte) (int, error) {
+	// use readv() to read from the tunnel device, to eliminate the need for copying the buffer
+	if t.devFd < 0 {
+		return -1, syscall.EINVAL
+	}
+
 	// first 4 bytes is protocol family, in network byte order
-	var head [4]byte
-	iovecs := [2]syscall.Iovec{
+	head := make([]byte, 4)
+
+	iovecs := []syscall.Iovec{
 		{&head[0], 4},
 		{&to[0], uint64(len(to))},
 	}
-	for {
-		n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
-		if errno == 0 {
-			bytesRead := int(n)
-			if bytesRead < 4 {
-				return 0, nil
-			}
-			return bytesRead - 4, nil
-		}
-		switch errno {
-		case unix.EAGAIN:
-			if err := t.blockOnRead(); err != nil {
-				return 0, err
-			}
-		case unix.EINTR:
-			// retry
-		case unix.EBADF:
-			return 0, os.ErrClosed
-		default:
-			return 0, errno
-		}
+
+	n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
+
+	var err error
+	if errno != 0 {
+		err = syscall.Errno(errno)
+	} else {
+		err = nil
+	}
+	// fix bytes read number to exclude header
+	bytesRead := int(n)
+	if bytesRead < 0 {
+		return bytesRead, err
+	} else if bytesRead < 4 {
+		return 0, err
+	} else {
+		return bytesRead - 4, err
 	}
 }
 
 // Write is only valid for single threaded use
 func (t *tun) Write(from []byte) (int, error) {
+	// use writev() to write to the tunnel device, to eliminate the need for copying the buffer
+	if t.devFd < 0 {
+		return -1, syscall.EINVAL
+	}
+
 	if len(from) <= 1 {
 		return 0, syscall.EIO
 	}
-
 	ipVer := from[0] >> 4
-	var head [4]byte
+	var head []byte
 	// first 4 bytes is protocol family, in network byte order
-	switch ipVer {
-	case 4:
-		head[3] = syscall.AF_INET
-	case 6:
-		head[3] = syscall.AF_INET6
-	default:
+	if ipVer == 4 {
+		head = []byte{0, 0, 0, syscall.AF_INET}
+	} else if ipVer == 6 {
+		head = []byte{0, 0, 0, syscall.AF_INET6}
+	} else {
 		return 0, fmt.Errorf("unable to determine IP version from packet")
 	}
-
-	iovecs := [2]syscall.Iovec{
+	iovecs := []syscall.Iovec{
 		{&head[0], 4},
 		{&from[0], uint64(len(from))},
 	}
-	for {
-		n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
-		if errno == 0 {
-			return int(n) - 4, nil
-		}
-		switch errno {
-		case unix.EAGAIN:
-			if err := t.blockOnWrite(); err != nil {
-				return 0, err
-			}
-		case unix.EINTR:
-			// retry
-		case unix.EBADF:
-			return 0, os.ErrClosed
-		default:
-			return 0, errno
-		}
+
+	n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.devFd), uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
+
+	var err error
+	if errno != 0 {
+		err = syscall.Errno(errno)
+	} else {
+		err = nil
 	}
+
+	return int(n) - 4, err
 }
 
 func (t *tun) Close() error {
-	if t.closed.Swap(true) {
-		return nil
-	}
-
-	// Closing the write end of the shutdown pipe causes any blocked Poll to
-	// return with POLLHUP on the shutdown fd, so readers/writers wake up and
-	// exit with os.ErrClosed.
-	if t.shutdownW >= 0 {
-		_ = unix.Close(t.shutdownW)
-		t.shutdownW = -1
-	}
-
-	if t.fd >= 0 {
-		if err := unix.Close(t.fd); err != nil {
+	if t.devFd >= 0 {
+		err := syscall.Close(t.devFd)
+		if err != nil {
 			t.l.WithError(err).Error("Error closing device")
 		}
-		t.fd = -1
-	}
+		t.devFd = -1
 
-	if t.shutdownR >= 0 {
-		_ = unix.Close(t.shutdownR)
-		t.shutdownR = -1
-	}
+		c := make(chan struct{})
+		go func() {
+			// destroying the interface can block if a read() is still pending. Do this asynchronously.
+			defer close(c)
+			s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
+			if err == nil {
+				defer syscall.Close(s)
+				ifreq := ifreqDestroy{Name: t.deviceBytes()}
+				err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
+			}
+			if err != nil {
+				t.l.WithError(err).Error("Error destroying tunnel")
+			}
+		}()
 
-	c := make(chan struct{})
-	go func() {
-		// destroying the interface can block if a read() is still pending. Do this asynchronously.
-		defer close(c)
-		s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
-		if err == nil {
-			defer syscall.Close(s)
-			ifreq := ifreqDestroy{Name: t.deviceBytes()}
-			err = ioctl(uintptr(s), syscall.SIOCIFDESTROY, uintptr(unsafe.Pointer(&ifreq)))
+		// wait up to 1 second so we start blocking at the ioctl
+		select {
+		case <-c:
+		case <-time.After(1 * time.Second):
 		}
-		if err != nil {
-			t.l.WithError(err).Error("Error destroying tunnel")
-		}
-	}()
-
-	// wait up to 1 second so we start blocking at the ioctl
-	select {
-	case <-c:
-	case <-time.After(1 * time.Second):
 	}
 
 	return nil
@@ -287,38 +209,16 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 	var err error
 	deviceName := c.GetString("tun.dev", "")
 	if deviceName != "" {
-		fd, err = unix.Open("/dev/"+deviceName, os.O_RDWR, 0)
+		fd, err = syscall.Open("/dev/"+deviceName, syscall.O_RDWR, 0)
 	}
 	if errors.Is(err, fs.ErrNotExist) || deviceName == "" {
 		// If the device doesn't already exist, request a new one and rename it
-		fd, err = unix.Open("/dev/tun", os.O_RDWR, 0)
+		fd, err = syscall.Open("/dev/tun", syscall.O_RDWR, 0)
 	}
 	if err != nil {
 		return nil, err
 	}
 
-	if err = unix.SetNonblock(fd, true); err != nil {
-		_ = unix.Close(fd)
-		return nil, fmt.Errorf("failed to set tun device as nonblocking: %w", err)
-	}
-
-	// Shutdown pipe lets Close wake any reader/writer blocked in Poll.
-	var pipeFds [2]int
-	if err = unix.Pipe2(pipeFds[:], unix.O_CLOEXEC|unix.O_NONBLOCK); err != nil {
-		_ = unix.Close(fd)
-		return nil, fmt.Errorf("failed to create shutdown pipe: %w", err)
-	}
-	shutdownR, shutdownW := pipeFds[0], pipeFds[1]
-
-	closeOnErr := true
-	defer func() {
-		if closeOnErr {
-			_ = unix.Close(fd)
-			_ = unix.Close(shutdownR)
-			_ = unix.Close(shutdownW)
-		}
-	}()
-
 	// Read the name of the interface
 	var name [16]byte
 	arg := fiodgnameArg{length: 16, buf: unsafe.Pointer(&name)}
@@ -337,7 +237,7 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 	}
 
 	if ctrlErr != nil {
-		return nil, ctrlErr
+		return nil, err
 	}
 
 	ifName := string(bytes.TrimRight(name[:], "\x00"))
@@ -353,6 +253,8 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		}
 		defer syscall.Close(s)
 
+		fd := uintptr(s)
+
 		var fromName [16]byte
 		var toName [16]byte
 		copy(fromName[:], ifName)
@@ -364,7 +266,7 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		}
 
 		// Set the device name
-		_ = ioctl(uintptr(s), syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr)))
+		ioctl(fd, syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr)))
 	}
 
 	t := &tun{
@@ -372,24 +274,13 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		vpnNetworks: vpnNetworks,
 		MTU:         c.GetInt("tun.mtu", DefaultMTU),
 		l:           l,
-		fd:          fd,
-		shutdownR:   shutdownR,
-		shutdownW:   shutdownW,
-		readPoll: [2]unix.PollFd{
-			{Fd: int32(fd), Events: unix.POLLIN},
-			{Fd: int32(shutdownR), Events: unix.POLLIN},
-		},
-		writePoll: [2]unix.PollFd{
-			{Fd: int32(fd), Events: unix.POLLOUT},
-			{Fd: int32(shutdownR), Events: unix.POLLIN},
-		},
+		devFd:       fd,
 	}
 
 	err = t.reload(c, true)
 	if err != nil {
 		return nil, err
 	}
-	closeOnErr = false
 
 	c.RegisterReloadCallback(func(c *config.C) {
 		err := t.reload(c, false)
@@ -559,10 +450,6 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) SupportsMultiqueue() bool {
-	return false
-}
-
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
 }

overlay/tun_ios.go

@@ -151,10 +151,6 @@ func (t *tun) Name() string {
 	return "iOS"
 }
 
-func (t *tun) SupportsMultiqueue() bool {
-	return false
-}
-
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for ios")
 }

overlay/tun_linux.go

@@ -4,14 +4,13 @@
 package overlay
 
 import (
-	"encoding/binary"
 	"fmt"
 	"io"
 	"net"
 	"net/netip"
 	"os"
+	"runtime"
 	"strings"
-	"sync"
 	"sync/atomic"
 	"time"
 	"unsafe"
@@ -21,179 +20,14 @@ import (
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/routing"
 	"github.com/slackhq/nebula/util"
+	wgtun "github.com/slackhq/nebula/wgstack/tun"
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
 )
 
-// tunFile wraps a TUN file descriptor with poll-based reads. The FD provided will be changed to non-blocking.
-// A shared eventfd allows Close to wake all readers blocked in poll.
-type tunFile struct {
-	fd         int
-	shutdownFd int
-	lastOne    bool
-	readPoll   [2]unix.PollFd
-	writePoll  [2]unix.PollFd
-	closed     bool
-}
-
-// newFriend makes a tunFile for a MultiQueueReader that copies the shutdown eventfd from the parent tun
-func (r *tunFile) newFriend(fd int) (*tunFile, error) {
-	if err := unix.SetNonblock(fd, true); err != nil {
-		return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
-	}
-	return &tunFile{
-		fd:         fd,
-		shutdownFd: r.shutdownFd,
-		readPoll: [2]unix.PollFd{
-			{Fd: int32(fd), Events: unix.POLLIN},
-			{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
-		},
-		writePoll: [2]unix.PollFd{
-			{Fd: int32(fd), Events: unix.POLLOUT},
-			{Fd: int32(r.shutdownFd), Events: unix.POLLIN},
-		},
-	}, nil
-}
-
-func newTunFd(fd int) (*tunFile, error) {
-	if err := unix.SetNonblock(fd, true); err != nil {
-		return nil, fmt.Errorf("failed to set tun fd non-blocking: %w", err)
-	}
-
-	shutdownFd, err := unix.Eventfd(0, unix.EFD_NONBLOCK|unix.EFD_CLOEXEC)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create eventfd: %w", err)
-	}
-
-	out := &tunFile{
-		fd:         fd,
-		shutdownFd: shutdownFd,
-		lastOne:    true,
-		readPoll: [2]unix.PollFd{
-			{Fd: int32(fd), Events: unix.POLLIN},
-			{Fd: int32(shutdownFd), Events: unix.POLLIN},
-		},
-		writePoll: [2]unix.PollFd{
-			{Fd: int32(fd), Events: unix.POLLOUT},
-			{Fd: int32(shutdownFd), Events: unix.POLLIN},
-		},
-	}
-
-	return out, nil
-}
-
-func (r *tunFile) blockOnRead() error {
-	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
-	var err error
-	for {
-		_, err = unix.Poll(r.readPoll[:], -1)
-		if err != unix.EINTR {
-			break
-		}
-	}
-	//always reset these!
-	tunEvents := r.readPoll[0].Revents
-	shutdownEvents := r.readPoll[1].Revents
-	r.readPoll[0].Revents = 0
-	r.readPoll[1].Revents = 0
-	//do the err check before trusting the potentially bogus bits we just got
-	if err != nil {
-		return err
-	}
-	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
-		return os.ErrClosed
-	} else if tunEvents&problemFlags != 0 {
-		return os.ErrClosed
-	}
-	return nil
-}
-
-func (r *tunFile) blockOnWrite() error {
-	const problemFlags = unix.POLLHUP | unix.POLLNVAL | unix.POLLERR
-	var err error
-	for {
-		_, err = unix.Poll(r.writePoll[:], -1)
-		if err != unix.EINTR {
-			break
-		}
-	}
-	//always reset these!
-	tunEvents := r.writePoll[0].Revents
-	shutdownEvents := r.writePoll[1].Revents
-	r.writePoll[0].Revents = 0
-	r.writePoll[1].Revents = 0
-	//do the err check before trusting the potentially bogus bits we just got
-	if err != nil {
-		return err
-	}
-	if shutdownEvents&(unix.POLLIN|problemFlags) != 0 {
-		return os.ErrClosed
-	} else if tunEvents&problemFlags != 0 {
-		return os.ErrClosed
-	}
-	return nil
-}
-
-func (r *tunFile) Read(buf []byte) (int, error) {
-	for {
-		if n, err := unix.Read(r.fd, buf); err == nil {
-			return n, nil
-		} else if err == unix.EAGAIN {
-			if err = r.blockOnRead(); err != nil {
-				return 0, err
-			}
-			continue
-		} else if err == unix.EINTR {
-			continue
-		} else if err == unix.EBADF {
-			return 0, os.ErrClosed
-		} else {
-			return 0, err
-		}
-	}
-}
-
-func (r *tunFile) Write(buf []byte) (int, error) {
-	for {
-		if n, err := unix.Write(r.fd, buf); err == nil {
-			return n, nil
-		} else if err == unix.EAGAIN {
-			if err = r.blockOnWrite(); err != nil {
-				return 0, err
-			}
-			continue
-		} else if err == unix.EINTR {
-			continue
-		} else if err == unix.EBADF {
-			return 0, os.ErrClosed
-		} else {
-			return 0, err
-		}
-	}
-}
-
-func (r *tunFile) wakeForShutdown() error {
-	var buf [8]byte
-	binary.NativeEndian.PutUint64(buf[:], 1)
-	_, err := unix.Write(int(r.readPoll[1].Fd), buf[:])
-	return err
-}
-
-func (r *tunFile) Close() error {
-	if r.closed { // avoid closing more than once. Technically a fd could get re-used, which would be a problem
-		return nil
-	}
-	r.closed = true
-	if r.lastOne {
-		_ = unix.Close(r.shutdownFd)
-	}
-	return unix.Close(r.fd)
-}
-
 type tun struct {
-	*tunFile
-	readers     []*tunFile
-	closeLock   sync.Mutex
+	io.ReadWriteCloser
+	fd          int
 	Device      string
 	vpnNetworks []netip.Prefix
 	MaxMTU      int
@@ -201,6 +35,7 @@ type tun struct {
 	TXQueueLen  int
 	deviceIndex int
 	ioctlFd     uintptr
+	wgDevice    wgtun.Device
 
 	Routes                    atomic.Pointer[[]Route]
 	routeTree                 atomic.Pointer[bart.Table[routing.Gateways]]
@@ -208,11 +43,6 @@ type tun struct {
 	useSystemRoutes           bool
 	useSystemRoutesBufferSize int
 
-	// These are routes learned from `tun.use_system_route_table`
-	// stored here to make it easier to restore them after a reload
-	routesFromSystem     map[netip.Prefix]routing.Gateways
-	routesFromSystemLock sync.Mutex
-
 	l *logrus.Logger
 }
 
@@ -239,7 +69,11 @@ type ifreqQLEN struct {
 }
 
 func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
-	t, err := newTunGeneric(c, l, deviceFd, vpnNetworks)
+	file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
+
+	useWGDefault := runtime.GOOS == "linux"
+	useWG := c.GetBool("tun.use_wireguard_stack", c.GetBool("listen.use_wireguard_stack", useWGDefault))
+	t, err := newTunGeneric(c, l, file, vpnNetworks, useWG)
 	if err != nil {
 		return nil, err
 	}
@@ -277,18 +111,16 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueu
 	if multiqueue {
 		req.Flags |= unix.IFF_MULTI_QUEUE
 	}
-	nameStr := c.GetString("tun.dev", "")
-	copy(req.Name[:], nameStr)
+	copy(req.Name[:], c.GetString("tun.dev", ""))
 	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
-		_ = unix.Close(fd)
-		return nil, &NameError{
-			Name:       nameStr,
-			Underlying: err,
-		}
+		return nil, err
 	}
 	name := strings.Trim(string(req.Name[:]), "\x00")
 
-	t, err := newTunGeneric(c, l, fd, vpnNetworks)
+	file := os.NewFile(uintptr(fd), "/dev/net/tun")
+	useWGDefault := runtime.GOOS == "linux"
+	useWG := c.GetBool("tun.use_wireguard_stack", c.GetBool("listen.use_wireguard_stack", useWGDefault))
+	t, err := newTunGeneric(c, l, file, vpnNetworks, useWG)
 	if err != nil {
 		return nil, err
 	}
@@ -298,27 +130,48 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, multiqueu
 	return t, nil
 }
 
-// newTunGeneric does all the stuff common to different tun initialization paths. It will close your files on error.
-func newTunGeneric(c *config.C, l *logrus.Logger, fd int, vpnNetworks []netip.Prefix) (*tun, error) {
-	tfd, err := newTunFd(fd)
-	if err != nil {
-		_ = unix.Close(fd)
-		return nil, err
+func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, vpnNetworks []netip.Prefix, useWireguard bool) (*tun, error) {
+	var (
+		rw    io.ReadWriteCloser = file
+		fd                       = int(file.Fd())
+		wgDev wgtun.Device
+	)
+
+	if useWireguard {
+		dev, err := wgtun.CreateTUNFromFile(file, c.GetInt("tun.mtu", DefaultMTU))
+		if err != nil {
+			return nil, fmt.Errorf("failed to initialize wireguard tun device: %w", err)
+		}
+		wgDev = dev
+		rw = newWireguardTunIO(dev, c.GetInt("tun.mtu", DefaultMTU))
+		fd = int(dev.File().Fd())
 	}
+
 	t := &tun{
-		tunFile:                   tfd,
-		readers:                   []*tunFile{tfd},
-		closeLock:                 sync.Mutex{},
+		ReadWriteCloser:           rw,
+		fd:                        fd,
 		vpnNetworks:               vpnNetworks,
 		TXQueueLen:                c.GetInt("tun.tx_queue", 500),
 		useSystemRoutes:           c.GetBool("tun.use_system_route_table", false),
 		useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0),
-		routesFromSystem:          map[netip.Prefix]routing.Gateways{},
 		l:                         l,
 	}
+	if wgDev != nil {
+		t.wgDevice = wgDev
+	}
+	if wgDev != nil {
+		// replace ioctl fd with device file descriptor to keep route management working
+		file = wgDev.File()
+		t.fd = int(file.Fd())
+		t.ioctlFd = file.Fd()
+	}
 
-	if err = t.reload(c, true); err != nil {
-		_ = t.Close()
+	if t.ioctlFd == 0 {
+		t.ioctlFd = file.Fd()
+	}
+
+	err := t.reload(c, true)
+	if err != nil {
 		return nil, err
 	}
 
@@ -347,13 +200,6 @@ func (t *tun) reload(c *config.C, initial bool) error {
 		return err
 	}
 
-	// Bring along any routes learned from the system route table on reload
-	t.routesFromSystemLock.Lock()
-	for dst, gw := range t.routesFromSystem {
-		routeTree.Insert(dst, gw)
-	}
-	t.routesFromSystemLock.Unlock()
-
 	oldDefaultMTU := t.DefaultMTU
 	oldMaxMTU := t.MaxMTU
 	newDefaultMTU := c.GetInt("tun.mtu", DefaultMTU)
@@ -406,14 +252,7 @@ func (t *tun) reload(c *config.C, initial bool) error {
 	return nil
 }
 
-func (t *tun) SupportsMultiqueue() bool {
-	return true
-}
-
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
-	t.closeLock.Lock()
-	defer t.closeLock.Unlock()
-
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
 	if err != nil {
 		return nil, err
@@ -423,19 +262,12 @@ func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	req.Flags = uint16(unix.IFF_TUN | unix.IFF_NO_PI | unix.IFF_MULTI_QUEUE)
 	copy(req.Name[:], t.Device)
 	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
-		_ = unix.Close(fd)
 		return nil, err
 	}
 
-	out, err := t.tunFile.newFriend(fd)
-	if err != nil {
-		_ = unix.Close(fd)
-		return nil, err
-	}
+	file := os.NewFile(uintptr(fd), "/dev/net/tun")
 
-	t.readers = append(t.readers, out)
-
-	return out, nil
+	return file, nil
 }
 
 func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
@@ -443,6 +275,29 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
 	return r
 }
 
+func (t *tun) Write(b []byte) (int, error) {
+	var nn int
+	maximum := len(b)
+
+	for {
+		n, err := unix.Write(t.fd, b[nn:maximum])
+		if n > 0 {
+			nn += n
+		}
+		if nn == len(b) {
+			return nn, err
+		}
+
+		if err != nil {
+			return nn, err
+		}
+
+		if n == 0 {
+			return nn, io.ErrUnexpectedEOF
+		}
+	}
+}
+
 func (t *tun) deviceBytes() (o [16]byte) {
 	for i, c := range t.Device {
 		o[i] = byte(c)
@@ -763,42 +618,48 @@ func (t *tun) isGatewayInVpnNetworks(gwAddr netip.Addr) bool {
 }
 
 func (t *tun) getGatewaysFromRoute(r *netlink.Route) routing.Gateways {
+
 	var gateways routing.Gateways
 
 	link, err := netlink.LinkByName(t.Device)
 	if err != nil {
-		t.l.WithField("deviceName", t.Device).Error("Ignoring route update: failed to get link by name")
+		t.l.WithField("Devicename", t.Device).Error("Ignoring route update: failed to get link by name")
 		return gateways
 	}
 
 	// If this route is relevant to our interface and there is a gateway then add it
-	if r.LinkIndex == link.Attrs().Index {
-		gwAddr, ok := getGatewayAddr(r.Gw, r.Via)
-		if ok {
-			if t.isGatewayInVpnNetworks(gwAddr) {
-				gateways = append(gateways, routing.NewGateway(gwAddr, 1))
-			} else {
-				// Gateway isn't in our overlay network, ignore
-				t.l.WithField("route", r).Debug("Ignoring route update, gateway is not in our network")
-			}
+	if r.LinkIndex == link.Attrs().Index && len(r.Gw) > 0 {
+		gwAddr, ok := netip.AddrFromSlice(r.Gw)
+		if !ok {
+			t.l.WithField("route", r).Debug("Ignoring route update, invalid gateway address")
 		} else {
-			t.l.WithField("route", r).Debug("Ignoring route update, invalid gateway or via address")
+			gwAddr = gwAddr.Unmap()
+
+			if !t.isGatewayInVpnNetworks(gwAddr) {
+				// Gateway isn't in our overlay network, ignore
+				t.l.WithField("route", r).Debug("Ignoring route update, not in our network")
+			} else {
+				gateways = append(gateways, routing.NewGateway(gwAddr, 1))
+			}
 		}
 	}
 
 	for _, p := range r.MultiPath {
 		// If this route is relevant to our interface and there is a gateway then add it
-		if p.LinkIndex == link.Attrs().Index {
-			gwAddr, ok := getGatewayAddr(p.Gw, p.Via)
-			if ok {
-				if t.isGatewayInVpnNetworks(gwAddr) {
-					gateways = append(gateways, routing.NewGateway(gwAddr, p.Hops+1))
-				} else {
-					// Gateway isn't in our overlay network, ignore
-					t.l.WithField("route", r).Debug("Ignoring route update, gateway is not in our network")
-				}
+		if p.LinkIndex == link.Attrs().Index && len(p.Gw) > 0 {
+			gwAddr, ok := netip.AddrFromSlice(p.Gw)
+			if !ok {
+				t.l.WithField("route", r).Debug("Ignoring multipath route update, invalid gateway address")
 			} else {
-				t.l.WithField("route", r).Debug("Ignoring route update, invalid gateway or via address")
+				gwAddr = gwAddr.Unmap()
+
+				if !t.isGatewayInVpnNetworks(gwAddr) {
+					// Gateway isn't in our overlay network, ignore
+					t.l.WithField("route", r).Debug("Ignoring route update, not in our network")
+				} else {
+					// p.Hops+1 = weight of the route
+					gateways = append(gateways, routing.NewGateway(gwAddr, p.Hops+1))
+				}
 			}
 		}
 	}
@@ -807,27 +668,10 @@ func (t *tun) getGatewaysFromRoute(r *netlink.Route) routing.Gateways {
 	return gateways
 }
 
-func getGatewayAddr(gw net.IP, via netlink.Destination) (netip.Addr, bool) {
-	// Try to use the old RTA_GATEWAY first
-	gwAddr, ok := netip.AddrFromSlice(gw)
-	if !ok {
-		// Fallback to the new RTA_VIA
-		rVia, ok := via.(*netlink.Via)
-		if ok {
-			gwAddr, ok = netip.AddrFromSlice(rVia.Addr)
-		}
-	}
-
-	if gwAddr.IsValid() {
-		gwAddr = gwAddr.Unmap()
-		return gwAddr, true
-	}
-
-	return netip.Addr{}, false
-}
-
 func (t *tun) updateRoutes(r netlink.RouteUpdate) {
+
 	gateways := t.getGatewaysFromRoute(&r.Route)
+
 	if len(gateways) == 0 {
 		// No gateways relevant to our network, no routing changes required.
 		t.l.WithField("route", r).Debug("Ignoring route update, no gateways")
@@ -850,56 +694,37 @@ func (t *tun) updateRoutes(r netlink.RouteUpdate) {
 
 	newTree := t.routeTree.Load().Clone()
 
-	t.routesFromSystemLock.Lock()
 	if r.Type == unix.RTM_NEWROUTE {
 		t.l.WithField("destination", dst).WithField("via", gateways).Info("Adding route")
-		t.routesFromSystem[dst] = gateways
 		newTree.Insert(dst, gateways)
 
 	} else {
 		t.l.WithField("destination", dst).WithField("via", gateways).Info("Removing route")
-		delete(t.routesFromSystem, dst)
 		newTree.Delete(dst)
 	}
-	t.routesFromSystemLock.Unlock()
 	t.routeTree.Store(newTree)
 }
 
 func (t *tun) Close() error {
-	t.closeLock.Lock()
-	defer t.closeLock.Unlock()
-
 	if t.routeChan != nil {
 		close(t.routeChan)
-		t.routeChan = nil
 	}
 
-	// Signal all readers blocked in poll to wake up and exit
-	_ = t.tunFile.wakeForShutdown()
+	if t.ReadWriteCloser != nil {
+		_ = t.ReadWriteCloser.Close()
+	}
+
+	if t.wgDevice != nil {
+		_ = t.wgDevice.Close()
+		if t.ioctlFd > 0 {
+			// underlying fd already closed by the device
+			t.ioctlFd = 0
+		}
+	}
 
 	if t.ioctlFd > 0 {
-		_ = unix.Close(int(t.ioctlFd))
-		t.ioctlFd = 0
+		_ = os.NewFile(t.ioctlFd, "ioctlFd").Close()
 	}
 
-	for i := range t.readers {
-		if i == 0 {
-			continue //we want to close the zeroth reader last
-		}
-		err := t.readers[i].Close()
-		if err != nil {
-			t.l.WithField("reader", i).WithError(err).Error("error closing tun reader")
-		} else {
-			t.l.WithField("reader", i).Info("closed tun reader")
-		}
-	}
-
-	//this is t.readers[0] too
-	err := t.tunFile.Close()
-	if err != nil {
-		t.l.WithField("reader", 0).WithError(err).Error("error closing tun reader")
-	} else {
-		t.l.WithField("reader", 0).Info("closed tun reader")
-	}
-	return err
+	return nil
 }

overlay/tun_linux_batch.go

@@ -0,0 +1,56 @@
+//go:build linux && !android && !e2e_testing
+
+package overlay
+
+import "fmt"
+
+func (t *tun) batchIO() (*wireguardTunIO, bool) {
+	io, ok := t.ReadWriteCloser.(*wireguardTunIO)
+	return io, ok
+}
+
+func (t *tun) ReadIntoBatch(pool *PacketPool) ([]*Packet, error) {
+	io, ok := t.batchIO()
+	if !ok {
+		return nil, fmt.Errorf("wireguard batch I/O not enabled")
+	}
+	return io.ReadIntoBatch(pool)
+}
+
+func (t *tun) WriteBatch(packets []*Packet) (int, error) {
+	io, ok := t.batchIO()
+	if ok {
+		return io.WriteBatch(packets)
+	}
+	for _, pkt := range packets {
+		if pkt == nil {
+			continue
+		}
+		if _, err := t.Write(pkt.Payload()[:pkt.Len]); err != nil {
+			return 0, err
+		}
+		pkt.Release()
+	}
+	return len(packets), nil
+}
+
+func (t *tun) BatchHeadroom() int {
+	if io, ok := t.batchIO(); ok {
+		return io.BatchHeadroom()
+	}
+	return 0
+}
+
+func (t *tun) BatchPayloadCap() int {
+	if io, ok := t.batchIO(); ok {
+		return io.BatchPayloadCap()
+	}
+	return 0
+}
+
+func (t *tun) BatchSize() int {
+	if io, ok := t.batchIO(); ok {
+		return io.BatchSize()
+	}
+	return 1
+}

overlay/tun_netbsd.go

@@ -390,10 +390,6 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) SupportsMultiqueue() bool {
-	return false
-}
-
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for netbsd")
 }

overlay/tun_openbsd.go

@@ -310,10 +310,6 @@ func (t *tun) Name() string {
 	return t.Device
 }
 
-func (t *tun) SupportsMultiqueue() bool {
-	return false
-}
-
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for openbsd")
 }

overlay/tun_tester.go

@@ -132,10 +132,6 @@ func (t *TestTun) Read(b []byte) (int, error) {
 	return len(p), nil
 }
 
-func (t *TestTun) SupportsMultiqueue() bool {
-	return false
-}
-
 func (t *TestTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented")
 }

overlay/tun_windows.go

@@ -74,10 +74,7 @@ func newTun(c *config.C, l *logrus.Logger, vpnNetworks []netip.Prefix, _ bool) (
 		l.WithError(err).Debug("Failed to create wintun device, retrying")
 		tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU)
 		if err != nil {
-			return nil, &NameError{
-				Name:       deviceName,
-				Underlying: fmt.Errorf("create TUN device failed: %w", err),
-			}
+			return nil, fmt.Errorf("create TUN device failed: %w", err)
 		}
 	}
 	t.tun = tunDevice.(*wintun.NativeTun)
@@ -237,10 +234,6 @@ func (t *winTun) Write(b []byte) (int, error) {
 	return t.tun.Write(b, 0)
 }
 
-func (t *winTun) SupportsMultiqueue() bool {
-	return false
-}
-
 func (t *winTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for windows")
 }

overlay/user.go

@@ -46,10 +46,6 @@ func (d *UserDevice) RoutesFor(ip netip.Addr) routing.Gateways {
 	return routing.Gateways{routing.NewGateway(ip, 1)}
 }
 
-func (d *UserDevice) SupportsMultiqueue() bool {
-	return true
-}
-
 func (d *UserDevice) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return d, nil
 }

overlay/wireguard_tun_linux.go

@@ -0,0 +1,220 @@
+//go:build linux && !android && !e2e_testing
+
+package overlay
+
+import (
+	"fmt"
+	"sync"
+
+	wgtun "github.com/slackhq/nebula/wgstack/tun"
+)
+
+type wireguardTunIO struct {
+	dev       wgtun.Device
+	mtu       int
+	batchSize int
+
+	readMu      sync.Mutex
+	readBuffers [][]byte
+	readLens    []int
+	legacyBuf   []byte
+
+	writeMu      sync.Mutex
+	writeBuf     []byte
+	writeWrap    [][]byte
+	writeBuffers [][]byte
+}
+
+func newWireguardTunIO(dev wgtun.Device, mtu int) *wireguardTunIO {
+	batch := dev.BatchSize()
+	if batch <= 0 {
+		batch = 1
+	}
+	if mtu <= 0 {
+		mtu = DefaultMTU
+	}
+	return &wireguardTunIO{
+		dev:       dev,
+		mtu:       mtu,
+		batchSize: batch,
+		readLens:  make([]int, batch),
+		legacyBuf: make([]byte, wgtun.VirtioNetHdrLen+mtu),
+		writeBuf:  make([]byte, wgtun.VirtioNetHdrLen+mtu),
+		writeWrap: make([][]byte, 1),
+	}
+}
+
+func (w *wireguardTunIO) Read(p []byte) (int, error) {
+	w.readMu.Lock()
+	defer w.readMu.Unlock()
+
+	bufs := w.readBuffers
+	if len(bufs) == 0 {
+		bufs = [][]byte{w.legacyBuf}
+		w.readBuffers = bufs
+	}
+	n, err := w.dev.Read(bufs[:1], w.readLens[:1], wgtun.VirtioNetHdrLen)
+	if err != nil {
+		return 0, err
+	}
+	if n == 0 {
+		return 0, nil
+	}
+	length := w.readLens[0]
+	copy(p, w.legacyBuf[wgtun.VirtioNetHdrLen:wgtun.VirtioNetHdrLen+length])
+	return length, nil
+}
+
+func (w *wireguardTunIO) Write(p []byte) (int, error) {
+	if len(p) > w.mtu {
+		return 0, fmt.Errorf("wireguard tun: payload exceeds MTU (%d > %d)", len(p), w.mtu)
+	}
+	w.writeMu.Lock()
+	defer w.writeMu.Unlock()
+	buf := w.writeBuf[:wgtun.VirtioNetHdrLen+len(p)]
+	for i := 0; i < wgtun.VirtioNetHdrLen; i++ {
+		buf[i] = 0
+	}
+	copy(buf[wgtun.VirtioNetHdrLen:], p)
+	w.writeWrap[0] = buf
+	n, err := w.dev.Write(w.writeWrap, wgtun.VirtioNetHdrLen)
+	if err != nil {
+		return n, err
+	}
+	return len(p), nil
+}
+
+func (w *wireguardTunIO) ReadIntoBatch(pool *PacketPool) ([]*Packet, error) {
+	if pool == nil {
+		return nil, fmt.Errorf("wireguard tun: packet pool is nil")
+	}
+
+	w.readMu.Lock()
+	defer w.readMu.Unlock()
+
+	if len(w.readBuffers) < w.batchSize {
+		w.readBuffers = make([][]byte, w.batchSize)
+	}
+	if len(w.readLens) < w.batchSize {
+		w.readLens = make([]int, w.batchSize)
+	}
+
+	packets := make([]*Packet, w.batchSize)
+	requiredHeadroom := w.BatchHeadroom()
+	requiredPayload := w.BatchPayloadCap()
+	headroom := 0
+	for i := 0; i < w.batchSize; i++ {
+		pkt := pool.Get()
+		if pkt == nil {
+			releasePackets(packets[:i])
+			return nil, fmt.Errorf("wireguard tun: packet pool returned nil packet")
+		}
+		if pkt.Capacity() < requiredPayload {
+			pkt.Release()
+			releasePackets(packets[:i])
+			return nil, fmt.Errorf("wireguard tun: packet capacity %d below required %d", pkt.Capacity(), requiredPayload)
+		}
+		if i == 0 {
+			headroom = pkt.Offset
+			if headroom < requiredHeadroom {
+				pkt.Release()
+				releasePackets(packets[:i])
+				return nil, fmt.Errorf("wireguard tun: packet headroom %d below virtio requirement %d", headroom, requiredHeadroom)
+			}
+		} else if pkt.Offset != headroom {
+			pkt.Release()
+			releasePackets(packets[:i])
+			return nil, fmt.Errorf("wireguard tun: inconsistent packet headroom (%d != %d)", pkt.Offset, headroom)
+		}
+		packets[i] = pkt
+		w.readBuffers[i] = pkt.Buf
+	}
+
+	n, err := w.dev.Read(w.readBuffers[:w.batchSize], w.readLens[:w.batchSize], headroom)
+	if err != nil {
+		releasePackets(packets)
+		return nil, err
+	}
+	if n == 0 {
+		releasePackets(packets)
+		return nil, nil
+	}
+	for i := 0; i < n; i++ {
+		packets[i].Len = w.readLens[i]
+	}
+	for i := n; i < w.batchSize; i++ {
+		packets[i].Release()
+		packets[i] = nil
+	}
+	return packets[:n], nil
+}
+
+func (w *wireguardTunIO) WriteBatch(packets []*Packet) (int, error) {
+	if len(packets) == 0 {
+		return 0, nil
+	}
+	requiredHeadroom := w.BatchHeadroom()
+	offset := packets[0].Offset
+	if offset < requiredHeadroom {
+		releasePackets(packets)
+		return 0, fmt.Errorf("wireguard tun: packet offset %d smaller than required headroom %d", offset, requiredHeadroom)
+	}
+	for _, pkt := range packets {
+		if pkt == nil {
+			continue
+		}
+		if pkt.Offset != offset {
+			releasePackets(packets)
+			return 0, fmt.Errorf("wireguard tun: mixed packet offsets not supported")
+		}
+		limit := pkt.Offset + pkt.Len
+		if limit > len(pkt.Buf) {
+			releasePackets(packets)
+			return 0, fmt.Errorf("wireguard tun: packet length %d exceeds buffer capacity %d", pkt.Len, len(pkt.Buf)-pkt.Offset)
+		}
+	}
+	w.writeMu.Lock()
+	defer w.writeMu.Unlock()
+
+	if len(w.writeBuffers) < len(packets) {
+		w.writeBuffers = make([][]byte, len(packets))
+	}
+	for i, pkt := range packets {
+		if pkt == nil {
+			w.writeBuffers[i] = nil
+			continue
+		}
+		limit := pkt.Offset + pkt.Len
+		w.writeBuffers[i] = pkt.Buf[:limit]
+	}
+	n, err := w.dev.Write(w.writeBuffers[:len(packets)], offset)
+	if err != nil {
+		return n, err
+	}
+	releasePackets(packets)
+	return n, nil
+}
+
+func (w *wireguardTunIO) BatchHeadroom() int {
+	return wgtun.VirtioNetHdrLen
+}
+
+func (w *wireguardTunIO) BatchPayloadCap() int {
+	return w.mtu
+}
+
+func (w *wireguardTunIO) BatchSize() int {
+	return w.batchSize
+}
+
+func (w *wireguardTunIO) Close() error {
+	return nil
+}
+
+func releasePackets(pkts []*Packet) {
+	for _, pkt := range pkts {
+		if pkt != nil {
+			pkt.Release()
+		}
+	}
+}

pki.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
 	"net"
 	"net/netip"
 	"os"
@@ -488,25 +487,25 @@ func loadCertificate(b []byte) (cert.Certificate, []byte, error) {
 }
 
 func loadCAPoolFromConfig(l *logrus.Logger, c *config.C) (*cert.CAPool, error) {
+	var rawCA []byte
+	var err error
+
 	caPathOrPEM := c.GetString("pki.ca", "")
 	if caPathOrPEM == "" {
 		return nil, errors.New("no pki.ca path or PEM data provided")
 	}
 
-	var caReader io.ReadCloser
-	var err error
-
 	if strings.Contains(caPathOrPEM, "-----BEGIN") {
-		caReader = io.NopCloser(strings.NewReader(caPathOrPEM))
+		rawCA = []byte(caPathOrPEM)
+
 	} else {
-		caReader, err = os.Open(caPathOrPEM)
+		rawCA, err = os.ReadFile(caPathOrPEM)
 		if err != nil {
 			return nil, fmt.Errorf("unable to read pki.ca file %s: %s", caPathOrPEM, err)
 		}
 	}
-	defer caReader.Close()
 
-	caPool, err := cert.NewCAPoolFromPEMReader(caReader)
+	caPool, err := cert.NewCAPoolFromPEM(rawCA)
 	if errors.Is(err, cert.ErrExpired) {
 		var expired int
 		for _, crt := range caPool.CAs {
@@ -524,13 +523,9 @@ func loadCAPoolFromConfig(l *logrus.Logger, c *config.C) (*cert.CAPool, error) {
 		return nil, fmt.Errorf("error while adding CA certificate to CA trust store: %s", err)
 	}
 
-	bl := c.GetStringSlice("pki.blocklist", []string{})
-	if len(bl) > 0 {
-		for _, fp := range bl {
-			caPool.BlocklistFingerprint(fp)
-		}
-
-		l.WithField("fingerprintCount", len(bl)).Info("Blocklisted certificates")
+	for _, fp := range c.GetStringSlice("pki.blocklist", []string{}) {
+		l.WithField("fingerprint", fp).Info("Blocklisting cert")
+		caPool.BlocklistFingerprint(fp)
 	}
 
 	return caPool, nil

pki_hup_benchmark_test.go

@@ -1,121 +0,0 @@
-package nebula
-
-import (
-	"bytes"
-	"fmt"
-	"net/netip"
-	"os"
-	"path/filepath"
-	"runtime"
-	"testing"
-	"time"
-
-	"github.com/slackhq/nebula/cert"
-	cert_test "github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/test"
-	"github.com/stretchr/testify/require"
-)
-
-func BenchmarkReloadConfigWithCAs(b *testing.B) {
-	prevProcs := runtime.GOMAXPROCS(1)
-	b.Cleanup(func() { runtime.GOMAXPROCS(prevProcs) })
-
-	for _, size := range []int{100, 250, 500, 1000, 5000} {
-		b.Run(fmt.Sprintf("%dCAs", size), func(b *testing.B) {
-			l := test.NewLogger()
-			dir := b.TempDir()
-
-			ca, caKey, caBundle := buildCABundle(b, size)
-			caPath, certPath, keyPath := writePKIFiles(b, dir, ca, caKey, caBundle)
-
-			configBody := fmt.Sprintf(`pki:
-  ca: %s
-  cert: %s
-  key: %s
-`, caPath, certPath, keyPath)
-
-			configPath := filepath.Join(dir, "config.yml")
-			require.NoError(b, os.WriteFile(configPath, []byte(configBody), 0o600))
-
-			c := config.NewC(l)
-			require.NoError(b, c.Load(dir))
-
-			_, err := NewPKIFromConfig(l, c)
-			require.NoError(b, err)
-
-			b.ReportAllocs()
-			b.ResetTimer()
-
-			for b.Loop() {
-				c.ReloadConfig()
-			}
-		})
-	}
-}
-
-func buildCABundle(b *testing.B, count int) (cert.Certificate, []byte, []byte) {
-	b.Helper()
-	require.GreaterOrEqual(b, count, 1)
-
-	before := time.Now().Add(-24 * time.Hour)
-	after := time.Now().Add(24 * time.Hour)
-
-	ca, _, caKey, pem := cert_test.NewTestCaCert(
-		cert.Version2,
-		cert.Curve_CURVE25519,
-		before,
-		after,
-		nil,
-		nil,
-		nil,
-	)
-
-	buf := bytes.NewBuffer(pem)
-	buf.Write([]byte("\n# a comment!\n"))
-
-	for i := 1; i < count; i++ {
-		_, _, _, extraPEM := cert_test.NewTestCaCert(
-			cert.Version2,
-			cert.Curve_CURVE25519,
-			time.Now(),
-			time.Now().Add(time.Hour),
-			nil,
-			nil,
-			nil,
-		)
-		buf.Write([]byte("\n# a comment!\n"))
-		buf.Write(extraPEM)
-	}
-
-	return ca, caKey, buf.Bytes()
-}
-
-func writePKIFiles(b *testing.B, dir string, ca cert.Certificate, caKey []byte, caBundle []byte) (string, string, string) {
-	b.Helper()
-
-	networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
-
-	_, _, keyPEM, certPEM := cert_test.NewTestCert(
-		cert.Version2,
-		cert.Curve_CURVE25519,
-		ca,
-		caKey,
-		"reload-benchmark",
-		time.Now(),
-		time.Now().Add(time.Hour),
-		networks,
-		nil,
-		nil,
-	)
-
-	caPath := filepath.Join(dir, "ca.pem")
-	certPath := filepath.Join(dir, "cert.pem")
-	keyPath := filepath.Join(dir, "key.pem")
-
-	require.NoError(b, os.WriteFile(caPath, caBundle, 0o600))
-	require.NoError(b, os.WriteFile(certPath, certPEM, 0o600))
-	require.NoError(b, os.WriteFile(keyPath, keyPEM, 0o600))
-
-	return caPath, certPath, keyPath
-}

relay_manager.go

@@ -55,7 +55,7 @@ func (rm *relayManager) setAmRelay(v bool) {
 func AddRelay(l *logrus.Logger, relayHostInfo *HostInfo, hm *HostMap, vpnIp netip.Addr, remoteIdx *uint32, relayType int, state int) (uint32, error) {
 	hm.Lock()
 	defer hm.Unlock()
-	for range 32 {
+	for i := 0; i < 32; i++ {
 		index, err := generateIndex(l)
 		if err != nil {
 			return 0, err