Fix testify lint errors

.github/ISSUE_TEMPLATE/config.yml

@@ -1,21 +1,13 @@
 blank_issues_enabled: true
 contact_links:
-  - name: 💨 Performance Issues
-    url: https://github.com/slackhq/nebula/discussions/new/choose
-    about: 'We ask that you create a discussion instead of an issue for performance-related questions. This allows us to have a more open conversation about the issue and helps us to better understand the problem.'
-
-  - name: 📄 Documentation Issues
-    url: https://github.com/definednet/nebula-docs
-    about: "If you've found an issue with the website documentation, please file it in the nebula-docs repository."
-
-  - name: 📱 Mobile Nebula Issues
-    url: https://github.com/definednet/mobile_nebula
-    about: "If you're using the mobile Nebula app and have found an issue, please file it in the mobile_nebula repository."
-
   - name: 📘 Documentation
     url: https://nebula.defined.net/docs/
-    about: 'The documentation is the best place to start if you are new to Nebula.'
+    about: Review documentation.
 
   - name: 💁 Support/Chat
-    url: https://join.slack.com/t/nebulaoss/shared_invite/zt-39pk4xopc-CUKlGcb5Z39dQ0cK1v7ehA
+    url: https://join.slack.com/t/nebulaoss/shared_invite/enQtOTA5MDI4NDg3MTg4LTkwY2EwNTI4NzQyMzc0M2ZlODBjNWI3NTY1MzhiOThiMmZlZjVkMTI0NGY4YTMyNjUwMWEyNzNkZTJmYzQxOGU
-    about: 'For faster support, join us on Slack for assistance!'
+    about: 'This issue tracker is not for support questions. Join us on Slack for assistance!'
+
+  - name: 📱 Mobile Nebula
+    url: https://github.com/definednet/mobile_nebula
+    about: 'This issue tracker is not for mobile support. Try the Mobile Nebula repo instead!'

.github/pull_request_template.md

@@ -1,11 +0,0 @@
-<!--
-Thank you for taking the time to submit a pull request!
-
-Please be sure to provide a clear description of what you're trying to achieve with the change.
-
-- If you're submitting a new feature, please explain how to use it and document any new config options in the example config.
-- If you're submitting a bugfix, please link the related issue or describe the circumstances surrounding the issue.
-- If you're changing a default, explain why you believe the new default is appropriate for most users.
-
-P.S. If you're only updating the README or other docs, please file a pull request here instead: https://github.com/DefinedNet/nebula-docs
--->

.github/workflows/gofmt.yml

@@ -14,11 +14,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v5
       with:
-        go-version: '1.25'
+        go-version: '1.23'
         check-latest: true
 
     - name: Install goimports

.github/workflows/release.yml

@@ -10,11 +10,11 @@ jobs:
     name: Build Linux/BSD All
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.25'
+          go-version: '1.23'
           check-latest: true
 
       - name: Build
@@ -24,7 +24,7 @@ jobs:
           mv build/*.tar.gz release
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v4
         with:
           name: linux-latest
           path: release
@@ -33,11 +33,11 @@ jobs:
     name: Build Windows
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.25'
+          go-version: '1.23'
           check-latest: true
 
       - name: Build
@@ -55,7 +55,7 @@ jobs:
           mv dist\windows\wintun build\dist\windows\
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v4
         with:
           name: windows-latest
           path: build
@@ -66,16 +66,16 @@ jobs:
       HAS_SIGNING_CREDS: ${{ secrets.AC_USERNAME != '' }}
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-go@v6
+      - uses: actions/setup-go@v5
         with:
-          go-version: '1.25'
+          go-version: '1.23'
           check-latest: true
 
       - name: Import certificates
         if: env.HAS_SIGNING_CREDS == 'true'
-        uses: Apple-Actions/import-codesign-certs@v6
+        uses: Apple-Actions/import-codesign-certs@v3
         with:
           p12-file-base64: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_P12_BASE64 }}
           p12-password: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_PASSWORD }}
@@ -104,7 +104,7 @@ jobs:
           fi
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v4
         with:
           name: darwin-latest
           path: ./release/*
@@ -124,11 +124,11 @@ jobs:
       # be overwritten
       - name: Checkout code
         if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
-        uses: actions/checkout@v6
+        uses: actions/checkout@v4
 
       - name: Download artifacts
         if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v4
         with:
           name: linux-latest
           path: artifacts
@@ -160,10 +160,10 @@ jobs:
     needs: [build-linux, build-darwin, build-windows]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6
+      - uses: actions/checkout@v4
 
       - name: Download artifacts
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v4
         with:
           path: artifacts
 
@@ -209,11 +209,10 @@ jobs:
         id: create_release
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          GITHUB_REF_NAME: ${{ github.ref_name }}
         run: |
           cd artifacts
           gh release create \
             --verify-tag \
-            --title "Release ${GITHUB_REF_NAME}" \
+            --title "Release ${{ github.ref_name }}" \
-            "${GITHUB_REF_NAME}" \
+            "${{ github.ref_name }}" \
             SHASUM256.txt *-latest/*.zip *-latest/*.tar.gz

.github/workflows/smoke-extra.yml

@@ -18,27 +18,20 @@ jobs:
     if: github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'smoke-test-extra')
     name: Run extra smoke tests
     runs-on: ubuntu-latest
-    env:
-      VAGRANT_DEFAULT_PROVIDER: libvirt
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v5
       with:
-        go-version: '1.25'
+        go-version-file: 'go.mod'
         check-latest: true
 
     - name: add hashicorp source
       run: wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg && echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
 
-    - name: install vagrant and libvirt
+    - name: install vagrant
-      run: |
+      run: sudo apt-get update && sudo apt-get install -y vagrant virtualbox
-        sudo apt-get update && sudo apt-get install -y vagrant libvirt-daemon-system libvirt-dev
-        sudo chmod 666 /dev/kvm
-        sudo usermod -aG libvirt $(whoami)
-        sudo chmod 666 /var/run/libvirt/libvirt-sock
-        vagrant plugin install vagrant-libvirt
 
     - name: freebsd-amd64
       run: make smoke-vagrant/freebsd-amd64
@@ -49,19 +42,10 @@ jobs:
     - name: netbsd-amd64
       run: make smoke-vagrant/netbsd-amd64
 
+    - name: linux-386
+      run: make smoke-vagrant/linux-386
+
     - name: linux-amd64-ipv6disable
       run: make smoke-vagrant/linux-amd64-ipv6disable
 
-    # linux-386 runs last because it requires disabling KVM to use VirtualBox,
-    # which prevents libvirt (used by the other tests) from working after this point.
-    - name: install virtualbox for i386 test
-      run: |
-        sudo apt-get install -y virtualbox
-        sudo rmmod kvm_amd kvm_intel kvm 2>/dev/null || true
-
-    - name: linux-386
-      env:
-        VAGRANT_DEFAULT_PROVIDER: virtualbox
-      run: make smoke-vagrant/linux-386
-
     timeout-minutes: 30

.github/workflows/smoke.yml

@@ -18,11 +18,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v5
       with:
-        go-version: '1.25'
+        go-version: '1.23'
         check-latest: true
 
     - name: build

.github/workflows/smoke/build-relay.sh

@@ -16,10 +16,8 @@ relay:
   am_relay: true
 EOF
 
-    # TEST-NET-3 placeholder IPs; smoke-relay.sh seds them to real container IPs.
+    export LIGHTHOUSES="192.168.100.1 172.17.0.2:4242"
-    # Mapping: .2 lighthouse1, .3 host2, .4 host3, .5 host4.
+    export REMOTE_ALLOW_LIST='{"172.17.0.4/32": false, "172.17.0.5/32": false}'
-    export LIGHTHOUSES="192.168.100.1 203.0.113.2:4242"
-    export REMOTE_ALLOW_LIST='{"203.0.113.4/32": false, "203.0.113.5/32": false}'
 
     HOST="host2" ../genconfig.sh >host2.yml <<EOF
 relay:
@@ -27,7 +25,7 @@ relay:
     - 192.168.100.1
 EOF
 
-    export REMOTE_ALLOW_LIST='{"203.0.113.3/32": false}'
+    export REMOTE_ALLOW_LIST='{"172.17.0.3/32": false}'
 
     HOST="host3" ../genconfig.sh >host3.yml
 

.github/workflows/smoke/build.sh

@@ -5,15 +5,9 @@ set -e -x
 rm -rf ./build
 mkdir ./build
 
-# Smoke containers run on a dedicated docker network whose subnet is allocated
+# TODO: Assumes your docker bridge network is a /24, and the first container that launches will be .1
-# at smoke time, not known at build time. Configs are written with TEST-NET-3
+# - We could make this better by launching the lighthouse first and then fetching what IP it is.
-# placeholder IPs (RFC 5737) and smoke.sh / smoke-vagrant.sh / smoke-relay.sh
+NET="$(docker network inspect bridge -f '{{ range .IPAM.Config }}{{ .Subnet }}{{ end }}' | cut -d. -f1-3)"
-# sed the real container IPs in before starting nebula.
-#
-# Placeholder mapping (last octet == fixed container slot):
-#   203.0.113.2 -> lighthouse1, 203.0.113.3 -> host2,
-#   203.0.113.4 -> host3,       203.0.113.5 -> host4.
-LIGHTHOUSE_IP="203.0.113.2"
 
 (
     cd build
@@ -31,16 +25,16 @@ LIGHTHOUSE_IP="203.0.113.2"
         ../genconfig.sh >lighthouse1.yml
 
     HOST="host2" \
-        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
+        LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
         ../genconfig.sh >host2.yml
 
     HOST="host3" \
-        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
+        LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
         INBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
         ../genconfig.sh >host3.yml
 
     HOST="host4" \
-        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
+        LIGHTHOUSES="192.168.100.1 $NET.2:4242" \
         OUTBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
         ../genconfig.sh >host4.yml
 

.github/workflows/smoke/smoke-relay.sh

@@ -6,8 +6,6 @@ set -o pipefail
 
 mkdir -p logs
 
-NETWORK="nebula-smoke-relay"
-
 cleanup() {
     echo
     echo " *** cleanup"
@@ -18,53 +16,22 @@ cleanup() {
     then
         docker kill lighthouse1 host2 host3 host4
     fi
-    docker network rm "$NETWORK" >/dev/null 2>&1
 }
 
 trap cleanup EXIT
 
-# Create a dedicated smoke network with an explicit subnet (required for --ip
-# below). Probe a short list of candidates so a locally-used range doesn't
-# fail the whole test — we only need one to be free.
-docker network rm "$NETWORK" >/dev/null 2>&1 || true
-for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
-    if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
-        break
-    fi
-done
-if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
-    echo "failed to create $NETWORK: every candidate subnet is in use" >&2
-    exit 1
-fi
-
-# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
-# .3 host2, .4 host3, .5 host4 — matches the placeholders in build-relay.sh.
-SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
-PREFIX="${SUBNET%/*}"
-PREFIX="${PREFIX%.*}"
-LIGHTHOUSE_IP="$PREFIX.2"
-HOST2_IP="$PREFIX.3"
-HOST3_IP="$PREFIX.4"
-HOST4_IP="$PREFIX.5"
-
-# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
-for f in build/host2.yml build/host3.yml build/host4.yml; do
-    sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
-    mv "$f.tmp" "$f"
-done
-
 docker run --name lighthouse1 --rm nebula:smoke-relay -config lighthouse1.yml -test
-docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" nebula:smoke-relay -config host2.yml -test
+docker run --name host2 --rm nebula:smoke-relay -config host2.yml -test
-docker run --name host3 --rm -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" nebula:smoke-relay -config host3.yml -test
+docker run --name host3 --rm nebula:smoke-relay -config host3.yml -test
-docker run --name host4 --rm -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" nebula:smoke-relay -config host4.yml -test
+docker run --name host4 --rm nebula:smoke-relay -config host4.yml -test
 
-docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
+docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
 sleep 1
-docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
+docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
 sleep 1
-docker run --name host3 --network "$NETWORK" --ip "$HOST3_IP" -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
+docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
 sleep 1
-docker run --name host4 --network "$NETWORK" --ip "$HOST4_IP" -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
+docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm nebula:smoke-relay -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
 sleep 1
 
 set +x
@@ -109,13 +76,7 @@ docker exec host4 sh -c 'kill 1'
 docker exec host3 sh -c 'kill 1'
 docker exec host2 sh -c 'kill 1'
 docker exec lighthouse1 sh -c 'kill 1'
-
+sleep 5
-# Wait up to 30s for all backgrounded jobs to exit rather than relying on a
-# fixed sleep.
-for _ in $(seq 1 30); do
-    [ -z "$(jobs -r)" ] && break
-    sleep 1
-done
 
 if [ "$(jobs -r)" ]
 then

.github/workflows/smoke/smoke-vagrant.sh

@@ -8,8 +8,6 @@ export VAGRANT_CWD="$PWD/vagrant-$1"
 
 mkdir -p logs
 
-NETWORK="nebula-smoke"
-
 cleanup() {
     echo
     echo " *** cleanup"
@@ -21,51 +19,21 @@ cleanup() {
         docker kill lighthouse1 host2
     fi
     vagrant destroy -f
-    docker network rm "$NETWORK" >/dev/null 2>&1
 }
 
 trap cleanup EXIT
 
-# Create a dedicated smoke network with an explicit subnet (required for --ip
-# below). Probe a short list of candidates so a locally-used range doesn't
-# fail the whole test — we only need one to be free.
-docker network rm "$NETWORK" >/dev/null 2>&1 || true
-for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
-    if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
-        break
-    fi
-done
-if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
-    echo "failed to create $NETWORK: every candidate subnet is in use" >&2
-    exit 1
-fi
-
-# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
-# .3 host2 — matches the placeholders in build.sh.
-SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
-PREFIX="${SUBNET%/*}"
-PREFIX="${PREFIX%.*}"
-LIGHTHOUSE_IP="$PREFIX.2"
-HOST2_IP="$PREFIX.3"
-
-# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
-# This must happen before `vagrant up` rsyncs build/ into the VM for host3.
-for f in build/host2.yml build/host3.yml; do
-    sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
-    mv "$f.tmp" "$f"
-done
-
 CONTAINER="nebula:${NAME:-smoke}"
 
 docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test
-docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" "$CONTAINER" -config host2.yml -test
+docker run --name host2 --rm "$CONTAINER" -config host2.yml -test
 
 vagrant up
 vagrant ssh -c "cd /nebula && /nebula/$1-nebula -config host3.yml -test" -- -T
 
-docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
+docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
 sleep 1
-docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
+docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
 sleep 1
 vagrant ssh -c "cd /nebula && sudo sh -c 'echo \$\$ >/nebula/pid && exec /nebula/$1-nebula -config host3.yml'" 2>&1 -- -T | tee logs/host3 | sed -u 's/^/  [host3]  /' &
 sleep 15
@@ -128,14 +96,7 @@ vagrant ssh -c "ping -c1 192.168.100.2" -- -T
 vagrant ssh -c "sudo xargs kill </nebula/pid" -- -T
 docker exec host2 sh -c 'kill 1'
 docker exec lighthouse1 sh -c 'kill 1'
-
+sleep 1
-# Wait up to 30s for all backgrounded jobs to exit. vagrant ssh in particular
-# takes a beat to tear down after nebula exits on the VM, so a fixed sleep is
-# racy.
-for _ in $(seq 1 30); do
-    [ -z "$(jobs -r)" ] && break
-    sleep 1
-done
 
 if [ "$(jobs -r)" ]
 then

.github/workflows/smoke/smoke.sh

@@ -6,8 +6,6 @@ set -o pipefail
 
 mkdir -p logs
 
-NETWORK="nebula-smoke"
-
 cleanup() {
     echo
     echo " *** cleanup"
@@ -18,71 +16,38 @@ cleanup() {
     then
         docker kill lighthouse1 host2 host3 host4
     fi
-    docker network rm "$NETWORK" >/dev/null 2>&1
 }
 
 trap cleanup EXIT
 
-# Create a dedicated smoke network with an explicit subnet (required for --ip
-# below). Probe a short list of candidates so a locally-used range doesn't
-# fail the whole test — we only need one to be free.
-docker network rm "$NETWORK" >/dev/null 2>&1 || true
-for candidate in 172.30.0.0/24 172.31.0.0/24 10.98.0.0/24 10.99.0.0/24 192.168.230.0/24; do
-    if docker network create --subnet "$candidate" "$NETWORK" >/dev/null 2>&1; then
-        break
-    fi
-done
-if ! docker network inspect "$NETWORK" >/dev/null 2>&1; then
-    echo "failed to create $NETWORK: every candidate subnet is in use" >&2
-    exit 1
-fi
-
-# Derive container IPs from the network's assigned subnet. Slots: .2 lighthouse1,
-# .3 host2, .4 host3, .5 host4 — matches the placeholders in build.sh.
-SUBNET="$(docker network inspect -f '{{(index .IPAM.Config 0).Subnet}}' "$NETWORK")"
-PREFIX="${SUBNET%/*}"
-PREFIX="${PREFIX%.*}"
-LIGHTHOUSE_IP="$PREFIX.2"
-HOST2_IP="$PREFIX.3"
-HOST3_IP="$PREFIX.4"
-HOST4_IP="$PREFIX.5"
-
-# Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
-# build/lighthouse1.yml has no IPs to rewrite so it's skipped.
-for f in build/host2.yml build/host3.yml build/host4.yml; do
-    sed "s|203\.0\.113\.|$PREFIX.|g" "$f" >"$f.tmp"
-    mv "$f.tmp" "$f"
-done
-
 CONTAINER="nebula:${NAME:-smoke}"
 
 docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test
-docker run --name host2 --rm -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" "$CONTAINER" -config host2.yml -test
+docker run --name host2 --rm "$CONTAINER" -config host2.yml -test
-docker run --name host3 --rm -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" "$CONTAINER" -config host3.yml -test
+docker run --name host3 --rm "$CONTAINER" -config host3.yml -test
-docker run --name host4 --rm -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" "$CONTAINER" -config host4.yml -test
+docker run --name host4 --rm "$CONTAINER" -config host4.yml -test
 
-docker run --name lighthouse1 --network "$NETWORK" --ip "$LIGHTHOUSE_IP" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
+docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
 sleep 1
-docker run --name host2 --network "$NETWORK" --ip "$HOST2_IP" -v "$PWD/build/host2.yml:/nebula/host2.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
+docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
 sleep 1
-docker run --name host3 --network "$NETWORK" --ip "$HOST3_IP" -v "$PWD/build/host3.yml:/nebula/host3.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
+docker run --name host3 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host3.yml 2>&1 | tee logs/host3 | sed -u 's/^/  [host3]  /' &
 sleep 1
-docker run --name host4 --network "$NETWORK" --ip "$HOST4_IP" -v "$PWD/build/host4.yml:/nebula/host4.yml:ro" --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
+docker run --name host4 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host4.yml 2>&1 | tee logs/host4 | sed -u 's/^/  [host4]  /' &
 sleep 1
 
 # grab tcpdump pcaps for debugging
-docker exec lighthouse1 tcpdump -i tun0 -q -w - -U 2>logs/lighthouse1.inside.log >logs/lighthouse1.inside.pcap &
+docker exec lighthouse1 tcpdump -i nebula1 -q -w - -U 2>logs/lighthouse1.inside.log >logs/lighthouse1.inside.pcap &
 docker exec lighthouse1 tcpdump -i eth0 -q -w - -U 2>logs/lighthouse1.outside.log >logs/lighthouse1.outside.pcap &
-docker exec host2 tcpdump -i tun0 -q -w - -U 2>logs/host2.inside.log >logs/host2.inside.pcap &
+docker exec host2 tcpdump -i nebula1 -q -w - -U 2>logs/host2.inside.log >logs/host2.inside.pcap &
 docker exec host2 tcpdump -i eth0 -q -w - -U 2>logs/host2.outside.log >logs/host2.outside.pcap &
-docker exec host3 tcpdump -i tun0 -q -w - -U 2>logs/host3.inside.log >logs/host3.inside.pcap &
+docker exec host3 tcpdump -i nebula1 -q -w - -U 2>logs/host3.inside.log >logs/host3.inside.pcap &
 docker exec host3 tcpdump -i eth0 -q -w - -U 2>logs/host3.outside.log >logs/host3.outside.pcap &
-docker exec host4 tcpdump -i tun0 -q -w - -U 2>logs/host4.inside.log >logs/host4.inside.pcap &
+docker exec host4 tcpdump -i nebula1 -q -w - -U 2>logs/host4.inside.log >logs/host4.inside.pcap &
 docker exec host4 tcpdump -i eth0 -q -w - -U 2>logs/host4.outside.log >logs/host4.outside.pcap &
 
 docker exec host2 ncat -nklv 0.0.0.0 2000 &
 docker exec host3 ncat -nklv 0.0.0.0 2000 &
-docker exec host4 ncat -nkluv 0.0.0.0 4000 &
 docker exec host2 ncat -e '/usr/bin/echo host2' -nkluv 0.0.0.0 3000 &
 docker exec host3 ncat -e '/usr/bin/echo host3' -nkluv 0.0.0.0 3000 &
 
@@ -154,24 +119,17 @@ echo
 echo " *** Testing conntrack"
 echo
 set -x
-
+# host2 can ping host3 now that host3 pinged it first
-# host2 speaking to host4 on UDP 4000 should allow it to reply, when firewall rules would normally not permit this
+docker exec host2 ping -c1 192.168.100.3
-docker exec host2 sh -c "/usr/bin/echo host2 | ncat -nuv 192.168.100.4 4000"
+# host4 can ping host2 once conntrack established
-docker exec host2 ncat -e '/usr/bin/echo helloagainfromhost2' -nkluv 0.0.0.0 4000 &
+docker exec host2 ping -c1 192.168.100.4
-sleep 1
+docker exec host4 ping -c1 192.168.100.2
-docker exec host4 sh -c "/usr/bin/echo host4 | ncat -nuv 192.168.100.2 4000"
 
 docker exec host4 sh -c 'kill 1'
 docker exec host3 sh -c 'kill 1'
 docker exec host2 sh -c 'kill 1'
 docker exec lighthouse1 sh -c 'kill 1'
-
+sleep 5
-# Wait up to 30s for all backgrounded jobs to exit rather than relying on a
-# fixed sleep.
-for _ in $(seq 1 30); do
-    [ -z "$(jobs -r)" ] && break
-    sleep 1
-done
 
 if [ "$(jobs -r)" ]
 then

.github/workflows/smoke/vagrant-linux-amd64-ipv6disable/Vagrantfile

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 Vagrant.configure("2") do |config|
-  config.vm.box = "bento/ubuntu-24.04"
+  config.vm.box = "ubuntu/jammy64"
 
   config.vm.synced_folder "../build", "/nebula"
 

.github/workflows/smoke/vagrant-openbsd-amd64/Vagrantfile

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 Vagrant.configure("2") do |config|
-  config.vm.box = "DefinedNet/openbsd78"
+  config.vm.box = "generic/openbsd7"
 
   config.vm.synced_folder "../build", "/nebula", type: "rsync"
 end

.github/workflows/test.yml

@@ -18,11 +18,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v5
       with:
-        go-version: '1.25'
+        go-version: '1.23'
         check-latest: true
 
     - name: Build
@@ -32,9 +32,9 @@ jobs:
       run: make vet
 
     - name: golangci-lint
-      uses: golangci/golangci-lint-action@v9
+      uses: golangci/golangci-lint-action@v6
       with:
-        version: v2.5
+        version: v1.64
 
     - name: Test
       run: make test
@@ -45,7 +45,7 @@ jobs:
     - name: Build test mobile
       run: make build-test-mobile
 
-    - uses: actions/upload-artifact@v6
+    - uses: actions/upload-artifact@v4
       with:
         name: e2e packet flow linux-latest
         path: e2e/mermaid/linux-latest
@@ -56,11 +56,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v5
       with:
-        go-version: '1.25'
+        go-version: '1.23'
         check-latest: true
 
     - name: Build
@@ -77,11 +77,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v5
       with:
-        go-version: '1.25'
+        go-version: '1.22'
         check-latest: true
 
     - name: Build
@@ -98,11 +98,11 @@ jobs:
         os: [windows-latest, macos-latest]
     steps:
 
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
 
-    - uses: actions/setup-go@v6
+    - uses: actions/setup-go@v5
       with:
-        go-version: '1.25'
+        go-version: '1.23'
         check-latest: true
 
     - name: Build nebula
@@ -115,9 +115,9 @@ jobs:
       run: make vet
 
     - name: golangci-lint
-      uses: golangci/golangci-lint-action@v9
+      uses: golangci/golangci-lint-action@v6
       with:
-        version: v2.5
+        version: v1.64
 
     - name: Test
       run: make test
@@ -125,7 +125,7 @@ jobs:
     - name: End 2 end
       run: make e2evv
 
-    - uses: actions/upload-artifact@v6
+    - uses: actions/upload-artifact@v4
       with:
         name: e2e packet flow ${{ matrix.os }}
         path: e2e/mermaid/${{ matrix.os }}

.golangci.yaml

@@ -1,23 +1,9 @@
-version: "2"
+# yaml-language-server: $schema=https://golangci-lint.run/jsonschema/golangci.jsonschema.json
 linters:
-  default: none
+  # Disable all linters.
+  # Default: false
+  disable-all: true
+  # Enable specific linter
+  # https://golangci-lint.run/usage/linters/#enabled-by-default
   enable:
     - testifylint
-  exclusions:
-    generated: lax
-    presets:
-      - comments
-      - common-false-positives
-      - legacy
-      - std-error-handling
-    paths:
-      - third_party$
-      - builtin$
-      - examples$
-formatters:
-  exclusions:
-    generated: lax
-    paths:
-      - third_party$
-      - builtin$
-      - examples$

CHANGELOG.md

@@ -7,130 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
-## [1.10.3] - 2026-02-06
-
-### Security
-
-- Fix an issue where blocklist bypass is possible when using curve P256 since the signature can have 2 valid representations.
-  Both fingerprint representations will be tested against the blocklist.
-  Any newly issued P256 based certificates will have their signature clamped to the low-s form.
-  Nebula will assert the low-s signature form when validating certificates in a future version. [GHSA-69x3-g4r3-p962](https://github.com/slackhq/nebula/security/advisories/GHSA-69x3-g4r3-p962)
-
-### Changed
-
-- Improve error reporting if nebula fails to start due to a tun device naming issue. (#1588)
-
-## [1.10.2] - 2026-01-21
-
-### Fixed
-
-- Fix panic when using `use_system_route_table` that was introduced in v1.10.1. (#1580)
-
-### Changed
-
-- Fix some typos in comments. (#1582)
-- Dependency updates. (#1581)
-
-## [1.10.1] - 2026-01-16
-
-See the [v1.10.1](https://github.com/slackhq/nebula/milestone/26?closed=1) milestone for a complete list of changes.
-
-### Fixed
-
-- Fix a bug where an unsafe route derived from the system route table could be lost on a config reload. (#1573)
-- Fix the PEM banner for ECDSA P256 public keys. (#1552)
-- Fix a regression on Windows from 1.9.x where nebula could fall back to a less performant UDP listener if 
-  non-critical ioctls failed. (#1568)
-- Fix a bug in handshake processing when a peer sends an unexpected public key. (#1566)
-
-### Added
-
-- Add a config option to control accepting `recv_error` packets which defaults to `always`. (#1569)
-
-### Changed
-
-- Various dependency updates. (#1541, #1549, #1550, #1557, #1558, #1560, #1561, #1570, #1571)
-
-## [1.10.0] - 2025-12-04
-
-See the [v1.10.0](https://github.com/slackhq/nebula/milestone/16?closed=1) milestone for a complete list of changes.
-
-### Added
-
-- Support for ipv6 and multiple ipv4/6 addresses in the overlay.
-  A new v2 ASN.1 based certificate format.
-  Certificates now have a unified interface for external implementations.
-  (#1212, #1216, #1345, #1359, #1381, #1419, #1464, #1466, #1451, #1476, #1467, #1481, #1399, #1488, #1492, #1495, #1468, #1521, #1535, #1538)
-- Add the ability to mark packets on linux to better target nebula packets in iptables/nftables. (#1331)
-- Add ECMP support for `unsafe_routes`. (#1332)
-- PKCS11 support for P256 keys when built with `pkcs11` tag (#1153, #1482)
-
-### Changed
-
-- **NOTE**: `default_local_cidr_any` now defaults to false, meaning that any firewall rule
-  intended to target an `unsafe_routes` entry must explicitly declare it via the
-  `local_cidr` field. This is almost always the intended behavior. This flag is
-  deprecated and will be removed in a future release. (#1373)
-- Improve logging when a relay is in use on an inbound packet. (#1533)
-- Avoid fatal errors if `rountines` is > 1 on systems that don't support more than 1 routine. (#1531)
-- Log a warning if a firewall rule contains an `any` that negates a more restrictive filter. (#1513)
-- Accept encrypted CA passphrase from an environment variable. (#1421)
-- Allow handshaking with any trusted remote. (#1509)
-- Log only the count of blocklisted certificate fingerprints instead of the entire list. (#1525)
-- Don't fatal when the ssh server is unable to be configured successfully. (#1520)
-- Update to build against go v1.25. (#1483)
-- Allow projects using `nebula` as a library with userspace networking to configure the `logger` and build version. (#1239) 
-- Upgrade to `yaml.v3`. (#1148, #1371, #1438, #1478)
-
-### Fixed
-
-- Fix a potential bug with udp ipv4 only on darwin. (#1532)
-- Improve lost packet statistics. (#1441, #1537)
-- Honor `remote_allow_list` in hole punch response. (#1186)
-- Fix a panic when `tun.use_system_route_table` is `true` and a route lacks a destination. (#1437) 
-- Fix an issue when `tun.use_system_route_table: true` could result in heavy CPU utilization when many thousands of routes
-  are present. (#1326) 
-- Fix tests for 32 bit machines. (#1394)
-- Fix a possible 32bit integer underflow in config handling. (#1353)
-- Fix moving a udp address from one vpn address to another in the `static_host_map`
-  which could cause rapid re-handshaking with an incorrect remote. (#1259)
-- Improve smoke tests in environments where the docker network is not the default. (#1347)
-
-## [1.9.7] - 2025-10-10
-
-### Security
-
-- Fix an issue where Nebula could incorrectly accept and process a packet from an erroneous source IP when the sender's
-  certificate is configured with unsafe_routes (cert v1/v2) or multiple IPs (cert v2). (#1494)
-
-### Changed
-
-- Disable sending `recv_error` messages when a packet is received outside the allowable counter window. (#1459)
-- Improve error messages and remove some unnecessary fatal conditions in the Windows and generic udp listener. (#1453)
-
-## [1.9.6] - 2025-7-15
-
-### Added
-
-- Support dropping inactive tunnels. This is disabled by default in this release but can be enabled with `tunnels.drop_inactive`. See example config for more details. (#1413)
-
-### Fixed
-
-- Fix Darwin freeze due to presence of some Network Extensions (#1426)
-- Ensure the same relay tunnel is always used when multiple relay tunnels are present (#1422)
-- Fix Windows freeze due to ICMP error handling (#1412)
-- Fix relay migration panic (#1403)
-
-## [1.9.5] - 2024-12-05
-
-### Added
-
-- Gracefully ignore v2 certificates. (#1282)
-
-### Fixed
-
-- Fix relays that refuse to re-establish after one of the remote tunnel pairs breaks. (#1277)
-
 ## [1.9.4] - 2024-09-09
 
 ### Added
@@ -788,14 +664,7 @@ created.)
 
 - Initial public release.
 
-[Unreleased]: https://github.com/slackhq/nebula/compare/v1.10.3...HEAD
+[Unreleased]: https://github.com/slackhq/nebula/compare/v1.9.4...HEAD
-[1.10.3]: https://github.com/slackhq/nebula/releases/tag/v1.10.3
-[1.10.2]: https://github.com/slackhq/nebula/releases/tag/v1.10.2
-[1.10.1]: https://github.com/slackhq/nebula/releases/tag/v1.10.1
-[1.10.0]: https://github.com/slackhq/nebula/releases/tag/v1.10.0
-[1.9.7]: https://github.com/slackhq/nebula/releases/tag/v1.9.7
-[1.9.6]: https://github.com/slackhq/nebula/releases/tag/v1.9.6
-[1.9.5]: https://github.com/slackhq/nebula/releases/tag/v1.9.5
 [1.9.4]: https://github.com/slackhq/nebula/releases/tag/v1.9.4
 [1.9.3]: https://github.com/slackhq/nebula/releases/tag/v1.9.3
 [1.9.2]: https://github.com/slackhq/nebula/releases/tag/v1.9.2

CODEOWNERS

@@ -1 +0,0 @@
-#ECCN:Open Source

README.md

@@ -4,7 +4,7 @@ It lets you seamlessly connect computers anywhere in the world. Nebula is portab
 It can be used to connect a small number of computers, but is also able to connect tens of thousands of computers.
 
 Nebula incorporates a number of existing concepts like encryption, security groups, certificates,
-and tunneling.
+and tunneling, and each of those individual pieces existed before Nebula in various forms.
 What makes Nebula different to existing offerings is that it brings all of these ideas together,
 resulting in a sum that is greater than its individual parts.
 
@@ -12,7 +12,7 @@ Further documentation can be found [here](https://nebula.defined.net/docs/).
 
 You can read more about Nebula [here](https://medium.com/p/884110a5579).
 
-You can also join the NebulaOSS Slack group [here](https://join.slack.com/t/nebulaoss/shared_invite/zt-39pk4xopc-CUKlGcb5Z39dQ0cK1v7ehA).
+You can also join the NebulaOSS Slack group [here](https://join.slack.com/t/nebulaoss/shared_invite/zt-2xqe6e7vn-k_KGi8s13nsr7cvHVvHvuQ).
 
 ## Supported Platforms
 
@@ -28,46 +28,46 @@ Check the [releases](https://github.com/slackhq/nebula/releases/latest) page for
 #### Distribution Packages
 
 - [Arch Linux](https://archlinux.org/packages/extra/x86_64/nebula/)
-    ```sh
+    ```
-    sudo pacman -S nebula
+    $ sudo pacman -S nebula
     ```
 
 - [Fedora Linux](https://src.fedoraproject.org/rpms/nebula)
-    ```sh
+    ```
-    sudo dnf install nebula
+    $ sudo dnf install nebula
     ```
 
 - [Debian Linux](https://packages.debian.org/source/stable/nebula)
-    ```sh
+    ```
-    sudo apt install nebula
+    $ sudo apt install nebula
     ```
 
 - [Alpine Linux](https://pkgs.alpinelinux.org/packages?name=nebula)
-    ```sh
+    ```
-    sudo apk add nebula
+    $ sudo apk add nebula
     ```
 
 - [macOS Homebrew](https://github.com/Homebrew/homebrew-core/blob/HEAD/Formula/n/nebula.rb)
-    ```sh
+    ```
-    brew install nebula
+    $ brew install nebula
     ```
 
 - [Docker](https://hub.docker.com/r/nebulaoss/nebula)
-    ```sh
+    ```
-    docker pull nebulaoss/nebula
+    $ docker pull nebulaoss/nebula
     ```
 
-#### Mobile ([source code](https://github.com/DefinedNet/mobile_nebula))
+#### Mobile
 
 - [iOS](https://apps.apple.com/us/app/mobile-nebula/id1509587936?itsct=apps_box&itscg=30200)
 - [Android](https://play.google.com/store/apps/details?id=net.defined.mobile_nebula&pcampaignid=pcampaignidMKT-Other-global-all-co-prtnr-py-PartBadge-Mar2515-1)
 
 ## Technical Overview
 
-Nebula is a mutually authenticated peer-to-peer software-defined network based on the [Noise Protocol Framework](https://noiseprotocol.org/).
+Nebula is a mutually authenticated peer-to-peer software defined network based on the [Noise Protocol Framework](https://noiseprotocol.org/).
 Nebula uses certificates to assert a node's IP address, name, and membership within user-defined groups.
 Nebula's user-defined groups allow for provider agnostic traffic filtering between nodes.
-Discovery nodes (aka lighthouses) allow individual peers to find each other and optionally use UDP hole punching to establish connections from behind most firewalls or NATs.
+Discovery nodes allow individual peers to find each other and optionally use UDP hole punching to establish connections from behind most firewalls or NATs.
 Users can move data between nodes in any number of cloud service providers, datacenters, and endpoints, without needing to maintain a particular addressing scheme.
 
 Nebula uses Elliptic-curve Diffie-Hellman (`ECDH`) key exchange and `AES-256-GCM` in its default configuration.
@@ -76,42 +76,34 @@ Nebula was created to provide a mechanism for groups of hosts to communicate sec
 
 ## Getting started (quickly)
 
-**Don't want to manage your own PKI and lighthouses?** [Managed Nebula](https://www.defined.net/) from Defined Networking handles all of this for you.
-
 To set up a Nebula network, you'll need:
 
 #### 1. The [Nebula binaries](https://github.com/slackhq/nebula/releases) or [Distribution Packages](https://github.com/slackhq/nebula#distribution-packages) for your specific platform. Specifically you'll need `nebula-cert` and the specific nebula binary for each platform you use.
 
 #### 2. (Optional, but you really should..) At least one discovery node with a routable IP address, which we call a lighthouse.
 
-Nebula lighthouses allow nodes to find each other, anywhere in the world. A lighthouse is the only node in a Nebula network whose IP should not change. Running a lighthouse requires very few compute resources, and you can easily use the least expensive option from a cloud hosting provider. If you're not sure which provider to use, a number of us have used $6/mo [DigitalOcean](https://digitalocean.com) droplets as lighthouses.
+Nebula lighthouses allow nodes to find each other, anywhere in the world. A lighthouse is the only node in a Nebula network whose IP should not change. Running a lighthouse requires very few compute resources, and you can easily use the least expensive option from a cloud hosting provider. If you're not sure which provider to use, a number of us have used $5/mo [DigitalOcean](https://digitalocean.com) droplets as lighthouses.
+
+  Once you have launched an instance, ensure that Nebula udp traffic (default port udp/4242) can reach it over the internet.
 
-Once you have launched an instance, ensure that Nebula udp traffic (default port udp/4242) can reach it over the internet.
 
 #### 3. A Nebula certificate authority, which will be the root of trust for a particular Nebula network.
 
-```sh
+  ```
-./nebula-cert ca -name "Myorganization, Inc"
+  ./nebula-cert ca -name "Myorganization, Inc"
-```
+  ```
-
+  This will create files named `ca.key` and `ca.cert` in the current directory. The `ca.key` file is the most sensitive file you'll create, because it is the key used to sign the certificates for individual nebula nodes/hosts. Please store this file somewhere safe, preferably with strong encryption.
-This will create files named `ca.key` and `ca.cert` in the current directory. The `ca.key` file is the most sensitive file you'll create, because it is the key used to sign the certificates for individual nebula nodes/hosts. Please store this file somewhere safe, preferably with strong encryption.
-
-**Be aware!** By default, certificate authorities have a 1-year lifetime before expiration. See [this guide](https://nebula.defined.net/docs/guides/rotating-certificate-authority/) for details on rotating a CA.
 
 #### 4. Nebula host keys and certificates generated from that certificate authority
-
 This assumes you have four nodes, named lighthouse1, laptop, server1, host3. You can name the nodes any way you'd like, including FQDN. You'll also need to choose IP addresses and the associated subnet. In this example, we are creating a nebula network that will use 192.168.100.x/24 as its network range. This example also demonstrates nebula groups, which can later be used to define traffic rules in a nebula network.
-```sh
+```
 ./nebula-cert sign -name "lighthouse1" -ip "192.168.100.1/24"
 ./nebula-cert sign -name "laptop" -ip "192.168.100.2/24" -groups "laptop,home,ssh"
 ./nebula-cert sign -name "server1" -ip "192.168.100.9/24" -groups "servers"
 ./nebula-cert sign -name "host3" -ip "192.168.100.10/24"
 ```
 
-By default, host certificates will expire 1 second before the CA expires. Use the `-duration` flag to specify a shorter lifetime.
-
 #### 5. Configuration files for each host
-
 Download a copy of the nebula [example configuration](https://github.com/slackhq/nebula/blob/master/examples/config.yml).
 
 * On the lighthouse node, you'll need to ensure `am_lighthouse: true` is set.
@@ -126,13 +118,10 @@ For each host, copy the nebula binary to the host, along with `config.yml` from
 **DO NOT COPY `ca.key` TO INDIVIDUAL NODES.**
 
 #### 7. Run nebula on each host
-
+```
-```sh
 ./nebula -config /path/to/config.yml
 ```
 
-For more detailed instructions, [find the full documentation here](https://nebula.defined.net/docs/).
-
 ## Building Nebula from source
 
 Make sure you have [go](https://go.dev/doc/install) installed and clone this repo. Change to the nebula directory.
@@ -151,10 +140,8 @@ The default curve used for cryptographic handshakes and signatures is Curve25519
 
 In addition, Nebula can be built using the [BoringCrypto GOEXPERIMENT](https://github.com/golang/go/blob/go1.20/src/crypto/internal/boring/README.md) by running either of the following make targets:
 
-```sh
+    make bin-boringcrypto
-make bin-boringcrypto
+    make release-boringcrypto
-make release-boringcrypto
-```
 
 This is not the recommended default deployment, but may be useful based on your compliance requirements.
 
@@ -162,3 +149,5 @@ This is not the recommended default deployment, but may be useful based on your
 
 Nebula was created at Slack Technologies, Inc by Nate Brown and Ryan Huber, with contributions from Oliver Fross, Alan Lam, Wade Simmons, and Lining Wang.
 
+
+

allow_list.go

@@ -36,7 +36,7 @@ type AllowListNameRule struct {
 
 func NewLocalAllowListFromConfig(c *config.C, k string) (*LocalAllowList, error) {
 	var nameRules []AllowListNameRule
-	handleKey := func(key string, value any) (bool, error) {
+	handleKey := func(key string, value interface{}) (bool, error) {
 		if key == "interfaces" {
 			var err error
 			nameRules, err = getAllowListInterfaces(k, value)
@@ -70,7 +70,7 @@ func NewRemoteAllowListFromConfig(c *config.C, k, rangesKey string) (*RemoteAllo
 
 // If the handleKey func returns true, the rest of the parsing is skipped
 // for this key. This allows parsing of special values like `interfaces`.
-func newAllowListFromConfig(c *config.C, k string, handleKey func(key string, value any) (bool, error)) (*AllowList, error) {
+func newAllowListFromConfig(c *config.C, k string, handleKey func(key string, value interface{}) (bool, error)) (*AllowList, error) {
 	r := c.Get(k)
 	if r == nil {
 		return nil, nil
@@ -81,8 +81,8 @@ func newAllowListFromConfig(c *config.C, k string, handleKey func(key string, va
 
 // If the handleKey func returns true, the rest of the parsing is skipped
 // for this key. This allows parsing of special values like `interfaces`.
-func newAllowList(k string, raw any, handleKey func(key string, value any) (bool, error)) (*AllowList, error) {
+func newAllowList(k string, raw interface{}, handleKey func(key string, value interface{}) (bool, error)) (*AllowList, error) {
-	rawMap, ok := raw.(map[string]any)
+	rawMap, ok := raw.(map[interface{}]interface{})
 	if !ok {
 		return nil, fmt.Errorf("config `%s` has invalid type: %T", k, raw)
 	}
@@ -100,7 +100,12 @@ func newAllowList(k string, raw any, handleKey func(key string, value any) (bool
 	rules4 := allowListRules{firstValue: true, allValuesMatch: true, defaultSet: false}
 	rules6 := allowListRules{firstValue: true, allValuesMatch: true, defaultSet: false}
 
-	for rawCIDR, rawValue := range rawMap {
+	for rawKey, rawValue := range rawMap {
+		rawCIDR, ok := rawKey.(string)
+		if !ok {
+			return nil, fmt.Errorf("config `%s` has invalid key (type %T): %v", k, rawKey, rawKey)
+		}
+
 		if handleKey != nil {
 			handled, err := handleKey(rawCIDR, rawValue)
 			if err != nil {
@@ -111,7 +116,7 @@ func newAllowList(k string, raw any, handleKey func(key string, value any) (bool
 			}
 		}
 
-		value, ok := config.AsBool(rawValue)
+		value, ok := rawValue.(bool)
 		if !ok {
 			return nil, fmt.Errorf("config `%s` has invalid value (type %T): %v", k, rawValue, rawValue)
 		}
@@ -168,18 +173,22 @@ func newAllowList(k string, raw any, handleKey func(key string, value any) (bool
 	return &AllowList{cidrTree: tree}, nil
 }
 
-func getAllowListInterfaces(k string, v any) ([]AllowListNameRule, error) {
+func getAllowListInterfaces(k string, v interface{}) ([]AllowListNameRule, error) {
 	var nameRules []AllowListNameRule
 
-	rawRules, ok := v.(map[string]any)
+	rawRules, ok := v.(map[interface{}]interface{})
 	if !ok {
 		return nil, fmt.Errorf("config `%s.interfaces` is invalid (type %T): %v", k, v, v)
 	}
 
 	firstEntry := true
 	var allValues bool
-	for name, rawAllow := range rawRules {
+	for rawName, rawAllow := range rawRules {
-		allow, ok := config.AsBool(rawAllow)
+		name, ok := rawName.(string)
+		if !ok {
+			return nil, fmt.Errorf("config `%s.interfaces` has invalid key (type %T): %v", k, rawName, rawName)
+		}
+		allow, ok := rawAllow.(bool)
 		if !ok {
 			return nil, fmt.Errorf("config `%s.interfaces` has invalid value (type %T): %v", k, rawAllow, rawAllow)
 		}
@@ -215,11 +224,16 @@ func getRemoteAllowRanges(c *config.C, k string) (*bart.Table[*AllowList], error
 
 	remoteAllowRanges := new(bart.Table[*AllowList])
 
-	rawMap, ok := value.(map[string]any)
+	rawMap, ok := value.(map[interface{}]interface{})
 	if !ok {
 		return nil, fmt.Errorf("config `%s` has invalid type: %T", k, value)
 	}
-	for rawCIDR, rawValue := range rawMap {
+	for rawKey, rawValue := range rawMap {
+		rawCIDR, ok := rawKey.(string)
+		if !ok {
+			return nil, fmt.Errorf("config `%s` has invalid key (type %T): %v", k, rawKey, rawKey)
+		}
+
 		allowList, err := newAllowList(fmt.Sprintf("%s.%s", k, rawCIDR), rawValue, nil)
 		if err != nil {
 			return nil, err

allow_list_test.go

@@ -15,27 +15,27 @@ import (
 func TestNewAllowListFromConfig(t *testing.T) {
 	l := test.NewLogger()
 	c := config.NewC(l)
-	c.Settings["allowlist"] = map[string]any{
+	c.Settings["allowlist"] = map[interface{}]interface{}{
 		"192.168.0.0": true,
 	}
 	r, err := newAllowListFromConfig(c, "allowlist", nil)
 	require.EqualError(t, err, "config `allowlist` has invalid CIDR: 192.168.0.0. netip.ParsePrefix(\"192.168.0.0\"): no '/'")
 	assert.Nil(t, r)
 
-	c.Settings["allowlist"] = map[string]any{
+	c.Settings["allowlist"] = map[interface{}]interface{}{
 		"192.168.0.0/16": "abc",
 	}
 	r, err = newAllowListFromConfig(c, "allowlist", nil)
 	require.EqualError(t, err, "config `allowlist` has invalid value (type string): abc")
 
-	c.Settings["allowlist"] = map[string]any{
+	c.Settings["allowlist"] = map[interface{}]interface{}{
 		"192.168.0.0/16": true,
 		"10.0.0.0/8":     false,
 	}
 	r, err = newAllowListFromConfig(c, "allowlist", nil)
 	require.EqualError(t, err, "config `allowlist` contains both true and false rules, but no default set for 0.0.0.0/0")
 
-	c.Settings["allowlist"] = map[string]any{
+	c.Settings["allowlist"] = map[interface{}]interface{}{
 		"0.0.0.0/0":      true,
 		"10.0.0.0/8":     false,
 		"10.42.42.0/24":  true,
@@ -45,7 +45,7 @@ func TestNewAllowListFromConfig(t *testing.T) {
 	r, err = newAllowListFromConfig(c, "allowlist", nil)
 	require.EqualError(t, err, "config `allowlist` contains both true and false rules, but no default set for ::/0")
 
-	c.Settings["allowlist"] = map[string]any{
+	c.Settings["allowlist"] = map[interface{}]interface{}{
 		"0.0.0.0/0":     true,
 		"10.0.0.0/8":    false,
 		"10.42.42.0/24": true,
@@ -55,7 +55,7 @@ func TestNewAllowListFromConfig(t *testing.T) {
 		assert.NotNil(t, r)
 	}
 
-	c.Settings["allowlist"] = map[string]any{
+	c.Settings["allowlist"] = map[interface{}]interface{}{
 		"0.0.0.0/0":      true,
 		"10.0.0.0/8":     false,
 		"10.42.42.0/24":  true,
@@ -70,16 +70,16 @@ func TestNewAllowListFromConfig(t *testing.T) {
 
 	// Test interface names
 
-	c.Settings["allowlist"] = map[string]any{
+	c.Settings["allowlist"] = map[interface{}]interface{}{
-		"interfaces": map[string]any{
+		"interfaces": map[interface{}]interface{}{
 			`docker.*`: "foo",
 		},
 	}
 	lr, err := NewLocalAllowListFromConfig(c, "allowlist")
 	require.EqualError(t, err, "config `allowlist.interfaces` has invalid value (type string): foo")
 
-	c.Settings["allowlist"] = map[string]any{
+	c.Settings["allowlist"] = map[interface{}]interface{}{
-		"interfaces": map[string]any{
+		"interfaces": map[interface{}]interface{}{
 			`docker.*`: false,
 			`eth.*`:    true,
 		},
@@ -87,8 +87,8 @@ func TestNewAllowListFromConfig(t *testing.T) {
 	lr, err = NewLocalAllowListFromConfig(c, "allowlist")
 	require.EqualError(t, err, "config `allowlist.interfaces` values must all be the same true/false value")
 
-	c.Settings["allowlist"] = map[string]any{
+	c.Settings["allowlist"] = map[interface{}]interface{}{
-		"interfaces": map[string]any{
+		"interfaces": map[interface{}]interface{}{
 			`docker.*`: false,
 		},
 	}

batch.go

@@ -1,70 +0,0 @@
-package nebula
-
-import "net/netip"
-
-// sendBatchCap is the maximum number of encrypted packets accumulated before a
-// flush is forced. TSO superpackets segment to at most ~45 packets on
-// reasonable MTUs, so 128 leaves headroom without bloating the backing
-// allocation.
-const sendBatchCap = 128
-
-// sendBatch accumulates encrypted UDP packets for a single sendmmsg flush.
-// One sendBatch is owned by each listenIn goroutine; no locking is needed.
-// The backing storage holds up to batchCap packets of slotCap bytes each;
-// bufs and dsts are parallel slices of committed slots.
-type sendBatch struct {
-	bufs     [][]byte
-	dsts     []netip.AddrPort
-	backing  []byte
-	slotCap  int
-	batchCap int
-	nextSlot int
-}
-
-func newSendBatch(batchCap, slotCap int) *sendBatch {
-	return &sendBatch{
-		bufs:     make([][]byte, 0, batchCap),
-		dsts:     make([]netip.AddrPort, 0, batchCap),
-		backing:  make([]byte, batchCap*slotCap),
-		slotCap:  slotCap,
-		batchCap: batchCap,
-	}
-}
-
-// Next returns a zero-length slice with slotCap capacity over the next unused
-// slot's backing bytes. The caller writes into the returned slice and then
-// calls Commit with the final length and destination. Next returns nil when
-// the batch is full.
-func (b *sendBatch) Next() []byte {
-	if b.nextSlot >= b.batchCap {
-		return nil
-	}
-	start := b.nextSlot * b.slotCap
-	return b.backing[start : start : start+b.slotCap]
-}
-
-// Commit records the slot just returned by Next as a packet of length n
-// destined for dst.
-func (b *sendBatch) Commit(n int, dst netip.AddrPort) {
-	start := b.nextSlot * b.slotCap
-	b.bufs = append(b.bufs, b.backing[start:start+n])
-	b.dsts = append(b.dsts, dst)
-	b.nextSlot++
-}
-
-// Reset clears committed slots; backing storage is retained for reuse.
-func (b *sendBatch) Reset() {
-	b.bufs = b.bufs[:0]
-	b.dsts = b.dsts[:0]
-	b.nextSlot = 0
-}
-
-// Len returns the number of committed packets.
-func (b *sendBatch) Len() int {
-	return len(b.bufs)
-}
-
-// Cap returns the maximum number of slots in the batch.
-func (b *sendBatch) Cap() int {
-	return b.batchCap
-}

batch_test.go

@@ -1,69 +0,0 @@
-package nebula
-
-import (
-	"net/netip"
-	"testing"
-)
-
-func TestSendBatchBookkeeping(t *testing.T) {
-	b := newSendBatch(4, 32)
-	if b.Len() != 0 || b.Cap() != 4 {
-		t.Fatalf("fresh batch: len=%d cap=%d", b.Len(), b.Cap())
-	}
-
-	ap := netip.MustParseAddrPort("10.0.0.1:4242")
-	for i := 0; i < 4; i++ {
-		slot := b.Next()
-		if slot == nil {
-			t.Fatalf("slot %d: Next returned nil before cap", i)
-		}
-		if cap(slot) != 32 || len(slot) != 0 {
-			t.Fatalf("slot %d: got len=%d cap=%d want len=0 cap=32", i, len(slot), cap(slot))
-		}
-		// Write a marker byte.
-		slot = append(slot, byte(i), byte(i+1), byte(i+2))
-		b.Commit(len(slot), ap)
-	}
-	if b.Next() != nil {
-		t.Fatalf("Next should return nil when full")
-	}
-	if b.Len() != 4 {
-		t.Fatalf("Len=%d want 4", b.Len())
-	}
-	for i, buf := range b.bufs {
-		if len(buf) != 3 || buf[0] != byte(i) {
-			t.Errorf("buf %d: %x", i, buf)
-		}
-		if b.dsts[i] != ap {
-			t.Errorf("dst %d: got %v want %v", i, b.dsts[i], ap)
-		}
-	}
-
-	// Reset returns empty and Next works again.
-	b.Reset()
-	if b.Len() != 0 {
-		t.Fatalf("after Reset Len=%d want 0", b.Len())
-	}
-	slot := b.Next()
-	if slot == nil || cap(slot) != 32 {
-		t.Fatalf("after Reset Next nil or wrong cap: %v cap=%d", slot == nil, cap(slot))
-	}
-}
-
-func TestSendBatchSlotsDoNotOverlap(t *testing.T) {
-	b := newSendBatch(3, 8)
-	ap := netip.MustParseAddrPort("10.0.0.1:80")
-
-	// Fill three slots, each with its own sentinel byte.
-	for i := 0; i < 3; i++ {
-		s := b.Next()
-		s = append(s, byte(0xA0+i), byte(0xB0+i))
-		b.Commit(len(s), ap)
-	}
-
-	for i, buf := range b.bufs {
-		if buf[0] != byte(0xA0+i) || buf[1] != byte(0xB0+i) {
-			t.Errorf("slot %d corrupted: %x", i, buf)
-		}
-	}
-}

bits.go

@@ -9,13 +9,14 @@ type Bits struct {
 	length             uint64
 	current            uint64
 	bits               []bool
+	firstSeen          bool
 	lostCounter        metrics.Counter
 	dupeCounter        metrics.Counter
 	outOfWindowCounter metrics.Counter
 }
 
 func NewBits(bits uint64) *Bits {
-	b := &Bits{
+	return &Bits{
 		length:             bits,
 		bits:               make([]bool, bits, bits),
 		current:            0,
@@ -23,37 +24,34 @@ func NewBits(bits uint64) *Bits {
 		dupeCounter:        metrics.GetOrRegisterCounter("network.packets.duplicate", nil),
 		outOfWindowCounter: metrics.GetOrRegisterCounter("network.packets.out_of_window", nil),
 	}
-
-	// There is no counter value 0, mark it to avoid counting a lost packet later.
-	b.bits[0] = true
-	b.current = 0
-	return b
 }
 
-func (b *Bits) Check(l *logrus.Logger, i uint64) bool {
+func (b *Bits) Check(l logrus.FieldLogger, i uint64) bool {
 	// If i is the next number, return true.
-	if i > b.current {
+	if i > b.current || (i == 0 && b.firstSeen == false && b.current < b.length) {
 		return true
 	}
 
-	// If i is within the window, check if it's been set already.
+	// If i is within the window, check if it's been set already. The first window will fail this check
-	if i > b.current-b.length || i < b.length && b.current < b.length {
+	if i > b.current-b.length {
+		return !b.bits[i%b.length]
+	}
+
+	// If i is within the first window
+	if i < b.length {
 		return !b.bits[i%b.length]
 	}
 
 	// Not within the window
-	if l.Level >= logrus.DebugLevel {
+	l.Debugf("rejected a packet (top) %d %d\n", b.current, i)
-		l.Debugf("rejected a packet (top) %d %d\n", b.current, i)
-	}
 	return false
 }
 
 func (b *Bits) Update(l *logrus.Logger, i uint64) bool {
 	// If i is the next number, return true and update current.
 	if i == b.current+1 {
-		// Check if the oldest bit was lost since we are shifting the window by 1 and occupying it with this counter
+		// Report missed packets, we can only understand what was missed after the first window has been gone through
-		// The very first window can only be tracked as lost once we are on the 2nd window or greater
+		if i > b.length && b.bits[i%b.length] == false {
-		if b.bits[i%b.length] == false && i > b.length {
 			b.lostCounter.Inc(1)
 		}
 		b.bits[i%b.length] = true
@@ -61,32 +59,61 @@ func (b *Bits) Update(l *logrus.Logger, i uint64) bool {
 		return true
 	}
 
-	// If i is a jump, adjust the window, record lost, update current, and return true
+	// If i packet is greater than current but less than the maximum length of our bitmap,
-	if i > b.current {
+	// flip everything in between to false and move ahead.
-		lost := int64(0)
+	if i > b.current && i < b.current+b.length {
-		// Zero out the bits between the current and the new counter value, limited by the window size,
+		// In between current and i need to be zero'd to allow those packets to come in later
-		// since the window is shifting
+		for n := b.current + 1; n < i; n++ {
-		for n := b.current + 1; n <= min(i, b.current+b.length); n++ {
-			if b.bits[n%b.length] == false && n > b.length {
-				lost++
-			}
 			b.bits[n%b.length] = false
 		}
 
-		// Only record any skipped packets as a result of the window moving further than the window length
+		b.bits[i%b.length] = true
-		// Any loss within the new window will be accounted for in future calls
+		b.current = i
-		lost += max(0, int64(i-b.current-b.length))
+		//l.Debugf("missed %d packets between %d and %d\n", i-b.current, i, b.current)
+		return true
+	}
+
+	// If i is greater than the delta between current and the total length of our bitmap,
+	// just flip everything in the map and move ahead.
+	if i >= b.current+b.length {
+		// The current window loss will be accounted for later, only record the jump as loss up until then
+		lost := maxInt64(0, int64(i-b.current-b.length))
+		//TODO: explain this
+		if b.current == 0 {
+			lost++
+		}
+
+		for n := range b.bits {
+			// Don't want to count the first window as a loss
+			//TODO: this is likely wrong, we are wanting to track only the bit slots that we aren't going to track anymore and this is marking everything as missed
+			//if b.bits[n] == false {
+			//	lost++
+			//}
+			b.bits[n] = false
+		}
+
 		b.lostCounter.Inc(lost)
 
+		if l.Level >= logrus.DebugLevel {
+			l.WithField("receiveWindow", m{"accepted": true, "currentCounter": b.current, "incomingCounter": i, "reason": "window shifting"}).
+				Debug("Receive window")
+		}
 		b.bits[i%b.length] = true
 		b.current = i
 		return true
 	}
 
-	// If i is within the current window but below the current counter,
+	// Allow for the 0 packet to come in within the first window
-	// Check to see if it's a duplicate
+	if i == 0 && b.firstSeen == false && b.current < b.length {
-	if i > b.current-b.length || i < b.length && b.current < b.length {
+		b.firstSeen = true
-		if b.current == i || b.bits[i%b.length] == true {
+		b.bits[i%b.length] = true
+		return true
+	}
+
+	// If i is within the window of current minus length (the total pat window size),
+	// allow it and flip to true but to NOT change current. We also have to account for the first window
+	if ((b.current >= b.length && i > b.current-b.length) || (b.current < b.length && i < b.length)) && i <= b.current {
+		if b.current == i {
 			if l.Level >= logrus.DebugLevel {
 				l.WithField("receiveWindow", m{"accepted": false, "currentCounter": b.current, "incomingCounter": i, "reason": "duplicate"}).
 					Debug("Receive window")
@@ -95,8 +122,18 @@ func (b *Bits) Update(l *logrus.Logger, i uint64) bool {
 			return false
 		}
 
+		if b.bits[i%b.length] == true {
+			if l.Level >= logrus.DebugLevel {
+				l.WithField("receiveWindow", m{"accepted": false, "currentCounter": b.current, "incomingCounter": i, "reason": "old duplicate"}).
+					Debug("Receive window")
+			}
+			b.dupeCounter.Inc(1)
+			return false
+		}
+
 		b.bits[i%b.length] = true
 		return true
+
 	}
 
 	// In all other cases, fail and don't change current.
@@ -110,3 +147,11 @@ func (b *Bits) Update(l *logrus.Logger, i uint64) bool {
 	}
 	return false
 }
+
+func maxInt64(a, b int64) int64 {
+	if a > b {
+		return a
+	}
+
+	return b
+}

bits_test.go

@@ -15,41 +15,48 @@ func TestBits(t *testing.T) {
 	assert.Len(t, b.bits, 10)
 
 	// This is initialized to zero - receive one. This should work.
+
 	assert.True(t, b.Check(l, 1))
-	assert.True(t, b.Update(l, 1))
+	u := b.Update(l, 1)
+	assert.True(t, u)
 	assert.EqualValues(t, 1, b.current)
-	g := []bool{true, true, false, false, false, false, false, false, false, false}
+	g := []bool{false, true, false, false, false, false, false, false, false, false}
 	assert.Equal(t, g, b.bits)
 
 	// Receive two
 	assert.True(t, b.Check(l, 2))
-	assert.True(t, b.Update(l, 2))
+	u = b.Update(l, 2)
+	assert.True(t, u)
 	assert.EqualValues(t, 2, b.current)
-	g = []bool{true, true, true, false, false, false, false, false, false, false}
+	g = []bool{false, true, true, false, false, false, false, false, false, false}
 	assert.Equal(t, g, b.bits)
 
 	// Receive two again - it will fail
 	assert.False(t, b.Check(l, 2))
-	assert.False(t, b.Update(l, 2))
+	u = b.Update(l, 2)
+	assert.False(t, u)
 	assert.EqualValues(t, 2, b.current)
 
 	// Jump ahead to 15, which should clear everything and set the 6th element
 	assert.True(t, b.Check(l, 15))
-	assert.True(t, b.Update(l, 15))
+	u = b.Update(l, 15)
+	assert.True(t, u)
 	assert.EqualValues(t, 15, b.current)
 	g = []bool{false, false, false, false, false, true, false, false, false, false}
 	assert.Equal(t, g, b.bits)
 
 	// Mark 14, which is allowed because it is in the window
 	assert.True(t, b.Check(l, 14))
-	assert.True(t, b.Update(l, 14))
+	u = b.Update(l, 14)
+	assert.True(t, u)
 	assert.EqualValues(t, 15, b.current)
 	g = []bool{false, false, false, false, true, true, false, false, false, false}
 	assert.Equal(t, g, b.bits)
 
 	// Mark 5, which is not allowed because it is not in the window
 	assert.False(t, b.Check(l, 5))
-	assert.False(t, b.Update(l, 5))
+	u = b.Update(l, 5)
+	assert.False(t, u)
 	assert.EqualValues(t, 15, b.current)
 	g = []bool{false, false, false, false, true, true, false, false, false, false}
 	assert.Equal(t, g, b.bits)
@@ -62,29 +69,10 @@ func TestBits(t *testing.T) {
 
 	// Walk through a few windows in order
 	b = NewBits(10)
-	for i := uint64(1); i <= 100; i++ {
+	for i := uint64(0); i <= 100; i++ {
 		assert.True(t, b.Check(l, i), "Error while checking %v", i)
 		assert.True(t, b.Update(l, i), "Error while updating %v", i)
 	}
-
-	assert.False(t, b.Check(l, 1), "Out of window check")
-}
-
-func TestBitsLargeJumps(t *testing.T) {
-	l := test.NewLogger()
-	b := NewBits(10)
-	b.lostCounter.Clear()
-
-	b = NewBits(10)
-	b.lostCounter.Clear()
-	assert.True(t, b.Update(l, 55)) // We saw packet 55 and can still track 45,46,47,48,49,50,51,52,53,54
-	assert.Equal(t, int64(45), b.lostCounter.Count())
-
-	assert.True(t, b.Update(l, 100)) // We saw packet 55 and 100 and can still track 90,91,92,93,94,95,96,97,98,99
-	assert.Equal(t, int64(89), b.lostCounter.Count())
-
-	assert.True(t, b.Update(l, 200)) // We saw packet 55, 100, and 200 and can still track 190,191,192,193,194,195,196,197,198,199
-	assert.Equal(t, int64(188), b.lostCounter.Count())
 }
 
 func TestBitsDupeCounter(t *testing.T) {
@@ -136,7 +124,8 @@ func TestBitsOutOfWindowCounter(t *testing.T) {
 	assert.False(t, b.Update(l, 0))
 	assert.Equal(t, int64(1), b.outOfWindowCounter.Count())
 
-	assert.Equal(t, int64(19), b.lostCounter.Count()) // packet 0 wasn't lost
+	//tODO: make sure lostcounter doesn't increase in orderly increment
+	assert.Equal(t, int64(20), b.lostCounter.Count())
 	assert.Equal(t, int64(0), b.dupeCounter.Count())
 	assert.Equal(t, int64(1), b.outOfWindowCounter.Count())
 }
@@ -148,6 +137,8 @@ func TestBitsLostCounter(t *testing.T) {
 	b.dupeCounter.Clear()
 	b.outOfWindowCounter.Clear()
 
+	//assert.True(t, b.Update(0))
+	assert.True(t, b.Update(l, 0))
 	assert.True(t, b.Update(l, 20))
 	assert.True(t, b.Update(l, 21))
 	assert.True(t, b.Update(l, 22))
@@ -158,7 +149,7 @@ func TestBitsLostCounter(t *testing.T) {
 	assert.True(t, b.Update(l, 27))
 	assert.True(t, b.Update(l, 28))
 	assert.True(t, b.Update(l, 29))
-	assert.Equal(t, int64(19), b.lostCounter.Count()) // packet 0 wasn't lost
+	assert.Equal(t, int64(20), b.lostCounter.Count())
 	assert.Equal(t, int64(0), b.dupeCounter.Count())
 	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())
 
@@ -167,6 +158,8 @@ func TestBitsLostCounter(t *testing.T) {
 	b.dupeCounter.Clear()
 	b.outOfWindowCounter.Clear()
 
+	assert.True(t, b.Update(l, 0))
+	assert.Equal(t, int64(0), b.lostCounter.Count())
 	assert.True(t, b.Update(l, 9))
 	assert.Equal(t, int64(0), b.lostCounter.Count())
 	// 10 will set 0 index, 0 was already set, no lost packets
@@ -221,62 +214,6 @@ func TestBitsLostCounter(t *testing.T) {
 	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())
 }
 
-func TestBitsLostCounterIssue1(t *testing.T) {
-	l := test.NewLogger()
-	b := NewBits(10)
-	b.lostCounter.Clear()
-	b.dupeCounter.Clear()
-	b.outOfWindowCounter.Clear()
-
-	assert.True(t, b.Update(l, 4))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 1))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 9))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 2))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 3))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 5))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 6))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 7))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	// assert.True(t, b.Update(l, 8))
-	assert.True(t, b.Update(l, 10))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 11))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-
-	assert.True(t, b.Update(l, 14))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	// Issue seems to be here, we reset missing packet 8 to false here and don't increment the lost counter
-	assert.True(t, b.Update(l, 19))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 12))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 13))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 15))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 16))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 17))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 18))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 20))
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.True(t, b.Update(l, 21))
-
-	// We missed packet 8 above
-	assert.Equal(t, int64(1), b.lostCounter.Count())
-	assert.Equal(t, int64(0), b.dupeCounter.Count())
-	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())
-}
-
 func BenchmarkBits(b *testing.B) {
 	z := NewBits(10)
 	for n := 0; n < b.N; n++ {

boring.go

@@ -1,4 +1,5 @@
 //go:build boringcrypto
+// +build boringcrypto
 
 package nebula
 

calculated_remote.go

@@ -84,11 +84,16 @@ func NewCalculatedRemotesFromConfig(c *config.C, k string) (*bart.Table[[]*calcu
 
 	calculatedRemotes := new(bart.Table[[]*calculatedRemote])
 
-	rawMap, ok := value.(map[string]any)
+	rawMap, ok := value.(map[any]any)
 	if !ok {
 		return nil, fmt.Errorf("config `%s` has invalid type: %T", k, value)
 	}
-	for rawCIDR, rawValue := range rawMap {
+	for rawKey, rawValue := range rawMap {
+		rawCIDR, ok := rawKey.(string)
+		if !ok {
+			return nil, fmt.Errorf("config `%s` has invalid key (type %T): %v", k, rawKey, rawKey)
+		}
+
 		cidr, err := netip.ParsePrefix(rawCIDR)
 		if err != nil {
 			return nil, fmt.Errorf("config `%s` has invalid CIDR: %s", k, rawCIDR)
@@ -124,7 +129,7 @@ func newCalculatedRemotesListFromConfig(cidr netip.Prefix, raw any) ([]*calculat
 }
 
 func newCalculatedRemotesEntryFromConfig(cidr netip.Prefix, raw any) (*calculatedRemote, error) {
-	rawMap, ok := raw.(map[string]any)
+	rawMap, ok := raw.(map[any]any)
 	if !ok {
 		return nil, fmt.Errorf("invalid type: %T", raw)
 	}

cert/ca_pool.go

@@ -1,14 +1,11 @@
 package cert
 
 import (
-	"bufio"
-	"bytes"
-	"encoding/pem"
 	"errors"
 	"fmt"
-	"io"
 	"net/netip"
 	"slices"
+	"strings"
 	"time"
 )
 
@@ -32,46 +29,22 @@ func NewCAPool() *CAPool {
 // If the pool contains any expired certificates, an ErrExpired will be
 // returned along with the pool. The caller must handle any such errors.
 func NewCAPoolFromPEM(caPEMs []byte) (*CAPool, error) {
-	return NewCAPoolFromPEMReader(bytes.NewReader(caPEMs))
-}
-
-// NewCAPoolFromPEMReader will create a new CA pool from the provided reader.
-// The reader must contain a PEM-encoded set of nebula certificates.
-func NewCAPoolFromPEMReader(r io.Reader) (*CAPool, error) {
 	pool := NewCAPool()
-
+	var err error
 	var expired bool
-
+	for {
-	scanner := bufio.NewScanner(r)
+		caPEMs, err = pool.AddCAFromPEM(caPEMs)
-	scanner.Split(SplitPEM)
+		if errors.Is(err, ErrExpired) {
-
+			expired = true
-	for scanner.Scan() {
+			err = nil
-		pemBytes := scanner.Bytes()
-
-		block, rest := pem.Decode(pemBytes)
-		if len(bytes.TrimSpace(rest)) > 0 {
-			return nil, ErrInvalidPEMBlock
 		}
-		if block == nil {
-			return nil, ErrInvalidPEMBlock
-		}
-
-		c, err := unmarshalCertificateBlock(block)
 		if err != nil {
 			return nil, err
 		}
-
+		if len(caPEMs) == 0 || strings.TrimSpace(string(caPEMs)) == "" {
-		err = pool.AddCA(c)
+			break
-		if errors.Is(err, ErrExpired) {
-			expired = true
-			continue
-		} else if err != nil {
-			return nil, err
 		}
 	}
-	if err := scanner.Err(); err != nil {
-		return nil, ErrInvalidPEMBlock
-	}
 
 	if expired {
 		return pool, ErrExpired
@@ -168,23 +141,10 @@ func (ncp *CAPool) VerifyCertificate(now time.Time, c Certificate) (*CachedCerti
 		return nil, err
 	}
 
-	// Pre nebula v1.10.3 could generate signatures in either high or low s form and validation
-	// of signatures allowed for either. Nebula v1.10.3 and beyond clamps signature generation to low-s form
-	// but validation still allows for either. Since a change in the signature bytes affects the fingerprint, we
-	// need to test both forms until such a time comes that we enforce low-s form on signature validation.
-	fp2, err := CalculateAlternateFingerprint(c)
-	if err != nil {
-		return nil, fmt.Errorf("could not calculate alternate fingerprint to verify: %w", err)
-	}
-	if fp2 != "" && ncp.IsBlocklisted(fp2) {
-		return nil, ErrBlockListed
-	}
-
 	cc := CachedCertificate{
 		Certificate:       c,
 		InvertedGroups:    make(map[string]struct{}),
 		Fingerprint:       fp,
-		fingerprint2:      fp2,
 		signerFingerprint: signer.Fingerprint,
 	}
 
@@ -198,11 +158,6 @@ func (ncp *CAPool) VerifyCertificate(now time.Time, c Certificate) (*CachedCerti
 // VerifyCachedCertificate is the same as VerifyCertificate other than it operates on a pre-verified structure and
 // is a cheaper operation to perform as a result.
 func (ncp *CAPool) VerifyCachedCertificate(now time.Time, c *CachedCertificate) error {
-	// Check any available alternate fingerprint forms for this certificate, re P256 high-s/low-s
-	if c.fingerprint2 != "" && ncp.IsBlocklisted(c.fingerprint2) {
-		return ErrBlockListed
-	}
-
 	_, err := ncp.verify(c.Certificate, now, c.Fingerprint, c.signerFingerprint)
 	return err
 }

cert/ca_pool_test.go

@@ -1,14 +1,10 @@
 package cert
 
 import (
-	"bytes"
-	"io"
 	"net/netip"
-	"strings"
 	"testing"
 	"time"
 
-	"github.com/slackhq/nebula/cert/p256"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -115,60 +111,6 @@ k+coOv04r+zh33ISyhbsafnYduN17p2eD7CmHvHuerguXD9f32gcxo/KsFCKEjMe
 	assert.Len(t, ppppp.CAs, 1)
 }
 
-// oneByteReader wraps a reader to return at most 1 byte per Read call,
-// exercising the streaming accumulation logic in NewCAPoolFromPEMReader.
-type oneByteReader struct {
-	r io.Reader
-}
-
-func (o *oneByteReader) Read(p []byte) (int, error) {
-	if len(p) == 0 {
-		return 0, nil
-	}
-	return o.r.Read(p[:1])
-}
-
-func TestNewCAPoolFromPEMReader_EmptyReader(t *testing.T) {
-	pool, err := NewCAPoolFromPEMReader(bytes.NewReader(nil))
-	require.NoError(t, err)
-	assert.Empty(t, pool.CAs)
-
-	pool, err = NewCAPoolFromPEMReader(strings.NewReader("   \n\t\n  "))
-	require.NoError(t, err)
-	assert.Empty(t, pool.CAs)
-}
-
-func TestNewCAPoolFromPEMReader_OneByteReads(t *testing.T) {
-	ca1, _, _, pem1 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil)
-	ca2, _, _, pem2 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil)
-
-	bundle := append(pem1, pem2...)
-	pool, err := NewCAPoolFromPEMReader(&oneByteReader{r: bytes.NewReader(bundle)})
-	require.NoError(t, err)
-	assert.Len(t, pool.CAs, 2)
-
-	fp1, err := ca1.Fingerprint()
-	require.NoError(t, err)
-	fp2, err := ca2.Fingerprint()
-	require.NoError(t, err)
-
-	assert.Contains(t, pool.CAs, fp1)
-	assert.Contains(t, pool.CAs, fp2)
-}
-
-func TestNewCAPoolFromPEMReader_TruncatedPEM(t *testing.T) {
-	_, err := NewCAPoolFromPEMReader(strings.NewReader("-----BEGIN NEBULA CERTIFICATE-----\npartialdata"))
-	assert.ErrorIs(t, err, ErrInvalidPEMBlock)
-}
-
-func TestNewCAPoolFromPEMReader_TrailingGarbage(t *testing.T) {
-	_, _, _, pem1 := NewTestCaCert(Version2, Curve_CURVE25519, time.Now(), time.Now().Add(time.Hour), nil, nil, nil)
-
-	bundle := append(pem1, []byte("some trailing garbage")...)
-	_, err := NewCAPoolFromPEMReader(bytes.NewReader(bundle))
-	assert.ErrorIs(t, err, ErrInvalidPEMBlock)
-}
-
 func TestCertificateV1_Verify(t *testing.T) {
 	ca, _, caKey, _ := NewTestCaCert(Version1, Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, nil)
 	c, _, _, _ := NewTestCert(Version1, Curve_CURVE25519, ca, caKey, "test cert", time.Now(), time.Now().Add(5*time.Minute), nil, nil, nil)
@@ -228,15 +170,6 @@ func TestCertificateV1_VerifyP256(t *testing.T) {
 	_, err = caPool.VerifyCertificate(time.Now(), c)
 	require.EqualError(t, err, "certificate is in the block list")
 
-	// Create a copy of the cert and swap to the alternate form for the signature
-	nc := c.Copy()
-	b, err := p256.Swap(c.Signature())
-	require.NoError(t, err)
-	require.NoError(t, nc.(*certificateV1).setSignature(b))
-
-	_, err = caPool.VerifyCertificate(time.Now(), nc)
-	require.EqualError(t, err, "certificate is in the block list")
-
 	caPool.ResetCertBlocklist()
 	_, err = caPool.VerifyCertificate(time.Now(), c)
 	require.NoError(t, err)
@@ -254,7 +187,7 @@ func TestCertificateV1_VerifyP256(t *testing.T) {
 	require.NoError(t, err)
 
 	caPool = NewCAPool()
-	b, err = caPool.AddCAFromPEM(caPem)
+	b, err := caPool.AddCAFromPEM(caPem)
 	require.NoError(t, err)
 	assert.Empty(t, b)
 
@@ -263,17 +196,7 @@ func TestCertificateV1_VerifyP256(t *testing.T) {
 	})
 
 	c, _, _, _ = NewTestCert(Version1, Curve_P256, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, nil, []string{"test1"})
-	cc, err := caPool.VerifyCertificate(time.Now(), c)
+	_, err = caPool.VerifyCertificate(time.Now(), c)
-	require.NoError(t, err)
-
-	// Reset the blocklist and block the alternate form fingerprint
-	caPool.ResetCertBlocklist()
-	caPool.BlocklistFingerprint(cc.fingerprint2)
-	err = caPool.VerifyCachedCertificate(time.Now(), cc)
-	require.EqualError(t, err, "certificate is in the block list")
-
-	caPool.ResetCertBlocklist()
-	err = caPool.VerifyCachedCertificate(time.Now(), cc)
 	require.NoError(t, err)
 }
 
@@ -471,15 +394,6 @@ func TestCertificateV2_VerifyP256(t *testing.T) {
 	_, err = caPool.VerifyCertificate(time.Now(), c)
 	require.EqualError(t, err, "certificate is in the block list")
 
-	// Create a copy of the cert and swap to the alternate form for the signature
-	nc := c.Copy()
-	b, err := p256.Swap(c.Signature())
-	require.NoError(t, err)
-	require.NoError(t, nc.(*certificateV2).setSignature(b))
-
-	_, err = caPool.VerifyCertificate(time.Now(), nc)
-	require.EqualError(t, err, "certificate is in the block list")
-
 	caPool.ResetCertBlocklist()
 	_, err = caPool.VerifyCertificate(time.Now(), c)
 	require.NoError(t, err)
@@ -497,7 +411,7 @@ func TestCertificateV2_VerifyP256(t *testing.T) {
 	require.NoError(t, err)
 
 	caPool = NewCAPool()
-	b, err = caPool.AddCAFromPEM(caPem)
+	b, err := caPool.AddCAFromPEM(caPem)
 	require.NoError(t, err)
 	assert.Empty(t, b)
 
@@ -506,17 +420,7 @@ func TestCertificateV2_VerifyP256(t *testing.T) {
 	})
 
 	c, _, _, _ = NewTestCert(Version2, Curve_P256, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), nil, nil, []string{"test1"})
-	cc, err := caPool.VerifyCertificate(time.Now(), c)
+	_, err = caPool.VerifyCertificate(time.Now(), c)
-	require.NoError(t, err)
-
-	// Reset the blocklist and block the alternate form fingerprint
-	caPool.ResetCertBlocklist()
-	caPool.BlocklistFingerprint(cc.fingerprint2)
-	err = caPool.VerifyCachedCertificate(time.Now(), cc)
-	require.EqualError(t, err, "certificate is in the block list")
-
-	caPool.ResetCertBlocklist()
-	err = caPool.VerifyCachedCertificate(time.Now(), cc)
 	require.NoError(t, err)
 }
 

cert/cert.go

@@ -4,8 +4,6 @@ import (
 	"fmt"
 	"net/netip"
 	"time"
-
-	"github.com/slackhq/nebula/cert/p256"
 )
 
 type Version uint8
@@ -60,9 +58,6 @@ type Certificate interface {
 	// PublicKey is the raw bytes to be used in asymmetric cryptographic operations.
 	PublicKey() []byte
 
-	// MarshalPublicKeyPEM is the value of PublicKey marshalled to PEM
-	MarshalPublicKeyPEM() []byte
-
 	// Curve identifies which curve was used for the PublicKey and Signature.
 	Curve() Curve
 
@@ -112,9 +107,6 @@ type CachedCertificate struct {
 	InvertedGroups    map[string]struct{}
 	Fingerprint       string
 	signerFingerprint string
-
-	// A place to store a 2nd fingerprint if the certificate could have one, such as with P256
-	fingerprint2 string
 }
 
 func (cc *CachedCertificate) String() string {
@@ -124,7 +116,6 @@ func (cc *CachedCertificate) String() string {
 // Recombine will attempt to unmarshal a certificate received in a handshake.
 // Handshakes save space by placing the peers public key in a different part of the packet, we have to
 // reassemble the actual certificate structure with that in mind.
-// Implementations MUST assert the public key is not in the raw certificate bytes if the passed in public key is not empty.
 func Recombine(v Version, rawCertBytes, publicKey []byte, curve Curve) (Certificate, error) {
 	if publicKey == nil {
 		return nil, ErrNoPeerStaticKey
@@ -144,7 +135,8 @@ func Recombine(v Version, rawCertBytes, publicKey []byte, curve Curve) (Certific
 	case Version2:
 		c, err = unmarshalCertificateV2(rawCertBytes, publicKey, curve)
 	default:
-		return nil, ErrUnknownVersion
+		//TODO: CERT-V2 make a static var
+		return nil, fmt.Errorf("unknown certificate version %d", v)
 	}
 
 	if err != nil {
@@ -157,31 +149,3 @@ func Recombine(v Version, rawCertBytes, publicKey []byte, curve Curve) (Certific
 
 	return c, nil
 }
-
-// CalculateAlternateFingerprint calculates a 2nd fingerprint representation for P256 certificates
-// CAPool blocklist testing through `VerifyCertificate` and `VerifyCachedCertificate` automatically performs this step.
-func CalculateAlternateFingerprint(c Certificate) (string, error) {
-	if c.Curve() != Curve_P256 {
-		return "", nil
-	}
-
-	nc := c.Copy()
-	b, err := p256.Swap(nc.Signature())
-	if err != nil {
-		return "", err
-	}
-
-	switch v := nc.(type) {
-	case *certificateV1:
-		err = v.setSignature(b)
-	case *certificateV2:
-		err = v.setSignature(b)
-	default:
-		return "", ErrUnknownVersion
-	}
-
-	if err != nil {
-		return "", err
-	}
-	return nc.Fingerprint()
-}

cert/cert_v1.go

@@ -41,7 +41,7 @@ type detailsV1 struct {
 	curve Curve
 }
 
-type m = map[string]any
+type m map[string]interface{}
 
 func (c *certificateV1) Version() Version {
 	return Version1
@@ -83,10 +83,6 @@ func (c *certificateV1) PublicKey() []byte {
 	return c.details.publicKey
 }
 
-func (c *certificateV1) MarshalPublicKeyPEM() []byte {
-	return marshalCertPublicKeyToPEM(c)
-}
-
 func (c *certificateV1) Signature() []byte {
 	return c.signature
 }
@@ -114,10 +110,8 @@ func (c *certificateV1) CheckSignature(key []byte) bool {
 	case Curve_CURVE25519:
 		return ed25519.Verify(key, b, c.signature)
 	case Curve_P256:
-		pubKey, err := ecdsa.ParseUncompressedPublicKey(elliptic.P256(), key)
+		x, y := elliptic.Unmarshal(elliptic.P256(), key)
-		if err != nil {
+		pubKey := &ecdsa.PublicKey{Curve: elliptic.P256(), X: x, Y: y}
-			return false
-		}
 		hashed := sha256.Sum256(b)
 		return ecdsa.VerifyASN1(pubKey, hashed[:], c.signature)
 	default:
@@ -426,7 +420,7 @@ func unmarshalCertificateV1(b []byte, publicKey []byte) (*certificateV1, error)
 			unsafeNetworks: make([]netip.Prefix, len(rc.Details.Subnets)/2),
 			notBefore:      time.Unix(rc.Details.NotBefore, 0),
 			notAfter:       time.Unix(rc.Details.NotAfter, 0),
-			publicKey:      nil,
+			publicKey:      make([]byte, len(rc.Details.PublicKey)),
 			isCA:           rc.Details.IsCA,
 			curve:          rc.Details.Curve,
 		},
@@ -437,19 +431,12 @@ func unmarshalCertificateV1(b []byte, publicKey []byte) (*certificateV1, error)
 	copy(nc.details.groups, rc.Details.Groups)
 	nc.details.issuer = hex.EncodeToString(rc.Details.Issuer)
 
-	// If a public key is passed in as an argument, the certificate pubkey must be empty
-	// and the passed-in pubkey copied into the cert.
 	if len(publicKey) > 0 {
-		if len(rc.Details.PublicKey) != 0 {
+		nc.details.publicKey = publicKey
-			return nil, ErrCertPubkeyPresent
-		}
-		nc.details.publicKey = make([]byte, len(publicKey))
-		copy(nc.details.publicKey, publicKey)
-	} else {
-		nc.details.publicKey = make([]byte, len(rc.Details.PublicKey))
-		copy(nc.details.publicKey, rc.Details.PublicKey)
 	}
 
+	copy(nc.details.publicKey, rc.Details.PublicKey)
+
 	var ip netip.Addr
 	for i, rawIp := range rc.Details.Ips {
 		if i%2 == 0 {

cert/cert_v1_test.go

@@ -1,7 +1,6 @@
 package cert
 
 import (
-	"crypto/ed25519"
 	"fmt"
 	"net/netip"
 	"testing"
@@ -14,7 +13,6 @@ import (
 )
 
 func TestCertificateV1_Marshal(t *testing.T) {
-	t.Parallel()
 	before := time.Now().Add(time.Second * -60).Round(time.Second)
 	after := time.Now().Add(time.Second * 60).Round(time.Second)
 	pubKey := []byte("1234567890abcedfghij1234567890ab")
@@ -62,120 +60,6 @@ func TestCertificateV1_Marshal(t *testing.T) {
 	assert.Equal(t, nc.Groups(), nc2.Groups())
 }
 
-func TestCertificateV1_Unmarshal(t *testing.T) {
-	t.Parallel()
-	before := time.Now().Add(time.Second * -60).Round(time.Second)
-	after := time.Now().Add(time.Second * 60).Round(time.Second)
-	pubKey := []byte("1234567890abcedfghij1234567890ab")
-	invalidPubkey := []byte("00000000000000000000000000000000")
-
-	nc := certificateV1{
-		details: detailsV1{
-			name: "testing",
-			networks: []netip.Prefix{
-				mustParsePrefixUnmapped("10.1.1.1/24"),
-				mustParsePrefixUnmapped("10.1.1.2/16"),
-			},
-			unsafeNetworks: []netip.Prefix{
-				mustParsePrefixUnmapped("9.1.1.2/24"),
-				mustParsePrefixUnmapped("9.1.1.3/16"),
-			},
-			groups:    []string{"test-group1", "test-group2", "test-group3"},
-			notBefore: before,
-			notAfter:  after,
-			publicKey: pubKey,
-			isCA:      false,
-			issuer:    "1234567890abcedfghij1234567890ab",
-		},
-		signature: []byte("1234567890abcedfghij1234567890ab"),
-	}
-
-	// This certificate has a pubkey included
-	certWithPubkey, err := nc.Marshal()
-	require.NoError(t, err)
-
-	// This certificate is missing the pubkey section
-	certWithoutPubkey, err := nc.MarshalForHandshakes()
-	require.NoError(t, err)
-
-	// Cert has no pubkey and no pubkey passed in must fail to validate
-	isNil, err := unmarshalCertificateV1(certWithoutPubkey, nil)
-	require.Error(t, err)
-
-	// Cert has different pubkey than one passed in must fail
-	isNil, err = unmarshalCertificateV1(certWithPubkey, invalidPubkey)
-	require.Nil(t, isNil)
-	require.Error(t, err)
-
-	// Cert has pubkey and no pubkey argument works ok
-	_, err = unmarshalCertificateV1(certWithPubkey, nil)
-	require.NoError(t, err)
-
-	// Cert has no pubkey and valid, correctly signed pubkey passed in
-	nc2, err := unmarshalCertificateV1(certWithoutPubkey, pubKey)
-	require.NoError(t, err)
-
-	assert.Equal(t, pubKey, nc2.PublicKey())
-}
-
-func TestCertificateV1_PublicKeyPem(t *testing.T) {
-	t.Parallel()
-	before := time.Now().Add(time.Second * -60).Round(time.Second)
-	after := time.Now().Add(time.Second * 60).Round(time.Second)
-	pubKey := ed25519.PublicKey("1234567890abcedfghij1234567890ab")
-
-	nc := certificateV1{
-		details: detailsV1{
-			name:           "testing",
-			networks:       []netip.Prefix{},
-			unsafeNetworks: []netip.Prefix{},
-			groups:         []string{"test-group1", "test-group2", "test-group3"},
-			notBefore:      before,
-			notAfter:       after,
-			publicKey:      pubKey,
-			isCA:           false,
-			issuer:         "1234567890abcedfghij1234567890ab",
-		},
-		signature: []byte("1234567890abcedfghij1234567890ab"),
-	}
-
-	assert.Equal(t, Version1, nc.Version())
-	assert.Equal(t, Curve_CURVE25519, nc.Curve())
-	pubPem := "-----BEGIN NEBULA X25519 PUBLIC KEY-----\nMTIzNDU2Nzg5MGFiY2VkZmdoaWoxMjM0NTY3ODkwYWI=\n-----END NEBULA X25519 PUBLIC KEY-----\n"
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), pubPem)
-	assert.False(t, nc.IsCA())
-
-	nc.details.isCA = true
-	assert.Equal(t, Curve_CURVE25519, nc.Curve())
-	pubPem = "-----BEGIN NEBULA ED25519 PUBLIC KEY-----\nMTIzNDU2Nzg5MGFiY2VkZmdoaWoxMjM0NTY3ODkwYWI=\n-----END NEBULA ED25519 PUBLIC KEY-----\n"
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), pubPem)
-	assert.True(t, nc.IsCA())
-
-	pubP256KeyPem := []byte(`-----BEGIN NEBULA P256 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA P256 PUBLIC KEY-----
-`)
-
-	pubP256KeyPemCA := []byte(`-----BEGIN NEBULA ECDSA P256 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA ECDSA P256 PUBLIC KEY-----
-`)
-	pubP256Key, _, _, err := UnmarshalPublicKeyFromPEM(pubP256KeyPem)
-	require.NoError(t, err)
-	nc.details.curve = Curve_P256
-	nc.details.publicKey = pubP256Key
-	assert.Equal(t, Curve_P256, nc.Curve())
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPemCA))
-	assert.True(t, nc.IsCA())
-
-	nc.details.isCA = false
-	assert.Equal(t, Curve_P256, nc.Curve())
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPem))
-	assert.False(t, nc.IsCA())
-}
-
 func TestCertificateV1_Expired(t *testing.T) {
 	nc := certificateV1{
 		details: detailsV1{

cert/cert_v2.go

@@ -114,10 +114,6 @@ func (c *certificateV2) PublicKey() []byte {
 	return c.publicKey
 }
 
-func (c *certificateV2) MarshalPublicKeyPEM() []byte {
-	return marshalCertPublicKeyToPEM(c)
-}
-
 func (c *certificateV2) Signature() []byte {
 	return c.signature
 }
@@ -153,10 +149,8 @@ func (c *certificateV2) CheckSignature(key []byte) bool {
 	case Curve_CURVE25519:
 		return ed25519.Verify(key, b, c.signature)
 	case Curve_P256:
-		pubKey, err := ecdsa.ParseUncompressedPublicKey(elliptic.P256(), key)
+		x, y := elliptic.Unmarshal(elliptic.P256(), key)
-		if err != nil {
+		pubKey := &ecdsa.PublicKey{Curve: elliptic.P256(), X: x, Y: y}
-			return false
-		}
 		hashed := sha256.Sum256(b)
 		return ecdsa.VerifyASN1(pubKey, hashed[:], c.signature)
 	default:
@@ -592,13 +586,7 @@ func unmarshalCertificateV2(b []byte, publicKey []byte, curve Curve) (*certifica
 	// Maybe grab the public key
 	var rawPublicKey cryptobyte.String
 	if len(publicKey) > 0 {
-		// If a public key is passed in, then the handshake certificate must
+		rawPublicKey = publicKey
-		// not have a public key present
-		if input.PeekASN1Tag(TagCertPublicKey) {
-			return nil, ErrCertPubkeyPresent
-		}
-		rawPublicKey = make(cryptobyte.String, len(publicKey))
-		copy(rawPublicKey, publicKey)
 	} else if !input.ReadOptionalASN1(&rawPublicKey, nil, TagCertPublicKey) {
 		return nil, ErrBadFormat
 	}

cert/cert_v2_test.go

@@ -15,7 +15,6 @@ import (
 )
 
 func TestCertificateV2_Marshal(t *testing.T) {
-	t.Parallel()
 	before := time.Now().Add(time.Second * -60).Round(time.Second)
 	after := time.Now().Add(time.Second * 60).Round(time.Second)
 	pubKey := []byte("1234567890abcedfghij1234567890ab")
@@ -76,117 +75,6 @@ func TestCertificateV2_Marshal(t *testing.T) {
 	assert.Equal(t, nc.Groups(), nc2.Groups())
 }
 
-func TestCertificateV2_Unmarshal(t *testing.T) {
-	t.Parallel()
-	before := time.Now().Add(time.Second * -60).Round(time.Second)
-	after := time.Now().Add(time.Second * 60).Round(time.Second)
-	pubKey := []byte("1234567890abcedfghij1234567890ab")
-
-	nc := certificateV2{
-		details: detailsV2{
-			name: "testing",
-			networks: []netip.Prefix{
-				mustParsePrefixUnmapped("10.1.1.2/16"),
-				mustParsePrefixUnmapped("10.1.1.1/24"),
-			},
-			unsafeNetworks: []netip.Prefix{
-				mustParsePrefixUnmapped("9.1.1.3/16"),
-				mustParsePrefixUnmapped("9.1.1.2/24"),
-			},
-			groups:    []string{"test-group1", "test-group2", "test-group3"},
-			notBefore: before,
-			notAfter:  after,
-			isCA:      false,
-			issuer:    "1234567890abcdef1234567890abcdef",
-		},
-		signature: []byte("1234567890abcdef1234567890abcdef"),
-		publicKey: pubKey,
-	}
-
-	db, err := nc.details.Marshal()
-	require.NoError(t, err)
-	nc.rawDetails = db
-
-	certWithPubkey, err := nc.Marshal()
-	require.NoError(t, err)
-	//t.Log("Cert size:", len(b))
-	certWithoutPubkey, err := nc.MarshalForHandshakes()
-	require.NoError(t, err)
-
-	// Cert must not have a pubkey if one is passed in as an argument
-	_, err = unmarshalCertificateV2(certWithPubkey, pubKey, Curve_CURVE25519)
-	require.ErrorIs(t, err, ErrCertPubkeyPresent)
-
-	// Certs must have pubkeys
-	_, err = unmarshalCertificateV2(certWithoutPubkey, nil, Curve_CURVE25519)
-	require.ErrorIs(t, err, ErrBadFormat)
-
-	// Ensure proper unmarshal if a pubkey is passed in
-	nc2, err := unmarshalCertificateV2(certWithoutPubkey, pubKey, Curve_CURVE25519)
-	require.NoError(t, err)
-
-	assert.Equal(t, nc.PublicKey(), nc2.PublicKey())
-}
-
-func TestCertificateV2_PublicKeyPem(t *testing.T) {
-	t.Parallel()
-	before := time.Now().Add(time.Second * -60).Round(time.Second)
-	after := time.Now().Add(time.Second * 60).Round(time.Second)
-	pubKey := ed25519.PublicKey("1234567890abcedfghij1234567890ab")
-
-	nc := certificateV2{
-		details: detailsV2{
-			name:           "testing",
-			networks:       []netip.Prefix{},
-			unsafeNetworks: []netip.Prefix{},
-			groups:         []string{"test-group1", "test-group2", "test-group3"},
-			notBefore:      before,
-			notAfter:       after,
-			isCA:           false,
-			issuer:         "1234567890abcedfghij1234567890ab",
-		},
-		publicKey: pubKey,
-		signature: []byte("1234567890abcedfghij1234567890ab"),
-	}
-
-	assert.Equal(t, Version2, nc.Version())
-	assert.Equal(t, Curve_CURVE25519, nc.Curve())
-	pubPem := "-----BEGIN NEBULA X25519 PUBLIC KEY-----\nMTIzNDU2Nzg5MGFiY2VkZmdoaWoxMjM0NTY3ODkwYWI=\n-----END NEBULA X25519 PUBLIC KEY-----\n"
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), pubPem)
-	assert.False(t, nc.IsCA())
-
-	nc.details.isCA = true
-	assert.Equal(t, Curve_CURVE25519, nc.Curve())
-	pubPem = "-----BEGIN NEBULA ED25519 PUBLIC KEY-----\nMTIzNDU2Nzg5MGFiY2VkZmdoaWoxMjM0NTY3ODkwYWI=\n-----END NEBULA ED25519 PUBLIC KEY-----\n"
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), pubPem)
-	assert.True(t, nc.IsCA())
-
-	pubP256KeyPem := []byte(`-----BEGIN NEBULA P256 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA P256 PUBLIC KEY-----
-`)
-
-	pubP256KeyPemCA := []byte(`-----BEGIN NEBULA ECDSA P256 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA ECDSA P256 PUBLIC KEY-----
-`)
-
-	pubP256Key, _, _, err := UnmarshalPublicKeyFromPEM(pubP256KeyPem)
-	require.NoError(t, err)
-	nc.curve = Curve_P256
-	nc.publicKey = pubP256Key
-	assert.Equal(t, Curve_P256, nc.Curve())
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPemCA))
-	assert.True(t, nc.IsCA())
-
-	nc.details.isCA = false
-	assert.Equal(t, Curve_P256, nc.Curve())
-	assert.Equal(t, string(nc.MarshalPublicKeyPEM()), string(pubP256KeyPem))
-	assert.False(t, nc.IsCA())
-}
-
 func TestCertificateV2_Expired(t *testing.T) {
 	nc := certificateV2{
 		details: detailsV2{

cert/crypto_test.go

@@ -10,14 +10,14 @@ import (
 
 func TestNewArgon2Parameters(t *testing.T) {
 	p := NewArgon2Parameters(64*1024, 4, 3)
-	assert.Equal(t, &Argon2Parameters{
+	assert.EqualValues(t, &Argon2Parameters{
 		version:     argon2.Version,
 		Memory:      64 * 1024,
 		Parallelism: 4,
 		Iterations:  3,
 	}, p)
 	p = NewArgon2Parameters(2*1024*1024, 2, 1)
-	assert.Equal(t, &Argon2Parameters{
+	assert.EqualValues(t, &Argon2Parameters{
 		version:     argon2.Version,
 		Memory:      2 * 1024 * 1024,
 		Parallelism: 2,
@@ -26,21 +26,21 @@ func TestNewArgon2Parameters(t *testing.T) {
 }
 
 func TestDecryptAndUnmarshalSigningPrivateKey(t *testing.T) {
-	passphrase := []byte("DO NOT USE")
+	passphrase := []byte("DO NOT USE THIS KEY")
 	privKey := []byte(`# A good key
 -----BEGIN NEBULA ED25519 ENCRYPTED PRIVATE KEY-----
-CjsKC0FFUy0yNTYtR0NNEiwIExCAgAQYAyAEKiCPoDfGQiosxNPTbPn5EsMlc2MI
+CjwKC0FFUy0yNTYtR0NNEi0IExCAgIABGAEgBCognnjujd67Vsv99p22wfAjQaDT
-c0Bt4oz6gTrFQhX3aBJcimhHKeAuhyTGvllD0Z19fe+DFPcLH3h5VrdjVfIAajg0
+oCMW1mdjkU3gACKNW4MSXOWR9Sts4C81yk1RUku2gvGKs3TB9LYoklLsIizSYOLl
-KrbV3n9UHif/Au5skWmquNJzoW1E4MTdRbvpti6o+WdQ49DxjBFhx0YH8LBqrbPU
++Vs//O1T0I1Xbml2XBAROsb/VSoDln/6LMqR4B6fn6B3GOsLBBqRI8daDl9lRMPB
-0BGkUHmIO7daP24=
+qrlJ69wer3ZUHFXA
 -----END NEBULA ED25519 ENCRYPTED PRIVATE KEY-----
 `)
 	shortKey := []byte(`# A key which, once decrypted, is too short
 -----BEGIN NEBULA ED25519 ENCRYPTED PRIVATE KEY-----
-CjsKC0FFUy0yNTYtR0NNEiwIExCAgAQYAyAEKiAVJwdfl3r+eqi/vF6S7OMdpjfo
+CjwKC0FFUy0yNTYtR0NNEi0IExCAgIABGAEgBCoga5h8owMEBWRSMMJKzuUvWce7
-hAzmTCRnr58Su4AqmBJbCv3zleYCEKYJP6UI3S8ekLMGISsgO4hm5leukCCyqT0Z
+k0qlBkQmCxiuLh80MuASW70YcKt8jeEIS2axo2V6zAKA9TSMcCsJW1kDDXEtL/xe
-cQ76yrberpzkJKoPLGisX8f+xdy4aXSZl7oEYWQte1+vqbtl/eY9PGZhxUQdcyq7
+GLF5T7sDl5COp4LU3pGxpV+KoeQ/S3gQCAAcnaOtnJQX+aSDnbO3jCHyP7U9CHbs
-hqzIyrRqfUgVuA==
+rQr3bdH3Oy/WiYU=
 -----END NEBULA ED25519 ENCRYPTED PRIVATE KEY-----
 `)
 	invalidBanner := []byte(`# Invalid banner (not encrypted)
@@ -79,7 +79,7 @@ qrlJ69wer3ZUHFXA
 	assert.Nil(t, k)
 	assert.Equal(t, rest, invalidPem)
 
-	// Fail due to invalid PEM format, because
+	// Fail due to ivalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	curve, k, rest, err = DecryptAndUnmarshalSigningPrivateKey(passphrase, rest)
 	require.EqualError(t, err, "input did not contain a valid PEM encoded block")

cert/errors.go

@@ -20,8 +20,6 @@ var (
 	ErrPublicPrivateKeyMismatch   = errors.New("public key and private key are not a pair")
 	ErrPrivateKeyEncrypted        = errors.New("private key must be decrypted")
 	ErrCaNotFound                 = errors.New("could not find ca for the certificate")
-	ErrUnknownVersion             = errors.New("certificate version unrecognized")
-	ErrCertPubkeyPresent          = errors.New("certificate has unexpected pubkey present")
 
 	ErrInvalidPEMBlock                   = errors.New("input did not contain a valid PEM encoded block")
 	ErrInvalidPEMCertificateBanner       = errors.New("bytes did not contain a proper certificate banner")

cert/p256/p256.go

@@ -1,127 +0,0 @@
-package p256
-
-import (
-	"crypto/elliptic"
-	"errors"
-	"math/big"
-
-	"filippo.io/bigmod"
-
-	"golang.org/x/crypto/cryptobyte"
-	"golang.org/x/crypto/cryptobyte/asn1"
-)
-
-var halfN = new(big.Int).Rsh(elliptic.P256().Params().N, 1)
-var nMod *bigmod.Modulus
-
-func init() {
-	n, err := bigmod.NewModulus(elliptic.P256().Params().N.Bytes())
-	if err != nil {
-		panic(err)
-	}
-	nMod = n
-}
-
-func IsNormalized(sig []byte) (bool, error) {
-	r, s, err := parseSignature(sig)
-	if err != nil {
-		return false, err
-	}
-	return checkLowS(r, s), nil
-}
-
-func checkLowS(_, s []byte) bool {
-	bigS := new(big.Int).SetBytes(s)
-	// Check if S <= (N/2), because we want to include the midpoint in the set of low-s
-	return bigS.Cmp(halfN) <= 0
-}
-
-func swap(r, s []byte) ([]byte, []byte, error) {
-	var err error
-	bigS, err := bigmod.NewNat().SetBytes(s, nMod)
-	if err != nil {
-		return nil, nil, err
-	}
-	sNormalized := nMod.Nat().Sub(bigS, nMod)
-
-	result := sNormalized.Bytes(nMod)
-	for len(result) > 1 && result[0] == 0 {
-		result = result[1:]
-	}
-
-	return r, result, nil
-}
-
-func Normalize(sig []byte) ([]byte, error) {
-	r, s, err := parseSignature(sig)
-	if err != nil {
-		return nil, err
-	}
-
-	if checkLowS(r, s) {
-		return sig, nil
-	}
-
-	newR, newS, err := swap(r, s)
-	if err != nil {
-		return nil, err
-	}
-
-	return encodeSignature(newR, newS)
-}
-
-// Swap will change sig between its current form to the opposite high or low form.
-func Swap(sig []byte) ([]byte, error) {
-	r, s, err := parseSignature(sig)
-	if err != nil {
-		return nil, err
-	}
-
-	newR, newS, err := swap(r, s)
-	if err != nil {
-		return nil, err
-	}
-
-	return encodeSignature(newR, newS)
-}
-
-// parseSignature taken exactly from crypto/ecdsa/ecdsa.go
-func parseSignature(sig []byte) (r, s []byte, err error) {
-	var inner cryptobyte.String
-	input := cryptobyte.String(sig)
-	if !input.ReadASN1(&inner, asn1.SEQUENCE) ||
-		!input.Empty() ||
-		!inner.ReadASN1Integer(&r) ||
-		!inner.ReadASN1Integer(&s) ||
-		!inner.Empty() {
-		return nil, nil, errors.New("invalid ASN.1")
-	}
-	return r, s, nil
-}
-
-func encodeSignature(r, s []byte) ([]byte, error) {
-	var b cryptobyte.Builder
-	b.AddASN1(asn1.SEQUENCE, func(b *cryptobyte.Builder) {
-		addASN1IntBytes(b, r)
-		addASN1IntBytes(b, s)
-	})
-	return b.Bytes()
-}
-
-// addASN1IntBytes encodes in ASN.1 a positive integer represented as
-// a big-endian byte slice with zero or more leading zeroes.
-func addASN1IntBytes(b *cryptobyte.Builder, bytes []byte) {
-	for len(bytes) > 0 && bytes[0] == 0 {
-		bytes = bytes[1:]
-	}
-	if len(bytes) == 0 {
-		b.SetError(errors.New("invalid integer"))
-		return
-	}
-	b.AddASN1(asn1.INTEGER, func(c *cryptobyte.Builder) {
-		if bytes[0]&0x80 != 0 {
-			c.AddUint8(0)
-		}
-		c.AddBytes(bytes)
-	})
-}

cert/p256/p256_test.go

@@ -1,28 +0,0 @@
-package p256
-
-import (
-	"crypto/ecdsa"
-	"crypto/elliptic"
-	"crypto/rand"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestFlipping(t *testing.T) {
-	priv, err1 := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	require.NoError(t, err1)
-
-	out, err := ecdsa.SignASN1(rand.Reader, priv, []byte("big chungus"))
-	require.NoError(t, err)
-
-	r, s, err := parseSignature(out)
-	require.NoError(t, err)
-
-	r, s1, err := swap(r, s)
-	require.NoError(t, err)
-	r, s2, err := swap(r, s1)
-	require.NoError(t, err)
-	require.Equal(t, s, s2)
-	require.NotEqual(t, s, s1)
-}

cert/pem.go

@@ -1,86 +1,25 @@
 package cert
 
 import (
-	"bytes"
 	"encoding/pem"
-	"errors"
 	"fmt"
 
 	"golang.org/x/crypto/ed25519"
 )
 
-var ErrTruncatedPEMBlock = errors.New("truncated PEM block")
+const (
+	CertificateBanner                = "NEBULA CERTIFICATE"
+	CertificateV2Banner              = "NEBULA CERTIFICATE V2"
+	X25519PrivateKeyBanner           = "NEBULA X25519 PRIVATE KEY"
+	X25519PublicKeyBanner            = "NEBULA X25519 PUBLIC KEY"
+	EncryptedEd25519PrivateKeyBanner = "NEBULA ED25519 ENCRYPTED PRIVATE KEY"
+	Ed25519PrivateKeyBanner          = "NEBULA ED25519 PRIVATE KEY"
+	Ed25519PublicKeyBanner           = "NEBULA ED25519 PUBLIC KEY"
 
-// SplitPEM is a split function for bufio.Scanner that returns each PEM block.
+	P256PrivateKeyBanner               = "NEBULA P256 PRIVATE KEY"
-func SplitPEM(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	P256PublicKeyBanner                = "NEBULA P256 PUBLIC KEY"
-	// Look for the start of a PEM block
-	start := bytes.Index(data, []byte("-----BEGIN "))
-	if start == -1 {
-		if atEOF && len(bytes.TrimSpace(data)) > 0 {
-			// Non-whitespace content with no PEM block
-			return 0, nil, ErrTruncatedPEMBlock
-		}
-		if atEOF {
-			return len(data), nil, nil
-		}
-		// Request more data
-		return 0, nil, nil
-	}
-
-	// Look for the end marker
-	endMarkerStart := bytes.Index(data[start:], []byte("-----END "))
-	if endMarkerStart == -1 {
-		if atEOF {
-			// Incomplete PEM block at EOF
-			return 0, nil, ErrTruncatedPEMBlock
-		}
-		// Need more data to find the end
-		return 0, nil, nil
-	}
-
-	// Find the actual end of the END line (after the newline)
-	endMarkerStart += start
-	endLineEnd := bytes.IndexByte(data[endMarkerStart:], '\n')
-	var end int
-	if endLineEnd == -1 {
-		if atEOF {
-			// END marker without newline at EOF - take it anyway
-			end = len(data)
-		} else {
-			// Need more data
-			return 0, nil, nil
-		}
-	} else {
-		end = endMarkerStart + endLineEnd + 1
-	}
-
-	// Extract the PEM block
-	pemBlock := data[start:end]
-
-	// Return the valid PEM block
-	return end, pemBlock, nil
-}
-
-const ( //cert banners
-	CertificateBanner   = "NEBULA CERTIFICATE"
-	CertificateV2Banner = "NEBULA CERTIFICATE V2"
-)
-
-const ( //key-agreement-key banners
-	X25519PrivateKeyBanner = "NEBULA X25519 PRIVATE KEY"
-	X25519PublicKeyBanner  = "NEBULA X25519 PUBLIC KEY"
-	P256PrivateKeyBanner   = "NEBULA P256 PRIVATE KEY"
-	P256PublicKeyBanner    = "NEBULA P256 PUBLIC KEY"
-)
-
-/* including "ECDSA" in the P256 banners is a clue that these keys should be used only for signing */
-const ( //signing key banners
 	EncryptedECDSAP256PrivateKeyBanner = "NEBULA ECDSA P256 ENCRYPTED PRIVATE KEY"
 	ECDSAP256PrivateKeyBanner          = "NEBULA ECDSA P256 PRIVATE KEY"
-	ECDSAP256PublicKeyBanner           = "NEBULA ECDSA P256 PUBLIC KEY"
-	EncryptedEd25519PrivateKeyBanner   = "NEBULA ED25519 ENCRYPTED PRIVATE KEY"
-	Ed25519PrivateKeyBanner            = "NEBULA ED25519 PRIVATE KEY"
-	Ed25519PublicKeyBanner             = "NEBULA ED25519 PUBLIC KEY"
 )
 
 // UnmarshalCertificateFromPEM will try to unmarshal the first pem block in a byte array, returning any non consumed
@@ -91,7 +30,19 @@ func UnmarshalCertificateFromPEM(b []byte) (Certificate, []byte, error) {
 		return nil, r, ErrInvalidPEMBlock
 	}
 
-	c, err := unmarshalCertificateBlock(p)
+	var c Certificate
+	var err error
+
+	switch p.Type {
+	// Implementations must validate the resulting certificate contains valid information
+	case CertificateBanner:
+		c, err = unmarshalCertificateV1(p.Bytes, nil)
+	case CertificateV2Banner:
+		c, err = unmarshalCertificateV2(p.Bytes, nil, Curve_CURVE25519)
+	default:
+		return nil, r, ErrInvalidPEMCertificateBanner
+	}
+
 	if err != nil {
 		return nil, r, err
 	}
@@ -100,30 +51,6 @@ func UnmarshalCertificateFromPEM(b []byte) (Certificate, []byte, error) {
 
 }
 
-// unmarshalCertificateBlock decodes a single PEM block into a certificate.
-// It expects a Nebula certificate banner and returns ErrInvalidPEMCertificateBanner otherwise.
-func unmarshalCertificateBlock(block *pem.Block) (Certificate, error) {
-	switch block.Type {
-	// Implementations must validate the resulting certificate contains valid information
-	case CertificateBanner:
-		return unmarshalCertificateV1(block.Bytes, nil)
-	case CertificateV2Banner:
-		return unmarshalCertificateV2(block.Bytes, nil, Curve_CURVE25519)
-	default:
-		return nil, ErrInvalidPEMCertificateBanner
-	}
-}
-
-func marshalCertPublicKeyToPEM(c Certificate) []byte {
-	if c.IsCA() {
-		return MarshalSigningPublicKeyToPEM(c.Curve(), c.PublicKey())
-	} else {
-		return MarshalPublicKeyToPEM(c.Curve(), c.PublicKey())
-	}
-}
-
-// MarshalPublicKeyToPEM returns a PEM representation of a public key used for ECDH.
-// if your public key came from a certificate, prefer Certificate.PublicKeyPEM() if possible, to avoid mistakes!
 func MarshalPublicKeyToPEM(curve Curve, b []byte) []byte {
 	switch curve {
 	case Curve_CURVE25519:
@@ -135,19 +62,6 @@ func MarshalPublicKeyToPEM(curve Curve, b []byte) []byte {
 	}
 }
 
-// MarshalSigningPublicKeyToPEM returns a PEM representation of a public key used for signing.
-// if your public key came from a certificate, prefer Certificate.PublicKeyPEM() if possible, to avoid mistakes!
-func MarshalSigningPublicKeyToPEM(curve Curve, b []byte) []byte {
-	switch curve {
-	case Curve_CURVE25519:
-		return pem.EncodeToMemory(&pem.Block{Type: Ed25519PublicKeyBanner, Bytes: b})
-	case Curve_P256:
-		return pem.EncodeToMemory(&pem.Block{Type: ECDSAP256PublicKeyBanner, Bytes: b})
-	default:
-		return nil
-	}
-}
-
 func UnmarshalPublicKeyFromPEM(b []byte) ([]byte, []byte, Curve, error) {
 	k, r := pem.Decode(b)
 	if k == nil {
@@ -159,7 +73,7 @@ func UnmarshalPublicKeyFromPEM(b []byte) ([]byte, []byte, Curve, error) {
 	case X25519PublicKeyBanner, Ed25519PublicKeyBanner:
 		expectedLen = 32
 		curve = Curve_CURVE25519
-	case P256PublicKeyBanner, ECDSAP256PublicKeyBanner:
+	case P256PublicKeyBanner:
 		// Uncompressed
 		expectedLen = 65
 		curve = Curve_P256

cert/pem_test.go

@@ -1,88 +1,12 @@
 package cert
 
 import (
-	"bufio"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
-func scanAll(t *testing.T, input string) ([]string, error) {
-	t.Helper()
-	scanner := bufio.NewScanner(strings.NewReader(input))
-	scanner.Split(SplitPEM)
-	var blocks []string
-	for scanner.Scan() {
-		blocks = append(blocks, scanner.Text())
-	}
-	return blocks, scanner.Err()
-}
-
-func TestSplitPEM_Single(t *testing.T) {
-	input := "-----BEGIN TEST-----\ndata\n-----END TEST-----\n"
-	blocks, err := scanAll(t, input)
-	require.NoError(t, err)
-	require.Len(t, blocks, 1)
-	require.Equal(t, input, blocks[0])
-}
-
-func TestSplitPEM_Multiple(t *testing.T) {
-	block1 := "-----BEGIN TEST-----\naaa\n-----END TEST-----\n"
-	block2 := "-----BEGIN TEST-----\nbbb\n-----END TEST-----\n"
-	blocks, err := scanAll(t, block1+block2)
-	require.NoError(t, err)
-	require.Len(t, blocks, 2)
-	require.Equal(t, block1, blocks[0])
-	require.Equal(t, block2, blocks[1])
-}
-
-func TestSplitPEM_CommentsAndWhitespaceBetweenBlocks(t *testing.T) {
-	input := "# comment\n\n-----BEGIN TEST-----\naaa\n-----END TEST-----\n\n# another comment\n\n-----BEGIN TEST-----\nbbb\n-----END TEST-----\n"
-	blocks, err := scanAll(t, input)
-	require.NoError(t, err)
-	require.Len(t, blocks, 2)
-}
-
-func TestSplitPEM_Empty(t *testing.T) {
-	blocks, err := scanAll(t, "")
-	require.NoError(t, err)
-	require.Empty(t, blocks)
-}
-
-func TestSplitPEM_WhitespaceOnly(t *testing.T) {
-	blocks, err := scanAll(t, "  \n\t\n  ")
-	require.NoError(t, err)
-	require.Empty(t, blocks)
-}
-
-func TestSplitPEM_TrailingGarbage(t *testing.T) {
-	input := "-----BEGIN TEST-----\ndata\n-----END TEST-----\ngarbage"
-	blocks, err := scanAll(t, input)
-	require.ErrorIs(t, err, ErrTruncatedPEMBlock)
-	require.Len(t, blocks, 1)
-}
-
-func TestSplitPEM_TruncatedBlock(t *testing.T) {
-	input := "-----BEGIN TEST-----\npartial data with no end"
-	_, err := scanAll(t, input)
-	require.ErrorIs(t, err, ErrTruncatedPEMBlock)
-}
-
-func TestSplitPEM_NoEndNewline(t *testing.T) {
-	input := "-----BEGIN TEST-----\ndata\n-----END TEST-----"
-	blocks, err := scanAll(t, input)
-	require.NoError(t, err)
-	require.Len(t, blocks, 1)
-	require.Equal(t, input, blocks[0])
-}
-
-func TestSplitPEM_GarbageOnly(t *testing.T) {
-	_, err := scanAll(t, "this is not PEM data")
-	require.ErrorIs(t, err, ErrTruncatedPEMBlock)
-}
-
 func TestUnmarshalCertificateFromPEM(t *testing.T) {
 	goodCert := []byte(`
 # A good cert
@@ -120,7 +44,7 @@ bzBEr00kERQxxTzTsH8cpYEgRoipvmExvg8WP8NdAJEYJosB
 	assert.Equal(t, rest, invalidPem)
 	require.EqualError(t, err, "bytes did not contain a proper certificate banner")
 
-	// Fail due to invalid PEM format, because
+	// Fail due to ivalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	cert, rest, err = UnmarshalCertificateFromPEM(rest)
 	assert.Nil(t, cert)
@@ -182,7 +106,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
 	assert.Equal(t, rest, invalidPem)
 	require.EqualError(t, err, "bytes did not contain a proper Ed25519/ECDSA private key banner")
 
-	// Fail due to invalid PEM format, because
+	// Fail due to ivalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	k, rest, curve, err = UnmarshalSigningPrivateKeyFromPEM(rest)
 	assert.Nil(t, k)
@@ -244,7 +168,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 	assert.Equal(t, rest, invalidPem)
 	require.EqualError(t, err, "bytes did not contain a proper private key banner")
 
-	// Fail due to invalid PEM format, because
+	// Fail due to ivalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	k, rest, curve, err = UnmarshalPrivateKeyFromPEM(rest)
 	assert.Nil(t, k)
@@ -253,7 +177,6 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 }
 
 func TestUnmarshalPublicKeyFromPEM(t *testing.T) {
-	t.Parallel()
 	pubKey := []byte(`# A good key
 -----BEGIN NEBULA ED25519 PUBLIC KEY-----
 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
@@ -297,7 +220,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 	require.EqualError(t, err, "bytes did not contain a proper public key banner")
 	assert.Equal(t, rest, invalidPem)
 
-	// Fail due to invalid PEM format, because
+	// Fail due to ivalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
 	assert.Nil(t, k)
@@ -307,7 +230,6 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 }
 
 func TestUnmarshalX25519PublicKey(t *testing.T) {
-	t.Parallel()
 	pubKey := []byte(`# A good key
 -----BEGIN NEBULA X25519 PUBLIC KEY-----
 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
@@ -318,12 +240,6 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
 AAAAAAAAAAAAAAAAAAAAAAA=
 -----END NEBULA P256 PUBLIC KEY-----
-`)
-	oldPubP256Key := []byte(`# A good key
------BEGIN NEBULA ECDSA P256 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA ECDSA P256 PUBLIC KEY-----
 `)
 	shortKey := []byte(`# A short key
 -----BEGIN NEBULA X25519 PUBLIC KEY-----
@@ -340,22 +256,15 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 -END NEBULA X25519 PUBLIC KEY-----`)
 
-	keyBundle := appendByteSlices(pubKey, pubP256Key, oldPubP256Key, shortKey, invalidBanner, invalidPem)
+	keyBundle := appendByteSlices(pubKey, pubP256Key, shortKey, invalidBanner, invalidPem)
 
 	// Success test case
 	k, rest, curve, err := UnmarshalPublicKeyFromPEM(keyBundle)
 	assert.Len(t, k, 32)
 	require.NoError(t, err)
-	assert.Equal(t, rest, appendByteSlices(pubP256Key, oldPubP256Key, shortKey, invalidBanner, invalidPem))
+	assert.Equal(t, rest, appendByteSlices(pubP256Key, shortKey, invalidBanner, invalidPem))
 	assert.Equal(t, Curve_CURVE25519, curve)
 
-	// Success test case
-	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
-	assert.Len(t, k, 65)
-	require.NoError(t, err)
-	assert.Equal(t, rest, appendByteSlices(oldPubP256Key, shortKey, invalidBanner, invalidPem))
-	assert.Equal(t, Curve_P256, curve)
-
 	// Success test case
 	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
 	assert.Len(t, k, 65)
@@ -375,7 +284,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 	require.EqualError(t, err, "bytes did not contain a proper public key banner")
 	assert.Equal(t, rest, invalidPem)
 
-	// Fail due to invalid PEM format, because
+	// Fail due to ivalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
 	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
 	assert.Nil(t, k)

cert/sign.go

@@ -7,10 +7,9 @@ import (
 	"crypto/rand"
 	"crypto/sha256"
 	"fmt"
+	"math/big"
 	"net/netip"
 	"time"
-
-	"github.com/slackhq/nebula/cert/p256"
 )
 
 // TBSCertificate represents a certificate intended to be signed.
@@ -56,10 +55,15 @@ func (t *TBSCertificate) Sign(signer Certificate, curve Curve, key []byte) (Cert
 		}
 		return t.SignWith(signer, curve, sp)
 	case Curve_P256:
-		pk, err := ecdsa.ParseRawPrivateKey(elliptic.P256(), key)
+		pk := &ecdsa.PrivateKey{
-		if err != nil {
+			PublicKey: ecdsa.PublicKey{
-			return nil, err
+				Curve: elliptic.P256(),
+			},
+			// ref: https://github.com/golang/go/blob/go1.19/src/crypto/x509/sec1.go#L95
+			D: new(big.Int).SetBytes(key),
 		}
+		// ref: https://github.com/golang/go/blob/go1.19/src/crypto/x509/sec1.go#L119
+		pk.X, pk.Y = pk.Curve.ScalarBaseMult(key)
 		sp := func(certBytes []byte) ([]byte, error) {
 			// We need to hash first for ECDSA
 			// - https://pkg.go.dev/crypto/ecdsa#SignASN1
@@ -128,13 +132,6 @@ func (t *TBSCertificate) SignWith(signer Certificate, curve Curve, sp SignerLamb
 		return nil, err
 	}
 
-	if curve == Curve_P256 {
-		sig, err = p256.Normalize(sig)
-		if err != nil {
-			return nil, err
-		}
-	}
-
 	err = c.setSignature(sig)
 	if err != nil {
 		return nil, err

cert/sign_test.go

@@ -9,7 +9,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/slackhq/nebula/cert/p256"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -90,48 +89,3 @@ func TestCertificateV1_SignP256(t *testing.T) {
 	require.NoError(t, err)
 	assert.NotNil(t, uc)
 }
-
-func TestCertificate_SignP256_AlwaysNormalized(t *testing.T) {
-	before := time.Now().Add(time.Second * -60).Round(time.Second)
-	after := time.Now().Add(time.Second * 60).Round(time.Second)
-	pubKey := []byte("01234567890abcedfghij1234567890ab1234567890abcedfghij1234567890ab")
-
-	tbs := TBSCertificate{
-		Version: Version1,
-		Name:    "testing",
-		Networks: []netip.Prefix{
-			mustParsePrefixUnmapped("10.1.1.1/24"),
-			mustParsePrefixUnmapped("10.1.1.2/16"),
-		},
-		UnsafeNetworks: []netip.Prefix{
-			mustParsePrefixUnmapped("9.1.1.2/24"),
-			mustParsePrefixUnmapped("9.1.1.3/16"),
-		},
-		Groups:    []string{"test-group1", "test-group2", "test-group3"},
-		NotBefore: before,
-		NotAfter:  after,
-		PublicKey: pubKey,
-		IsCA:      true,
-		Curve:     Curve_P256,
-	}
-
-	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
-	require.NoError(t, err)
-	pub := elliptic.Marshal(elliptic.P256(), priv.PublicKey.X, priv.PublicKey.Y)
-	rawPriv := priv.D.FillBytes(make([]byte, 32))
-
-	for i := 0; i < 1000; i++ {
-		if i&1 == 1 {
-			tbs.Version = Version1
-		} else {
-			tbs.Version = Version2
-		}
-		c, err := tbs.Sign(nil, Curve_P256, rawPriv)
-		require.NoError(t, err)
-		assert.NotNil(t, c)
-		assert.True(t, c.CheckSignature(pub))
-		normie, err := p256.IsNormalized(c.Signature())
-		require.NoError(t, err)
-		assert.True(t, normie)
-	}
-}

cert_test/cert.go

@@ -114,33 +114,6 @@ func NewTestCert(v cert.Version, curve cert.Curve, ca cert.Certificate, key []by
 	return c, pub, cert.MarshalPrivateKeyToPEM(curve, priv), pem
 }
 
-func NewTestCertDifferentVersion(c cert.Certificate, v cert.Version, ca cert.Certificate, key []byte) (cert.Certificate, []byte) {
-	nc := &cert.TBSCertificate{
-		Version:        v,
-		Curve:          c.Curve(),
-		Name:           c.Name(),
-		Networks:       c.Networks(),
-		UnsafeNetworks: c.UnsafeNetworks(),
-		Groups:         c.Groups(),
-		NotBefore:      time.Unix(c.NotBefore().Unix(), 0),
-		NotAfter:       time.Unix(c.NotAfter().Unix(), 0),
-		PublicKey:      c.PublicKey(),
-		IsCA:           false,
-	}
-
-	c, err := nc.Sign(ca, ca.Curve(), key)
-	if err != nil {
-		panic(err)
-	}
-
-	pem, err := c.MarshalPEM()
-	if err != nil {
-		panic(err)
-	}
-
-	return c, pem
-}
-
 func X25519Keypair() ([]byte, []byte) {
 	privkey := make([]byte, 32)
 	if _, err := io.ReadFull(rand.Reader, privkey); err != nil {

cmd/nebula-cert/ca.go

@@ -173,26 +173,23 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error
 
 	var passphrase []byte
 	if !isP11 && *cf.encryption {
-		passphrase = []byte(os.Getenv("NEBULA_CA_PASSPHRASE"))
+		for i := 0; i < 5; i++ {
+			out.Write([]byte("Enter passphrase: "))
+			passphrase, err = pr.ReadPassword()
+
+			if err == ErrNoTerminal {
+				return fmt.Errorf("out-key must be encrypted interactively")
+			} else if err != nil {
+				return fmt.Errorf("error reading passphrase: %s", err)
+			}
+
+			if len(passphrase) > 0 {
+				break
+			}
+		}
+
 		if len(passphrase) == 0 {
-			for i := 0; i < 5; i++ {
+			return fmt.Errorf("no passphrase specified, remove -encrypt flag to write out-key in plaintext")
-				out.Write([]byte("Enter passphrase: "))
-				passphrase, err = pr.ReadPassword()
-
-				if err == ErrNoTerminal {
-					return fmt.Errorf("out-key must be encrypted interactively")
-				} else if err != nil {
-					return fmt.Errorf("error reading passphrase: %s", err)
-				}
-
-				if len(passphrase) > 0 {
-					break
-				}
-			}
-
-			if len(passphrase) == 0 {
-				return fmt.Errorf("no passphrase specified, remove -encrypt flag to write out-key in plaintext")
-			}
 		}
 	}
 

cmd/nebula-cert/ca_test.go

@@ -90,26 +90,26 @@ func Test_ca(t *testing.T) {
 	assertHelpError(t, ca(
 		[]string{"-version", "1", "-out-key", "nope", "-out-crt", "nope", "duration", "100m"}, ob, eb, nopw,
 	), "-name is required")
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 
 	// ipv4 only ips
 	assertHelpError(t, ca([]string{"-version", "1", "-name", "ipv6", "-ips", "100::100/100"}, ob, eb, nopw), "invalid -networks definition: v1 certificates can only be ipv4, have 100::100/100")
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 
 	// ipv4 only subnets
 	assertHelpError(t, ca([]string{"-version", "1", "-name", "ipv6", "-subnets", "100::100/100"}, ob, eb, nopw), "invalid -unsafe-networks definition: v1 certificates can only be ipv4, have 100::100/100")
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 
 	// failed key write
 	ob.Reset()
 	eb.Reset()
 	args := []string{"-version", "1", "-name", "test", "-duration", "100m", "-out-crt", "/do/not/write/pleasecrt", "-out-key", "/do/not/write/pleasekey"}
 	require.EqualError(t, ca(args, ob, eb, nopw), "error while writing out-key: open /do/not/write/pleasekey: "+NoSuchDirError)
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 
 	// create temp key file
 	keyF, err := os.CreateTemp("", "test.key")
@@ -121,8 +121,8 @@ func Test_ca(t *testing.T) {
 	eb.Reset()
 	args = []string{"-version", "1", "-name", "test", "-duration", "100m", "-out-crt", "/do/not/write/pleasecrt", "-out-key", keyF.Name()}
 	require.EqualError(t, ca(args, ob, eb, nopw), "error while writing out-crt: open /do/not/write/pleasecrt: "+NoSuchDirError)
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 
 	// create temp cert file
 	crtF, err := os.CreateTemp("", "test.crt")
@@ -135,8 +135,8 @@ func Test_ca(t *testing.T) {
 	eb.Reset()
 	args = []string{"-version", "1", "-name", "test", "-duration", "100m", "-groups", "1,,   2    ,        ,,,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()}
 	require.NoError(t, ca(args, ob, eb, nopw))
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 
 	// read cert and key files
 	rb, _ := os.ReadFile(keyF.Name())
@@ -158,7 +158,7 @@ func Test_ca(t *testing.T) {
 	assert.Empty(t, lCrt.UnsafeNetworks())
 	assert.Len(t, lCrt.PublicKey(), 32)
 	assert.Equal(t, time.Duration(time.Minute*100), lCrt.NotAfter().Sub(lCrt.NotBefore()))
-	assert.Empty(t, lCrt.Issuer())
+	assert.Equal(t, "", lCrt.Issuer())
 	assert.True(t, lCrt.CheckSignature(lCrt.PublicKey()))
 
 	// test encrypted key
@@ -169,18 +169,7 @@ func Test_ca(t *testing.T) {
 	args = []string{"-version", "1", "-encrypt", "-name", "test", "-duration", "100m", "-groups", "1,2,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()}
 	require.NoError(t, ca(args, ob, eb, testpw))
 	assert.Equal(t, pwPromptOb, ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
-
-	// test encrypted key with passphrase environment variable
-	os.Remove(keyF.Name())
-	os.Remove(crtF.Name())
-	ob.Reset()
-	eb.Reset()
-	args = []string{"-version", "1", "-encrypt", "-name", "test", "-duration", "100m", "-groups", "1,2,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()}
-	os.Setenv("NEBULA_CA_PASSPHRASE", string(passphrase))
-	require.NoError(t, ca(args, ob, eb, testpw))
-	assert.Empty(t, eb.String())
-	os.Setenv("NEBULA_CA_PASSPHRASE", "")
 
 	// read encrypted key file and verify default params
 	rb, _ = os.ReadFile(keyF.Name())
@@ -200,7 +189,7 @@ func Test_ca(t *testing.T) {
 	assert.Empty(t, b)
 	assert.Len(t, lKey, 64)
 
-	// test when reading password results in an error
+	// test when reading passsword results in an error
 	os.Remove(keyF.Name())
 	os.Remove(crtF.Name())
 	ob.Reset()
@@ -208,7 +197,7 @@ func Test_ca(t *testing.T) {
 	args = []string{"-version", "1", "-encrypt", "-name", "test", "-duration", "100m", "-groups", "1,2,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()}
 	require.Error(t, ca(args, ob, eb, errpw))
 	assert.Equal(t, pwPromptOb, ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 
 	// test when user fails to enter a password
 	os.Remove(keyF.Name())
@@ -218,7 +207,7 @@ func Test_ca(t *testing.T) {
 	args = []string{"-version", "1", "-encrypt", "-name", "test", "-duration", "100m", "-groups", "1,2,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()}
 	require.EqualError(t, ca(args, ob, eb, nopw), "no passphrase specified, remove -encrypt flag to write out-key in plaintext")
 	assert.Equal(t, strings.Repeat(pwPromptOb, 5), ob.String()) // prompts 5 times before giving up
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 
 	// create valid cert/key for overwrite tests
 	os.Remove(keyF.Name())
@@ -233,8 +222,8 @@ func Test_ca(t *testing.T) {
 	eb.Reset()
 	args = []string{"-version", "1", "-name", "test", "-duration", "100m", "-groups", "1,,   2    ,        ,,,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()}
 	require.EqualError(t, ca(args, ob, eb, nopw), "refusing to overwrite existing CA key: "+keyF.Name())
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 
 	// test that we won't overwrite existing key file
 	os.Remove(keyF.Name())
@@ -242,8 +231,8 @@ func Test_ca(t *testing.T) {
 	eb.Reset()
 	args = []string{"-version", "1", "-name", "test", "-duration", "100m", "-groups", "1,,   2    ,        ,,,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()}
 	require.EqualError(t, ca(args, ob, eb, nopw), "refusing to overwrite existing CA cert: "+crtF.Name())
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 	os.Remove(keyF.Name())
 
 }

cmd/nebula-cert/keygen_test.go

@@ -37,20 +37,20 @@ func Test_keygen(t *testing.T) {
 
 	// required args
 	assertHelpError(t, keygen([]string{"-out-pub", "nope"}, ob, eb), "-out-key is required")
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 
 	assertHelpError(t, keygen([]string{"-out-key", "nope"}, ob, eb), "-out-pub is required")
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 
 	// failed key write
 	ob.Reset()
 	eb.Reset()
 	args := []string{"-out-pub", "/do/not/write/pleasepub", "-out-key", "/do/not/write/pleasekey"}
 	require.EqualError(t, keygen(args, ob, eb), "error while writing out-key: open /do/not/write/pleasekey: "+NoSuchDirError)
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 
 	// create temp key file
 	keyF, err := os.CreateTemp("", "test.key")
@@ -62,8 +62,8 @@ func Test_keygen(t *testing.T) {
 	eb.Reset()
 	args = []string{"-out-pub", "/do/not/write/pleasepub", "-out-key", keyF.Name()}
 	require.EqualError(t, keygen(args, ob, eb), "error while writing out-pub: open /do/not/write/pleasepub: "+NoSuchDirError)
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 
 	// create temp pub file
 	pubF, err := os.CreateTemp("", "test.pub")
@@ -75,8 +75,8 @@ func Test_keygen(t *testing.T) {
 	eb.Reset()
 	args = []string{"-out-pub", pubF.Name(), "-out-key", keyF.Name()}
 	require.NoError(t, keygen(args, ob, eb))
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 
 	// read cert and key files
 	rb, _ := os.ReadFile(keyF.Name())

cmd/nebula-cert/main.go

@@ -5,28 +5,10 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"runtime/debug"
-	"strings"
 )
 
-// A version string that can be set with
-//
-//	-ldflags "-X main.Build=SOMEVERSION"
-//
-// at compile-time.
 var Build string
 
-func init() {
-	if Build == "" {
-		info, ok := debug.ReadBuildInfo()
-		if !ok {
-			return
-		}
-
-		Build = strings.TrimPrefix(info.Main.Version, "v")
-	}
-}
-
 type helpError struct {
 	s string
 }
@@ -35,7 +17,7 @@ func (he *helpError) Error() string {
 	return he.s
 }
 
-func newHelpErrorf(s string, v ...any) error {
+func newHelpErrorf(s string, v ...interface{}) error {
 	return &helpError{s: fmt.Sprintf(s, v...)}
 }
 

cmd/nebula-cert/print_test.go

@@ -43,16 +43,16 @@ func Test_printCert(t *testing.T) {
 
 	// no path
 	err := printCert([]string{}, ob, eb)
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 	assertHelpError(t, err, "-path is required")
 
 	// no cert at path
 	ob.Reset()
 	eb.Reset()
 	err = printCert([]string{"-path", "does_not_exist"}, ob, eb)
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 	require.EqualError(t, err, "unable to read cert; open does_not_exist: "+NoSuchFileError)
 
 	// invalid cert at path
@@ -64,8 +64,8 @@ func Test_printCert(t *testing.T) {
 
 	tf.WriteString("-----BEGIN NOPE-----")
 	err = printCert([]string{"-path", tf.Name()}, ob, eb)
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 	require.EqualError(t, err, "error while unmarshaling cert: input did not contain a valid PEM encoded block")
 
 	// test multiple certs
@@ -155,7 +155,7 @@ func Test_printCert(t *testing.T) {
 `,
 		ob.String(),
 	)
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 
 	// test json
 	ob.Reset()
@@ -177,7 +177,7 @@ func Test_printCert(t *testing.T) {
 `,
 		ob.String(),
 	)
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 }
 
 // NewTestCaCert will generate a CA cert

cmd/nebula-cert/sign.go

@@ -43,7 +43,7 @@ type signFlags struct {
 func newSignFlags() *signFlags {
 	sf := signFlags{set: flag.NewFlagSet("sign", flag.ContinueOnError)}
 	sf.set.Usage = func() {}
-	sf.version = sf.set.Uint("version", 0, "Optional: version of the certificate format to use. The default is to match the version of the signing CA")
+	sf.version = sf.set.Uint("version", 0, "Optional: version of the certificate format to use, the default is to create both v1 and v2 certificates.")
 	sf.caKeyPath = sf.set.String("ca-key", "ca.key", "Optional: path to the signing CA key")
 	sf.caCertPath = sf.set.String("ca-crt", "ca.crt", "Optional: path to the signing CA cert")
 	sf.name = sf.set.String("name", "", "Required: name of the cert, usually a hostname")
@@ -116,28 +116,26 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 		// naively attempt to decode the private key as though it is not encrypted
 		caKey, _, curve, err = cert.UnmarshalSigningPrivateKeyFromPEM(rawCAKey)
 		if errors.Is(err, cert.ErrPrivateKeyEncrypted) {
+			// ask for a passphrase until we get one
 			var passphrase []byte
-			passphrase = []byte(os.Getenv("NEBULA_CA_PASSPHRASE"))
+			for i := 0; i < 5; i++ {
-			if len(passphrase) == 0 {
+				out.Write([]byte("Enter passphrase: "))
-				// ask for a passphrase until we get one
+				passphrase, err = pr.ReadPassword()
-				for i := 0; i < 5; i++ {
-					out.Write([]byte("Enter passphrase: "))
-					passphrase, err = pr.ReadPassword()
 
-					if errors.Is(err, ErrNoTerminal) {
+				if errors.Is(err, ErrNoTerminal) {
-						return fmt.Errorf("ca-key is encrypted and must be decrypted interactively")
+					return fmt.Errorf("ca-key is encrypted and must be decrypted interactively")
-					} else if err != nil {
+				} else if err != nil {
-						return fmt.Errorf("error reading password: %s", err)
+					return fmt.Errorf("error reading password: %s", err)
-					}
-
-					if len(passphrase) > 0 {
-						break
-					}
 				}
-				if len(passphrase) == 0 {
+
-					return fmt.Errorf("cannot open encrypted ca-key without passphrase")
+				if len(passphrase) > 0 {
+					break
 				}
 			}
+			if len(passphrase) == 0 {
+				return fmt.Errorf("cannot open encrypted ca-key without passphrase")
+			}
+
 			curve, caKey, _, err = cert.DecryptAndUnmarshalSigningPrivateKey(passphrase, rawCAKey)
 			if err != nil {
 				return fmt.Errorf("error while parsing encrypted ca-key: %s", err)
@@ -167,10 +165,6 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 		return fmt.Errorf("ca certificate is expired")
 	}
 
-	if version == 0 {
-		version = caCert.Version()
-	}
-
 	// if no duration is given, expire one second before the root expires
 	if *sf.duration <= 0 {
 		*sf.duration = time.Until(caCert.NotAfter()) - time.Second*1
@@ -283,19 +277,21 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 	notBefore := time.Now()
 	notAfter := notBefore.Add(*sf.duration)
 
-	switch version {
+	if version == 0 || version == cert.Version1 {
-	case cert.Version1:
+		// Make sure we at least have an ip
-		// Make sure we have only one ipv4 address
 		if len(v4Networks) != 1 {
 			return newHelpErrorf("invalid -networks definition: v1 certificates can only have a single ipv4 address")
 		}
 
-		if len(v6Networks) > 0 {
+		if version == cert.Version1 {
-			return newHelpErrorf("invalid -networks definition: v1 certificates can only contain ipv4 addresses")
+			// If we are asked to mint a v1 certificate only then we cant just ignore any v6 addresses
-		}
+			if len(v6Networks) > 0 {
+				return newHelpErrorf("invalid -networks definition: v1 certificates can only be ipv4")
+			}
 
-		if len(v6UnsafeNetworks) > 0 {
+			if len(v6UnsafeNetworks) > 0 {
-			return newHelpErrorf("invalid -unsafe-networks definition: v1 certificates can only contain ipv4 addresses")
+				return newHelpErrorf("invalid -unsafe-networks definition: v1 certificates can only be ipv4")
+			}
 		}
 
 		t := &cert.TBSCertificate{
@@ -325,8 +321,9 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 		}
 
 		crts = append(crts, nc)
+	}
 
-	case cert.Version2:
+	if version == 0 || version == cert.Version2 {
 		t := &cert.TBSCertificate{
 			Version:        cert.Version2,
 			Name:           *sf.name,
@@ -354,9 +351,6 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 		}
 
 		crts = append(crts, nc)
-	default:
-		// this should be unreachable
-		return fmt.Errorf("invalid version: %d", version)
 	}
 
 	if !isP11 && *sf.inPubPath == "" {

cmd/nebula-cert/sign_test.go

@@ -55,7 +55,7 @@ func Test_signHelp(t *testing.T) {
 			"  -unsafe-networks string\n"+
 			"    \tOptional: comma separated list of ip address and network in CIDR notation. Unsafe networks this cert can route for\n"+
 			"  -version uint\n"+
-			"    \tOptional: version of the certificate format to use. The default is to match the version of the signing CA\n",
+			"    \tOptional: version of the certificate format to use, the default is to create both v1 and v2 certificates.\n",
 		ob.String(),
 	)
 }
@@ -204,7 +204,7 @@ func Test_signCert(t *testing.T) {
 	ob.Reset()
 	eb.Reset()
 	args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m", "-subnets", "100::100/100"}
-	assertHelpError(t, signCert(args, ob, eb, nopw), "invalid -unsafe-networks definition: v1 certificates can only contain ipv4 addresses")
+	assertHelpError(t, signCert(args, ob, eb, nopw), "invalid -unsafe-networks definition: v1 certificates can only be ipv4")
 	assert.Empty(t, ob.String())
 	assert.Empty(t, eb.String())
 
@@ -379,15 +379,6 @@ func Test_signCert(t *testing.T) {
 	assert.Equal(t, "Enter passphrase: ", ob.String())
 	assert.Empty(t, eb.String())
 
-	// test with the proper password in the environment
-	os.Remove(crtF.Name())
-	os.Remove(keyF.Name())
-	args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, ,   10.2.2.2/32   ,   ,  ,, 10.5.5.5/32", "-groups", "1,,   2    ,        ,,,3,4,5"}
-	os.Setenv("NEBULA_CA_PASSPHRASE", string(passphrase))
-	require.NoError(t, signCert(args, ob, eb, testpw))
-	assert.Empty(t, eb.String())
-	os.Setenv("NEBULA_CA_PASSPHRASE", "")
-
 	// test with the wrong password
 	ob.Reset()
 	eb.Reset()
@@ -398,17 +389,6 @@ func Test_signCert(t *testing.T) {
 	assert.Equal(t, "Enter passphrase: ", ob.String())
 	assert.Empty(t, eb.String())
 
-	// test with the wrong password in environment
-	ob.Reset()
-	eb.Reset()
-
-	os.Setenv("NEBULA_CA_PASSPHRASE", "invalid password")
-	args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, ,   10.2.2.2/32   ,   ,  ,, 10.5.5.5/32", "-groups", "1,,   2    ,        ,,,3,4,5"}
-	require.EqualError(t, signCert(args, ob, eb, nopw), "error while parsing encrypted ca-key: invalid passphrase or corrupt private key")
-	assert.Empty(t, ob.String())
-	assert.Empty(t, eb.String())
-	os.Setenv("NEBULA_CA_PASSPHRASE", "")
-
 	// test with the user not entering a password
 	ob.Reset()
 	eb.Reset()

cmd/nebula-cert/verify.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/slackhq/nebula/cert"
@@ -39,15 +40,21 @@ func verify(args []string, out io.Writer, errOut io.Writer) error {
 		return err
 	}
 
-	caFile, err := os.Open(*vf.caPath)
+	rawCACert, err := os.ReadFile(*vf.caPath)
 	if err != nil {
 		return fmt.Errorf("error while reading ca: %w", err)
 	}
-	defer caFile.Close()
 
-	caPool, err := cert.NewCAPoolFromPEMReader(caFile)
+	caPool := cert.NewCAPool()
-	if err != nil && !errors.Is(err, cert.ErrExpired) {
+	for {
-		return fmt.Errorf("error while adding ca cert to pool: %w", err)
+		rawCACert, err = caPool.AddCAFromPEM(rawCACert)
+		if err != nil {
+			return fmt.Errorf("error while adding ca cert to pool: %w", err)
+		}
+
+		if rawCACert == nil || len(rawCACert) == 0 || strings.TrimSpace(string(rawCACert)) == "" {
+			break
+		}
 	}
 
 	rawCert, err := os.ReadFile(*vf.certPath)

cmd/nebula-cert/verify_test.go

@@ -38,19 +38,19 @@ func Test_verify(t *testing.T) {
 
 	// required args
 	assertHelpError(t, verify([]string{"-ca", "derp"}, ob, eb), "-crt is required")
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 
 	assertHelpError(t, verify([]string{"-crt", "derp"}, ob, eb), "-ca is required")
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 
 	// no ca at path
 	ob.Reset()
 	eb.Reset()
 	err := verify([]string{"-ca", "does_not_exist", "-crt", "does_not_exist"}, ob, eb)
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 	require.EqualError(t, err, "error while reading ca: open does_not_exist: "+NoSuchFileError)
 
 	// invalid ca at path
@@ -62,9 +62,9 @@ func Test_verify(t *testing.T) {
 
 	caFile.WriteString("-----BEGIN NOPE-----")
 	err = verify([]string{"-ca", caFile.Name(), "-crt", "does_not_exist"}, ob, eb)
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
-	require.ErrorIs(t, err, cert.ErrInvalidPEMBlock)
+	require.EqualError(t, err, "error while adding ca cert to pool: input did not contain a valid PEM encoded block")
 
 	// make a ca for later
 	caPub, caPriv, _ := ed25519.GenerateKey(rand.Reader)
@@ -76,8 +76,8 @@ func Test_verify(t *testing.T) {
 
 	// no crt at path
 	err = verify([]string{"-ca", caFile.Name(), "-crt", "does_not_exist"}, ob, eb)
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 	require.EqualError(t, err, "unable to read crt: open does_not_exist: "+NoSuchFileError)
 
 	// invalid crt at path
@@ -89,8 +89,8 @@ func Test_verify(t *testing.T) {
 
 	certFile.WriteString("-----BEGIN NOPE-----")
 	err = verify([]string{"-ca", caFile.Name(), "-crt", certFile.Name()}, ob, eb)
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 	require.EqualError(t, err, "error while parsing crt: input did not contain a valid PEM encoded block")
 
 	// unverifiable cert at path
@@ -106,8 +106,8 @@ func Test_verify(t *testing.T) {
 	certFile.Write(b)
 
 	err = verify([]string{"-ca", caFile.Name(), "-crt", certFile.Name()}, ob, eb)
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 	require.ErrorIs(t, err, cert.ErrSignatureMismatch)
 
 	// verified cert at path
@@ -118,7 +118,7 @@ func Test_verify(t *testing.T) {
 	certFile.Write(b)
 
 	err = verify([]string{"-ca", caFile.Name(), "-crt", certFile.Name()}, ob, eb)
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "", ob.String())
-	assert.Empty(t, eb.String())
+	assert.Equal(t, "", eb.String())
 	require.NoError(t, err)
 }

cmd/nebula-service/main.go

@@ -4,8 +4,6 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"runtime/debug"
-	"strings"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
@@ -20,17 +18,6 @@ import (
 // at compile-time.
 var Build string
 
-func init() {
-	if Build == "" {
-		info, ok := debug.ReadBuildInfo()
-		if !ok {
-			return
-		}
-
-		Build = strings.TrimPrefix(info.Main.Version, "v")
-	}
-}
-
 func main() {
 	serviceFlag := flag.String("service", "", "Control the system service.")
 	configPath := flag.String("config", "", "Path to either a file or directory to load configuration from")
@@ -78,20 +65,8 @@ func main() {
 	}
 
 	if !*configTest {
-		wait, err := ctrl.Start()
+		ctrl.Start()
-		if err != nil {
+		ctrl.ShutdownBlock()
-			util.LogWithContextIfNeeded("Error while running", err, l)
-			os.Exit(1)
-		}
-
-		go ctrl.ShutdownBlock()
-
-		if err := wait(); err != nil {
-			l.WithError(err).Error("Nebula stopped due to fatal error")
-			os.Exit(2)
-		}
-
-		l.Info("Goodbye")
 	}
 
 	os.Exit(0)

cmd/nebula/main.go

@@ -4,8 +4,6 @@ import (
 	"flag"
 	"fmt"
 	"os"
-	"runtime/debug"
-	"strings"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula"
@@ -20,17 +18,6 @@ import (
 // at compile-time.
 var Build string
 
-func init() {
-	if Build == "" {
-		info, ok := debug.ReadBuildInfo()
-		if !ok {
-			return
-		}
-
-		Build = strings.TrimPrefix(info.Main.Version, "v")
-	}
-}
-
 func main() {
 	configPath := flag.String("config", "", "Path to either a file or directory to load configuration from")
 	configTest := flag.Bool("test", false, "Test the config and print the end result. Non zero exit indicates a faulty config")
@@ -72,21 +59,9 @@ func main() {
 	}
 
 	if !*configTest {
-		wait, err := ctrl.Start()
+		ctrl.Start()
-		if err != nil {
-			util.LogWithContextIfNeeded("Error while running", err, l)
-			os.Exit(1)
-		}
-
-		go ctrl.ShutdownBlock()
 		notifyReady(l)
-
+		ctrl.ShutdownBlock()
-		if err := wait(); err != nil {
-			l.WithError(err).Error("Nebula stopped due to fatal error")
-			os.Exit(2)
-		}
-
-		l.Info("Goodbye")
 	}
 
 	os.Exit(0)

config/config.go

@@ -17,14 +17,14 @@ import (
 
 	"dario.cat/mergo"
 	"github.com/sirupsen/logrus"
-	"go.yaml.in/yaml/v3"
+	"gopkg.in/yaml.v2"
 )
 
 type C struct {
 	path        string
 	files       []string
-	Settings    map[string]any
+	Settings    map[interface{}]interface{}
-	oldSettings map[string]any
+	oldSettings map[interface{}]interface{}
 	callbacks   []func(*C)
 	l           *logrus.Logger
 	reloadLock  sync.Mutex
@@ -32,7 +32,7 @@ type C struct {
 
 func NewC(l *logrus.Logger) *C {
 	return &C{
-		Settings: make(map[string]any),
+		Settings: make(map[interface{}]interface{}),
 		l:        l,
 	}
 }
@@ -92,8 +92,8 @@ func (c *C) HasChanged(k string) bool {
 	}
 
 	var (
-		nv any
+		nv interface{}
-		ov any
+		ov interface{}
 	)
 
 	if k == "" {
@@ -147,7 +147,7 @@ func (c *C) ReloadConfig() {
 	c.reloadLock.Lock()
 	defer c.reloadLock.Unlock()
 
-	c.oldSettings = make(map[string]any)
+	c.oldSettings = make(map[interface{}]interface{})
 	for k, v := range c.Settings {
 		c.oldSettings[k] = v
 	}
@@ -167,7 +167,7 @@ func (c *C) ReloadConfigString(raw string) error {
 	c.reloadLock.Lock()
 	defer c.reloadLock.Unlock()
 
-	c.oldSettings = make(map[string]any)
+	c.oldSettings = make(map[interface{}]interface{})
 	for k, v := range c.Settings {
 		c.oldSettings[k] = v
 	}
@@ -201,7 +201,7 @@ func (c *C) GetStringSlice(k string, d []string) []string {
 		return d
 	}
 
-	rv, ok := r.([]any)
+	rv, ok := r.([]interface{})
 	if !ok {
 		return d
 	}
@@ -215,13 +215,13 @@ func (c *C) GetStringSlice(k string, d []string) []string {
 }
 
 // GetMap will get the map for k or return the default d if not found or invalid
-func (c *C) GetMap(k string, d map[string]any) map[string]any {
+func (c *C) GetMap(k string, d map[interface{}]interface{}) map[interface{}]interface{} {
 	r := c.Get(k)
 	if r == nil {
 		return d
 	}
 
-	v, ok := r.(map[string]any)
+	v, ok := r.(map[interface{}]interface{})
 	if !ok {
 		return d
 	}
@@ -243,7 +243,7 @@ func (c *C) GetInt(k string, d int) int {
 // GetUint32 will get the uint32 for k or return the default d if not found or invalid
 func (c *C) GetUint32(k string, d uint32) uint32 {
 	r := c.GetInt(k, int(d))
-	if r < 0 || uint64(r) > uint64(math.MaxUint32) {
+	if uint64(r) > uint64(math.MaxUint32) {
 		return d
 	}
 	return uint32(r)
@@ -266,22 +266,6 @@ func (c *C) GetBool(k string, d bool) bool {
 	return v
 }
 
-func AsBool(v any) (value bool, ok bool) {
-	switch x := v.(type) {
-	case bool:
-		return x, true
-	case string:
-		switch x {
-		case "y", "yes":
-			return true, true
-		case "n", "no":
-			return false, true
-		}
-	}
-
-	return false, false
-}
-
 // GetDuration will get the duration for k or return the default d if not found or invalid
 func (c *C) GetDuration(k string, d time.Duration) time.Duration {
 	r := c.GetString(k, "")
@@ -292,7 +276,7 @@ func (c *C) GetDuration(k string, d time.Duration) time.Duration {
 	return v
 }
 
-func (c *C) Get(k string) any {
+func (c *C) Get(k string) interface{} {
 	return c.get(k, c.Settings)
 }
 
@@ -300,10 +284,10 @@ func (c *C) IsSet(k string) bool {
 	return c.get(k, c.Settings) != nil
 }
 
-func (c *C) get(k string, v any) any {
+func (c *C) get(k string, v interface{}) interface{} {
 	parts := strings.Split(k, ".")
 	for _, p := range parts {
-		m, ok := v.(map[string]any)
+		m, ok := v.(map[interface{}]interface{})
 		if !ok {
 			return nil
 		}
@@ -362,7 +346,7 @@ func (c *C) addFile(path string, direct bool) error {
 }
 
 func (c *C) parseRaw(b []byte) error {
-	var m map[string]any
+	var m map[interface{}]interface{}
 
 	err := yaml.Unmarshal(b, &m)
 	if err != nil {
@@ -374,7 +358,7 @@ func (c *C) parseRaw(b []byte) error {
 }
 
 func (c *C) parse() error {
-	var m map[string]any
+	var m map[interface{}]interface{}
 
 	for _, path := range c.files {
 		b, err := os.ReadFile(path)
@@ -382,7 +366,7 @@ func (c *C) parse() error {
 			return err
 		}
 
-		var nm map[string]any
+		var nm map[interface{}]interface{}
 		err = yaml.Unmarshal(b, &nm)
 		if err != nil {
 			return err

config/config_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.yaml.in/yaml/v3"
+	"gopkg.in/yaml.v2"
 )
 
 func TestConfig_Load(t *testing.T) {
@@ -19,7 +19,7 @@ func TestConfig_Load(t *testing.T) {
 	// invalid yaml
 	c := NewC(l)
 	os.WriteFile(filepath.Join(dir, "01.yaml"), []byte(" invalid yaml"), 0644)
-	require.EqualError(t, c.Load(dir), "yaml: unmarshal errors:\n  line 1: cannot unmarshal !!str `invalid...` into map[string]interface {}")
+	require.EqualError(t, c.Load(dir), "yaml: unmarshal errors:\n  line 1: cannot unmarshal !!str `invalid...` into map[interface {}]interface {}")
 
 	// simple multi config merge
 	c = NewC(l)
@@ -31,8 +31,8 @@ func TestConfig_Load(t *testing.T) {
 	os.WriteFile(filepath.Join(dir, "01.yaml"), []byte("outer:\n  inner: hi"), 0644)
 	os.WriteFile(filepath.Join(dir, "02.yml"), []byte("outer:\n  inner: override\nnew: hi"), 0644)
 	require.NoError(t, c.Load(dir))
-	expected := map[string]any{
+	expected := map[interface{}]interface{}{
-		"outer": map[string]any{
+		"outer": map[interface{}]interface{}{
 			"inner": "override",
 		},
 		"new": "hi",
@@ -44,12 +44,12 @@ func TestConfig_Get(t *testing.T) {
 	l := test.NewLogger()
 	// test simple type
 	c := NewC(l)
-	c.Settings["firewall"] = map[string]any{"outbound": "hi"}
+	c.Settings["firewall"] = map[interface{}]interface{}{"outbound": "hi"}
 	assert.Equal(t, "hi", c.Get("firewall.outbound"))
 
 	// test complex type
-	inner := []map[string]any{{"port": "1", "code": "2"}}
+	inner := []map[interface{}]interface{}{{"port": "1", "code": "2"}}
-	c.Settings["firewall"] = map[string]any{"outbound": inner}
+	c.Settings["firewall"] = map[interface{}]interface{}{"outbound": inner}
 	assert.EqualValues(t, inner, c.Get("firewall.outbound"))
 
 	// test missing
@@ -59,7 +59,7 @@ func TestConfig_Get(t *testing.T) {
 func TestConfig_GetStringSlice(t *testing.T) {
 	l := test.NewLogger()
 	c := NewC(l)
-	c.Settings["slice"] = []any{"one", "two"}
+	c.Settings["slice"] = []interface{}{"one", "two"}
 	assert.Equal(t, []string{"one", "two"}, c.GetStringSlice("slice", []string{}))
 }
 
@@ -101,14 +101,14 @@ func TestConfig_HasChanged(t *testing.T) {
 	// Test key change
 	c = NewC(l)
 	c.Settings["test"] = "hi"
-	c.oldSettings = map[string]any{"test": "no"}
+	c.oldSettings = map[interface{}]interface{}{"test": "no"}
 	assert.True(t, c.HasChanged("test"))
 	assert.True(t, c.HasChanged(""))
 
 	// No key change
 	c = NewC(l)
 	c.Settings["test"] = "hi"
-	c.oldSettings = map[string]any{"test": "hi"}
+	c.oldSettings = map[interface{}]interface{}{"test": "hi"}
 	assert.False(t, c.HasChanged("test"))
 	assert.False(t, c.HasChanged(""))
 }
@@ -184,11 +184,11 @@ firewall:
 `),
 	}
 
-	var m map[string]any
+	var m map[any]any
 
 	// merge the same way config.parse() merges
 	for _, b := range configs {
-		var nm map[string]any
+		var nm map[any]any
 		err := yaml.Unmarshal(b, &nm)
 		require.NoError(t, err)
 
@@ -205,15 +205,15 @@ firewall:
 	t.Logf("Merged Config as YAML:\n%s", mYaml)
 
 	// If a bug is present, some items might be replaced instead of merged like we expect
-	expected := map[string]any{
+	expected := map[any]any{
-		"firewall": map[string]any{
+		"firewall": map[any]any{
 			"inbound": []any{
-				map[string]any{"host": "any", "port": "any", "proto": "icmp"},
+				map[any]any{"host": "any", "port": "any", "proto": "icmp"},
-				map[string]any{"groups": []any{"server"}, "port": 443, "proto": "tcp"},
+				map[any]any{"groups": []any{"server"}, "port": 443, "proto": "tcp"},
-				map[string]any{"groups": []any{"webapp"}, "port": 443, "proto": "tcp"}},
+				map[any]any{"groups": []any{"webapp"}, "port": 443, "proto": "tcp"}},
 			"outbound": []any{
-				map[string]any{"host": "any", "port": "any", "proto": "any"}}},
+				map[any]any{"host": "any", "port": "any", "proto": "any"}}},
-		"listen": map[string]any{
+		"listen": map[any]any{
 			"host": "0.0.0.0",
 			"port": 4242,
 		},

connection_manager.go

@@ -4,16 +4,13 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
-	"fmt"
 	"net/netip"
 	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/rcrowley/go-metrics"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
 )
 
@@ -30,124 +27,130 @@ const (
 )
 
 type connectionManager struct {
+	in     map[uint32]struct{}
+	inLock *sync.RWMutex
+
+	out     map[uint32]struct{}
+	outLock *sync.RWMutex
+
 	// relayUsed holds which relay localIndexs are in use
 	relayUsed     map[uint32]struct{}
 	relayUsedLock *sync.RWMutex
 
-	hostMap      *HostMap
+	hostMap                 *HostMap
-	trafficTimer *LockingTimerWheel[uint32]
+	trafficTimer            *LockingTimerWheel[uint32]
-	intf         *Interface
+	intf                    *Interface
-	punchy       *Punchy
+	pendingDeletion         map[uint32]struct{}
-
+	punchy                  *Punchy
-	// Configuration settings
 	checkInterval           time.Duration
 	pendingDeletionInterval time.Duration
-	inactivityTimeout       atomic.Int64
+	metricsTxPunchy         metrics.Counter
-	dropInactive            atomic.Bool
-
-	metricsTxPunchy metrics.Counter
 
 	l *logrus.Logger
 }
 
-func newConnectionManagerFromConfig(l *logrus.Logger, c *config.C, hm *HostMap, p *Punchy) *connectionManager {
+func newConnectionManager(ctx context.Context, l *logrus.Logger, intf *Interface, checkInterval, pendingDeletionInterval time.Duration, punchy *Punchy) *connectionManager {
-	cm := &connectionManager{
+	var max time.Duration
-		hostMap:         hm,
+	if checkInterval < pendingDeletionInterval {
-		l:               l,
+		max = pendingDeletionInterval
-		punchy:          p,
+	} else {
-		relayUsed:       make(map[uint32]struct{}),
+		max = checkInterval
-		relayUsedLock:   &sync.RWMutex{},
-		metricsTxPunchy: metrics.GetOrRegisterCounter("messages.tx.punchy", nil),
 	}
 
-	cm.reload(c, true)
+	nc := &connectionManager{
-	c.RegisterReloadCallback(func(c *config.C) {
+		hostMap:                 intf.hostMap,
-		cm.reload(c, false)
+		in:                      make(map[uint32]struct{}),
-	})
+		inLock:                  &sync.RWMutex{},
-
+		out:                     make(map[uint32]struct{}),
-	return cm
+		outLock:                 &sync.RWMutex{},
-}
+		relayUsed:               make(map[uint32]struct{}),
-
+		relayUsedLock:           &sync.RWMutex{},
-func (cm *connectionManager) reload(c *config.C, initial bool) {
+		trafficTimer:            NewLockingTimerWheel[uint32](time.Millisecond*500, max),
-	if initial {
+		intf:                    intf,
-		cm.checkInterval = time.Duration(c.GetInt("timers.connection_alive_interval", 5)) * time.Second
+		pendingDeletion:         make(map[uint32]struct{}),
-		cm.pendingDeletionInterval = time.Duration(c.GetInt("timers.pending_deletion_interval", 10)) * time.Second
+		checkInterval:           checkInterval,
-
+		pendingDeletionInterval: pendingDeletionInterval,
-		// We want at least a minimum resolution of 500ms per tick so that we can hit these intervals
+		punchy:                  punchy,
-		// pretty close to their configured duration.
+		metricsTxPunchy:         metrics.GetOrRegisterCounter("messages.tx.punchy", nil),
-		// The inactivity duration is checked each time a hostinfo ticks through so we don't need the wheel to contain it.
+		l:                       l,
-		minDuration := min(time.Millisecond*500, cm.checkInterval, cm.pendingDeletionInterval)
-		maxDuration := max(cm.checkInterval, cm.pendingDeletionInterval)
-		cm.trafficTimer = NewLockingTimerWheel[uint32](minDuration, maxDuration)
 	}
 
-	if initial || c.HasChanged("tunnels.inactivity_timeout") {
+	nc.Start(ctx)
-		old := cm.getInactivityTimeout()
+	return nc
-		cm.inactivityTimeout.Store((int64)(c.GetDuration("tunnels.inactivity_timeout", 10*time.Minute)))
-		if !initial {
-			cm.l.WithField("oldDuration", old).
-				WithField("newDuration", cm.getInactivityTimeout()).
-				Info("Inactivity timeout has changed")
-		}
-	}
-
-	if initial || c.HasChanged("tunnels.drop_inactive") {
-		old := cm.dropInactive.Load()
-		cm.dropInactive.Store(c.GetBool("tunnels.drop_inactive", false))
-		if !initial {
-			cm.l.WithField("oldBool", old).
-				WithField("newBool", cm.dropInactive.Load()).
-				Info("Drop inactive setting has changed")
-		}
-	}
 }
 
-func (cm *connectionManager) getInactivityTimeout() time.Duration {
+func (n *connectionManager) In(localIndex uint32) {
-	return (time.Duration)(cm.inactivityTimeout.Load())
+	n.inLock.RLock()
-}
-
-func (cm *connectionManager) In(h *HostInfo) {
-	h.in.Store(true)
-}
-
-func (cm *connectionManager) Out(h *HostInfo) {
-	h.out.Store(true)
-}
-
-func (cm *connectionManager) RelayUsed(localIndex uint32) {
-	cm.relayUsedLock.RLock()
 	// If this already exists, return
-	if _, ok := cm.relayUsed[localIndex]; ok {
+	if _, ok := n.in[localIndex]; ok {
-		cm.relayUsedLock.RUnlock()
+		n.inLock.RUnlock()
 		return
 	}
-	cm.relayUsedLock.RUnlock()
+	n.inLock.RUnlock()
-	cm.relayUsedLock.Lock()
+	n.inLock.Lock()
-	cm.relayUsed[localIndex] = struct{}{}
+	n.in[localIndex] = struct{}{}
-	cm.relayUsedLock.Unlock()
+	n.inLock.Unlock()
+}
+
+func (n *connectionManager) Out(localIndex uint32) {
+	n.outLock.RLock()
+	// If this already exists, return
+	if _, ok := n.out[localIndex]; ok {
+		n.outLock.RUnlock()
+		return
+	}
+	n.outLock.RUnlock()
+	n.outLock.Lock()
+	n.out[localIndex] = struct{}{}
+	n.outLock.Unlock()
+}
+
+func (n *connectionManager) RelayUsed(localIndex uint32) {
+	n.relayUsedLock.RLock()
+	// If this already exists, return
+	if _, ok := n.relayUsed[localIndex]; ok {
+		n.relayUsedLock.RUnlock()
+		return
+	}
+	n.relayUsedLock.RUnlock()
+	n.relayUsedLock.Lock()
+	n.relayUsed[localIndex] = struct{}{}
+	n.relayUsedLock.Unlock()
 }
 
 // getAndResetTrafficCheck returns if there was any inbound or outbound traffic within the last tick and
 // resets the state for this local index
-func (cm *connectionManager) getAndResetTrafficCheck(h *HostInfo, now time.Time) (bool, bool) {
+func (n *connectionManager) getAndResetTrafficCheck(localIndex uint32) (bool, bool) {
-	in := h.in.Swap(false)
+	n.inLock.Lock()
-	out := h.out.Swap(false)
+	n.outLock.Lock()
-	if in || out {
+	_, in := n.in[localIndex]
-		h.lastUsed = now
+	_, out := n.out[localIndex]
-	}
+	delete(n.in, localIndex)
+	delete(n.out, localIndex)
+	n.inLock.Unlock()
+	n.outLock.Unlock()
 	return in, out
 }
 
-// AddTrafficWatch must be called for every new HostInfo.
+func (n *connectionManager) AddTrafficWatch(localIndex uint32) {
-// We will continue to monitor the HostInfo until the tunnel is dropped.
+	// Use a write lock directly because it should be incredibly rare that we are ever already tracking this index
-func (cm *connectionManager) AddTrafficWatch(h *HostInfo) {
+	n.outLock.Lock()
-	if h.out.Swap(true) == false {
+	if _, ok := n.out[localIndex]; ok {
-		cm.trafficTimer.Add(h.localIndexId, cm.checkInterval)
+		n.outLock.Unlock()
+		return
 	}
+	n.out[localIndex] = struct{}{}
+	n.trafficTimer.Add(localIndex, n.checkInterval)
+	n.outLock.Unlock()
 }
 
-func (cm *connectionManager) Start(ctx context.Context) {
+func (n *connectionManager) Start(ctx context.Context) {
-	clockSource := time.NewTicker(cm.trafficTimer.t.tickDuration)
+	go n.Run(ctx)
+}
+
+func (n *connectionManager) Run(ctx context.Context) {
+	//TODO: this tick should be based on the min wheel tick? Check firewall
+	clockSource := time.NewTicker(500 * time.Millisecond)
 	defer clockSource.Stop()
 
 	p := []byte("")
@@ -160,61 +163,61 @@ func (cm *connectionManager) Start(ctx context.Context) {
 			return
 
 		case now := <-clockSource.C:
-			cm.trafficTimer.Advance(now)
+			n.trafficTimer.Advance(now)
 			for {
-				localIndex, has := cm.trafficTimer.Purge()
+				localIndex, has := n.trafficTimer.Purge()
 				if !has {
 					break
 				}
 
-				cm.doTrafficCheck(localIndex, p, nb, out, now)
+				n.doTrafficCheck(localIndex, p, nb, out, now)
 			}
 		}
 	}
 }
 
-func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) {
+func (n *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) {
-	decision, hostinfo, primary := cm.makeTrafficDecision(localIndex, now)
+	decision, hostinfo, primary := n.makeTrafficDecision(localIndex, now)
 
 	switch decision {
 	case deleteTunnel:
-		if cm.hostMap.DeleteHostInfo(hostinfo) {
+		if n.hostMap.DeleteHostInfo(hostinfo) {
 			// Only clearing the lighthouse cache if this is the last hostinfo for this vpn ip in the hostmap
-			cm.intf.lightHouse.DeleteVpnAddrs(hostinfo.vpnAddrs)
+			n.intf.lightHouse.DeleteVpnAddrs(hostinfo.vpnAddrs)
 		}
 
 	case closeTunnel:
-		cm.intf.sendCloseTunnel(hostinfo)
+		n.intf.sendCloseTunnel(hostinfo)
-		cm.intf.closeTunnel(hostinfo)
+		n.intf.closeTunnel(hostinfo)
 
 	case swapPrimary:
-		cm.swapPrimary(hostinfo, primary)
+		n.swapPrimary(hostinfo, primary)
 
 	case migrateRelays:
-		cm.migrateRelayUsed(hostinfo, primary)
+		n.migrateRelayUsed(hostinfo, primary)
 
 	case tryRehandshake:
-		cm.tryRehandshake(hostinfo)
+		n.tryRehandshake(hostinfo)
 
 	case sendTestPacket:
-		cm.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, p, nb, out)
+		n.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, p, nb, out)
 	}
 
-	cm.resetRelayTrafficCheck(hostinfo)
+	n.resetRelayTrafficCheck(hostinfo)
 }
 
-func (cm *connectionManager) resetRelayTrafficCheck(hostinfo *HostInfo) {
+func (n *connectionManager) resetRelayTrafficCheck(hostinfo *HostInfo) {
 	if hostinfo != nil {
-		cm.relayUsedLock.Lock()
+		n.relayUsedLock.Lock()
-		defer cm.relayUsedLock.Unlock()
+		defer n.relayUsedLock.Unlock()
 		// No need to migrate any relays, delete usage info now.
 		for _, idx := range hostinfo.relayState.CopyRelayForIdxs() {
-			delete(cm.relayUsed, idx)
+			delete(n.relayUsed, idx)
 		}
 	}
 }
 
-func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo) {
+func (n *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo) {
 	relayFor := oldhostinfo.relayState.CopyAllRelayFor()
 
 	for _, r := range relayFor {
@@ -224,51 +227,46 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
 		var relayFrom netip.Addr
 		var relayTo netip.Addr
 		switch {
-		case ok:
+		case ok && existing.State == Established:
-			switch existing.State {
+			// This relay already exists in newhostinfo, then do nothing.
-			case Established, PeerRequested, Disestablished:
+			continue
-				// This relay already exists in newhostinfo, then do nothing.
+		case ok && existing.State == Requested:
-				continue
+			// The relay exists in a Requested state; re-send the request
-			case Requested:
+			index = existing.LocalIndex
-				// The relay exists in a Requested state; re-send the request
+			switch r.Type {
-				index = existing.LocalIndex
+			case TerminalType:
-				switch r.Type {
+				relayFrom = n.intf.myVpnAddrs[0]
-				case TerminalType:
+				relayTo = existing.PeerAddr
-					relayFrom = cm.intf.myVpnAddrs[0]
+			case ForwardingType:
-					relayTo = existing.PeerAddr
+				relayFrom = existing.PeerAddr
-				case ForwardingType:
+				relayTo = newhostinfo.vpnAddrs[0]
-					relayFrom = existing.PeerAddr
+			default:
-					relayTo = newhostinfo.vpnAddrs[0]
+				// should never happen
-				default:
-					// should never happen
-					panic(fmt.Sprintf("Migrating unknown relay type: %v", r.Type))
-				}
 			}
 		case !ok:
-			cm.relayUsedLock.RLock()
+			n.relayUsedLock.RLock()
-			if _, relayUsed := cm.relayUsed[r.LocalIndex]; !relayUsed {
+			if _, relayUsed := n.relayUsed[r.LocalIndex]; !relayUsed {
 				// The relay hasn't been used; don't migrate it.
-				cm.relayUsedLock.RUnlock()
+				n.relayUsedLock.RUnlock()
 				continue
 			}
-			cm.relayUsedLock.RUnlock()
+			n.relayUsedLock.RUnlock()
 			// The relay doesn't exist at all; create some relay state and send the request.
 			var err error
-			index, err = AddRelay(cm.l, newhostinfo, cm.hostMap, r.PeerAddr, nil, r.Type, Requested)
+			index, err = AddRelay(n.l, newhostinfo, n.hostMap, r.PeerAddr, nil, r.Type, Requested)
 			if err != nil {
-				cm.l.WithError(err).Error("failed to migrate relay to new hostinfo")
+				n.l.WithError(err).Error("failed to migrate relay to new hostinfo")
 				continue
 			}
 			switch r.Type {
 			case TerminalType:
-				relayFrom = cm.intf.myVpnAddrs[0]
+				relayFrom = n.intf.myVpnAddrs[0]
 				relayTo = r.PeerAddr
 			case ForwardingType:
 				relayFrom = r.PeerAddr
 				relayTo = newhostinfo.vpnAddrs[0]
 			default:
 				// should never happen
-				panic(fmt.Sprintf("Migrating unknown relay type: %v", r.Type))
 			}
 		}
 
@@ -281,12 +279,12 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
 		switch newhostinfo.GetCert().Certificate.Version() {
 		case cert.Version1:
 			if !relayFrom.Is4() {
-				cm.l.Error("can not migrate v1 relay with a v6 network because the relay is not running a current nebula version")
+				n.l.Error("can not migrate v1 relay with a v6 network because the relay is not running a current nebula version")
 				continue
 			}
 
 			if !relayTo.Is4() {
-				cm.l.Error("can not migrate v1 relay with a v6 remote network because the relay is not running a current nebula version")
+				n.l.Error("can not migrate v1 relay with a v6 remote network because the relay is not running a current nebula version")
 				continue
 			}
 
@@ -298,16 +296,16 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
 			req.RelayFromAddr = netAddrToProtoAddr(relayFrom)
 			req.RelayToAddr = netAddrToProtoAddr(relayTo)
 		default:
-			newhostinfo.logger(cm.l).Error("Unknown certificate version found while attempting to migrate relay")
+			newhostinfo.logger(n.l).Error("Unknown certificate version found while attempting to migrate relay")
 			continue
 		}
 
 		msg, err := req.Marshal()
 		if err != nil {
-			cm.l.WithError(err).Error("failed to marshal Control message to migrate relay")
+			n.l.WithError(err).Error("failed to marshal Control message to migrate relay")
 		} else {
-			cm.intf.SendMessageToHostInfo(header.Control, 0, newhostinfo, msg, make([]byte, 12), make([]byte, mtu))
+			n.intf.SendMessageToHostInfo(header.Control, 0, newhostinfo, msg, make([]byte, 12), make([]byte, mtu))
-			cm.l.WithFields(logrus.Fields{
+			n.l.WithFields(logrus.Fields{
 				"relayFrom":           req.RelayFromAddr,
 				"relayTo":             req.RelayToAddr,
 				"initiatorRelayIndex": req.InitiatorRelayIndex,
@@ -318,44 +316,46 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
 	}
 }
 
-func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo) {
+func (n *connectionManager) makeTrafficDecision(localIndex uint32, now time.Time) (trafficDecision, *HostInfo, *HostInfo) {
-	// Read lock the main hostmap to order decisions based on tunnels being the primary tunnel
+	n.hostMap.RLock()
-	cm.hostMap.RLock()
+	defer n.hostMap.RUnlock()
-	defer cm.hostMap.RUnlock()
 
-	hostinfo := cm.hostMap.Indexes[localIndex]
+	hostinfo := n.hostMap.Indexes[localIndex]
 	if hostinfo == nil {
-		cm.l.WithField("localIndex", localIndex).Debugln("Not found in hostmap")
+		n.l.WithField("localIndex", localIndex).Debugf("Not found in hostmap")
+		delete(n.pendingDeletion, localIndex)
 		return doNothing, nil, nil
 	}
 
-	if cm.isInvalidCertificate(now, hostinfo) {
+	if n.isInvalidCertificate(now, hostinfo) {
+		delete(n.pendingDeletion, hostinfo.localIndexId)
 		return closeTunnel, hostinfo, nil
 	}
 
-	primary := cm.hostMap.Hosts[hostinfo.vpnAddrs[0]]
+	primary := n.hostMap.Hosts[hostinfo.vpnAddrs[0]]
 	mainHostInfo := true
 	if primary != nil && primary != hostinfo {
 		mainHostInfo = false
 	}
 
 	// Check for traffic on this hostinfo
-	inTraffic, outTraffic := cm.getAndResetTrafficCheck(hostinfo, now)
+	inTraffic, outTraffic := n.getAndResetTrafficCheck(localIndex)
 
 	// A hostinfo is determined alive if there is incoming traffic
 	if inTraffic {
 		decision := doNothing
-		if cm.l.Level >= logrus.DebugLevel {
+		if n.l.Level >= logrus.DebugLevel {
-			hostinfo.logger(cm.l).
+			hostinfo.logger(n.l).
 				WithField("tunnelCheck", m{"state": "alive", "method": "passive"}).
 				Debug("Tunnel status")
 		}
-		hostinfo.pendingDeletion.Store(false)
+		delete(n.pendingDeletion, hostinfo.localIndexId)
 
 		if mainHostInfo {
 			decision = tryRehandshake
+
 		} else {
-			if cm.shouldSwapPrimary(hostinfo) {
+			if n.shouldSwapPrimary(hostinfo, primary) {
 				decision = swapPrimary
 			} else {
 				// migrate the relays to the primary, if in use.
@@ -363,55 +363,46 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
 			}
 		}
 
-		cm.trafficTimer.Add(hostinfo.localIndexId, cm.checkInterval)
+		n.trafficTimer.Add(hostinfo.localIndexId, n.checkInterval)
 
 		if !outTraffic {
 			// Send a punch packet to keep the NAT state alive
-			cm.sendPunch(hostinfo)
+			n.sendPunch(hostinfo)
 		}
 
 		return decision, hostinfo, primary
 	}
 
-	if hostinfo.pendingDeletion.Load() {
+	if _, ok := n.pendingDeletion[hostinfo.localIndexId]; ok {
 		// We have already sent a test packet and nothing was returned, this hostinfo is dead
-		hostinfo.logger(cm.l).
+		hostinfo.logger(n.l).
 			WithField("tunnelCheck", m{"state": "dead", "method": "active"}).
 			Info("Tunnel status")
 
+		delete(n.pendingDeletion, hostinfo.localIndexId)
 		return deleteTunnel, hostinfo, nil
 	}
 
 	decision := doNothing
 	if hostinfo != nil && hostinfo.ConnectionState != nil && mainHostInfo {
 		if !outTraffic {
-			inactiveFor, isInactive := cm.isInactive(hostinfo, now)
-			if isInactive {
-				// Tunnel is inactive, tear it down
-				hostinfo.logger(cm.l).
-					WithField("inactiveDuration", inactiveFor).
-					WithField("primary", mainHostInfo).
-					Info("Dropping tunnel due to inactivity")
-
-				return closeTunnel, hostinfo, primary
-			}
-
 			// If we aren't sending or receiving traffic then its an unused tunnel and we don't to test the tunnel.
 			// Just maintain NAT state if configured to do so.
-			cm.sendPunch(hostinfo)
+			n.sendPunch(hostinfo)
-			cm.trafficTimer.Add(hostinfo.localIndexId, cm.checkInterval)
+			n.trafficTimer.Add(hostinfo.localIndexId, n.checkInterval)
 			return doNothing, nil, nil
+
 		}
 
-		if cm.punchy.GetTargetEverything() {
+		if n.punchy.GetTargetEverything() {
 			// This is similar to the old punchy behavior with a slight optimization.
 			// We aren't receiving traffic but we are sending it, punch on all known
 			// ips in case we need to re-prime NAT state
-			cm.sendPunch(hostinfo)
+			n.sendPunch(hostinfo)
 		}
 
-		if cm.l.Level >= logrus.DebugLevel {
+		if n.l.Level >= logrus.DebugLevel {
-			hostinfo.logger(cm.l).
+			hostinfo.logger(n.l).
 				WithField("tunnelCheck", m{"state": "testing", "method": "active"}).
 				Debug("Tunnel status")
 		}
@@ -420,33 +411,17 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
 		decision = sendTestPacket
 
 	} else {
-		if cm.l.Level >= logrus.DebugLevel {
+		if n.l.Level >= logrus.DebugLevel {
-			hostinfo.logger(cm.l).Debugf("Hostinfo sadness")
+			hostinfo.logger(n.l).Debugf("Hostinfo sadness")
 		}
 	}
 
-	hostinfo.pendingDeletion.Store(true)
+	n.pendingDeletion[hostinfo.localIndexId] = struct{}{}
-	cm.trafficTimer.Add(hostinfo.localIndexId, cm.pendingDeletionInterval)
+	n.trafficTimer.Add(hostinfo.localIndexId, n.pendingDeletionInterval)
 	return decision, hostinfo, nil
 }
 
-func (cm *connectionManager) isInactive(hostinfo *HostInfo, now time.Time) (time.Duration, bool) {
+func (n *connectionManager) shouldSwapPrimary(current, primary *HostInfo) bool {
-	if cm.dropInactive.Load() == false {
-		// We aren't configured to drop inactive tunnels
-		return 0, false
-	}
-
-	inactiveDuration := now.Sub(hostinfo.lastUsed)
-	if inactiveDuration < cm.getInactivityTimeout() {
-		// It's not considered inactive
-		return inactiveDuration, false
-	}
-
-	// The tunnel is inactive
-	return inactiveDuration, true
-}
-
-func (cm *connectionManager) shouldSwapPrimary(current *HostInfo) bool {
 	// The primary tunnel is the most recent handshake to complete locally and should work entirely fine.
 	// If we are here then we have multiple tunnels for a host pair and neither side believes the same tunnel is primary.
 	// Let's sort this out.
@@ -454,127 +429,83 @@ func (cm *connectionManager) shouldSwapPrimary(current *HostInfo) bool {
 	// Only one side should swap because if both swap then we may never resolve to a single tunnel.
 	// vpn addr is static across all tunnels for this host pair so lets
 	// use that to determine if we should consider swapping.
-	if current.vpnAddrs[0].Compare(cm.intf.myVpnAddrs[0]) < 0 {
+	if current.vpnAddrs[0].Compare(n.intf.myVpnAddrs[0]) < 0 {
 		// Their primary vpn addr is less than mine. Do not swap.
 		return false
 	}
 
-	crt := cm.intf.pki.getCertState().getCertificate(current.ConnectionState.myCert.Version())
+	crt := n.intf.pki.getCertState().getCertificate(current.ConnectionState.myCert.Version())
-	if crt == nil {
-		//my cert was reloaded away. We should definitely swap from this tunnel
-		return true
-	}
 	// If this tunnel is using the latest certificate then we should swap it to primary for a bit and see if things
 	// settle down.
 	return bytes.Equal(current.ConnectionState.myCert.Signature(), crt.Signature())
 }
 
-func (cm *connectionManager) swapPrimary(current, primary *HostInfo) {
+func (n *connectionManager) swapPrimary(current, primary *HostInfo) {
-	cm.hostMap.Lock()
+	n.hostMap.Lock()
 	// Make sure the primary is still the same after the write lock. This avoids a race with a rehandshake.
-	if cm.hostMap.Hosts[current.vpnAddrs[0]] == primary {
+	if n.hostMap.Hosts[current.vpnAddrs[0]] == primary {
-		cm.hostMap.unlockedMakePrimary(current)
+		n.hostMap.unlockedMakePrimary(current)
 	}
-	cm.hostMap.Unlock()
+	n.hostMap.Unlock()
 }
 
-// isInvalidCertificate decides if we should destroy a tunnel.
+// isInvalidCertificate will check if we should destroy a tunnel if pki.disconnect_invalid is true and
-// returns true if pki.disconnect_invalid is true and the certificate is no longer valid.
+// the certificate is no longer valid. Block listed certificates will skip the pki.disconnect_invalid
-// Blocklisted certificates will skip the pki.disconnect_invalid check and return true.
+// check and return true.
-func (cm *connectionManager) isInvalidCertificate(now time.Time, hostinfo *HostInfo) bool {
+func (n *connectionManager) isInvalidCertificate(now time.Time, hostinfo *HostInfo) bool {
 	remoteCert := hostinfo.GetCert()
 	if remoteCert == nil {
-		return false //don't tear down tunnels for handshakes in progress
-	}
-
-	caPool := cm.intf.pki.GetCAPool()
-	err := caPool.VerifyCachedCertificate(now, remoteCert)
-	if err == nil {
-		return false //cert is still valid! yay!
-	} else if err == cert.ErrBlockListed { //avoiding errors.Is for speed
-		// Block listed certificates should always be disconnected
-		hostinfo.logger(cm.l).WithError(err).
-			WithField("fingerprint", remoteCert.Fingerprint).
-			Info("Remote certificate is blocked, tearing down the tunnel")
-		return true
-	} else if cm.intf.disconnectInvalid.Load() {
-		hostinfo.logger(cm.l).WithError(err).
-			WithField("fingerprint", remoteCert.Fingerprint).
-			Info("Remote certificate is no longer valid, tearing down the tunnel")
-		return true
-	} else {
-		//if we reach here, the cert is no longer valid, but we're configured to keep tunnels from now-invalid certs open
 		return false
 	}
+
+	caPool := n.intf.pki.GetCAPool()
+	err := caPool.VerifyCachedCertificate(now, remoteCert)
+	if err == nil {
+		return false
+	}
+
+	if !n.intf.disconnectInvalid.Load() && err != cert.ErrBlockListed {
+		// Block listed certificates should always be disconnected
+		return false
+	}
+
+	hostinfo.logger(n.l).WithError(err).
+		WithField("fingerprint", remoteCert.Fingerprint).
+		Info("Remote certificate is no longer valid, tearing down the tunnel")
+
+	return true
 }
 
-func (cm *connectionManager) sendPunch(hostinfo *HostInfo) {
+func (n *connectionManager) sendPunch(hostinfo *HostInfo) {
-	if !cm.punchy.GetPunch() {
+	if !n.punchy.GetPunch() {
 		// Punching is disabled
 		return
 	}
 
-	if cm.intf.lightHouse.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
+	if n.punchy.GetTargetEverything() {
-		// Do not punch to lighthouses, we assume our lighthouse update interval is good enough.
+		hostinfo.remotes.ForEach(n.hostMap.GetPreferredRanges(), func(addr netip.AddrPort, preferred bool) {
-		// In the event the update interval is not sufficient to maintain NAT state then a publicly available lighthouse
+			n.metricsTxPunchy.Inc(1)
-		// would lose the ability to notify us and punchy.respond would become unreliable.
+			n.intf.outside.WriteTo([]byte{1}, addr)
-		return
-	}
-
-	if cm.punchy.GetTargetEverything() {
-		hostinfo.remotes.ForEach(cm.hostMap.GetPreferredRanges(), func(addr netip.AddrPort, preferred bool) {
-			cm.metricsTxPunchy.Inc(1)
-			cm.intf.outside.WriteTo([]byte{1}, addr)
 		})
 
 	} else if hostinfo.remote.IsValid() {
-		cm.metricsTxPunchy.Inc(1)
+		n.metricsTxPunchy.Inc(1)
-		cm.intf.outside.WriteTo([]byte{1}, hostinfo.remote)
+		n.intf.outside.WriteTo([]byte{1}, hostinfo.remote)
 	}
 }
 
-func (cm *connectionManager) tryRehandshake(hostinfo *HostInfo) {
+func (n *connectionManager) tryRehandshake(hostinfo *HostInfo) {
-	cs := cm.intf.pki.getCertState()
+	cs := n.intf.pki.getCertState()
 	curCrt := hostinfo.ConnectionState.myCert
-	curCrtVersion := curCrt.Version()
+	myCrt := cs.getCertificate(curCrt.Version())
-	myCrt := cs.getCertificate(curCrtVersion)
+	if curCrt.Version() >= cs.defaultVersion && bytes.Equal(curCrt.Signature(), myCrt.Signature()) == true {
-	if myCrt == nil {
+		// The current tunnel is using the latest certificate and version, no need to rehandshake.
-		cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
-			WithField("version", curCrtVersion).
-			WithField("reason", "local certificate removed").
-			Info("Re-handshaking with remote")
-		cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
 		return
 	}
-	peerCrt := hostinfo.ConnectionState.peerCert
-	if peerCrt != nil && curCrtVersion < peerCrt.Certificate.Version() {
-		// if our certificate version is less than theirs, and we have a matching version available, rehandshake?
-		if cs.getCertificate(peerCrt.Certificate.Version()) != nil {
-			cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
-				WithField("version", curCrtVersion).
-				WithField("peerVersion", peerCrt.Certificate.Version()).
-				WithField("reason", "local certificate version lower than peer, attempting to correct").
-				Info("Re-handshaking with remote")
-			cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], func(hh *HandshakeHostInfo) {
-				hh.initiatingVersionOverride = peerCrt.Certificate.Version()
-			})
-			return
-		}
-	}
-	if !bytes.Equal(curCrt.Signature(), myCrt.Signature()) {
-		cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
-			WithField("reason", "local certificate is not current").
-			Info("Re-handshaking with remote")
 
-		cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
+	n.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
-		return
+		WithField("reason", "local certificate is not current").
-	}
+		Info("Re-handshaking with remote")
-	if curCrtVersion < cs.initiatingVersion {
-		cm.l.WithField("vpnAddrs", hostinfo.vpnAddrs).
-			WithField("reason", "current cert version < pki.initiatingVersion").
-			Info("Re-handshaking with remote")
 
-		cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
+	n.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
-		return
-	}
 }

connection_manager_test.go

@@ -1,6 +1,7 @@
 package nebula
 
 import (
+	"context"
 	"crypto/ed25519"
 	"crypto/rand"
 	"net/netip"
@@ -10,7 +11,6 @@ import (
 	"github.com/flynn/noise"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay"
 	"github.com/slackhq/nebula/test"
 	"github.com/slackhq/nebula/udp"
 	"github.com/stretchr/testify/assert"
@@ -23,7 +23,7 @@ func newTestLighthouse() *LightHouse {
 		addrMap:   map[netip.Addr]*RemoteList{},
 		queryChan: make(chan netip.Addr, 10),
 	}
-	lighthouses := []netip.Addr{}
+	lighthouses := map[netip.Addr]struct{}{}
 	staticList := map[netip.Addr]struct{}{}
 
 	lh.lighthouses.Store(&lighthouses)
@@ -44,16 +44,16 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	hostMap.preferredRanges.Store(&preferredRanges)
 
 	cs := &CertState{
-		initiatingVersion: cert.Version1,
+		defaultVersion:   cert.Version1,
-		privateKey:        []byte{},
+		privateKey:       []byte{},
-		v1Cert:            &dummyCert{version: cert.Version1},
+		v1Cert:           &dummyCert{version: cert.Version1},
-		v1HandshakeBytes:  []byte{},
+		v1HandshakeBytes: []byte{},
 	}
 
 	lh := newTestLighthouse()
 	ifce := &Interface{
 		hostMap:          hostMap,
-		inside:           &overlay.NoopTun{},
+		inside:           &test.NoopTun{},
 		outside:          &udp.NoopConn{},
 		firewall:         &Firewall{},
 		lightHouse:       lh,
@@ -64,10 +64,10 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	ifce.pki.cs.Store(cs)
 
 	// Create manager
-	conf := config.NewC(l)
+	ctx, cancel := context.WithCancel(context.Background())
-	punchy := NewPunchyFromConfig(l, conf)
+	defer cancel()
-	nc := newConnectionManagerFromConfig(l, conf, hostMap, punchy)
+	punchy := NewPunchyFromConfig(l, config.NewC(l))
-	nc.intf = ifce
+	nc := newConnectionManager(ctx, l, ifce, 5, 10, punchy)
 	p := []byte("")
 	nb := make([]byte, 12, 12)
 	out := make([]byte, mtu)
@@ -85,33 +85,32 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
 
 	// We saw traffic out to vpnIp
-	nc.Out(hostinfo)
+	nc.Out(hostinfo.localIndexId)
-	nc.In(hostinfo)
+	nc.In(hostinfo.localIndexId)
-	assert.False(t, hostinfo.pendingDeletion.Load())
+	assert.NotContains(t, nc.pendingDeletion, hostinfo.localIndexId)
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
-	assert.True(t, hostinfo.out.Load())
+	assert.Contains(t, nc.out, hostinfo.localIndexId)
-	assert.True(t, hostinfo.in.Load())
 
 	// Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded
 	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
-	assert.False(t, hostinfo.pendingDeletion.Load())
+	assert.NotContains(t, nc.pendingDeletion, hostinfo.localIndexId)
-	assert.False(t, hostinfo.out.Load())
+	assert.NotContains(t, nc.out, hostinfo.localIndexId)
-	assert.False(t, hostinfo.in.Load())
+	assert.NotContains(t, nc.in, hostinfo.localIndexId)
 
 	// Do another traffic check tick, this host should be pending deletion now
-	nc.Out(hostinfo)
+	nc.Out(hostinfo.localIndexId)
-	assert.True(t, hostinfo.out.Load())
 	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
-	assert.True(t, hostinfo.pendingDeletion.Load())
+	assert.Contains(t, nc.pendingDeletion, hostinfo.localIndexId)
-	assert.False(t, hostinfo.out.Load())
+	assert.NotContains(t, nc.out, hostinfo.localIndexId)
-	assert.False(t, hostinfo.in.Load())
+	assert.NotContains(t, nc.in, hostinfo.localIndexId)
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
 
 	// Do a final traffic check tick, the host should now be removed
 	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
-	assert.NotContains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs)
+	assert.NotContains(t, nc.pendingDeletion, hostinfo.localIndexId)
+	assert.NotContains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
 	assert.NotContains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 }
 
@@ -127,16 +126,16 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 	hostMap.preferredRanges.Store(&preferredRanges)
 
 	cs := &CertState{
-		initiatingVersion: cert.Version1,
+		defaultVersion:   cert.Version1,
-		privateKey:        []byte{},
+		privateKey:       []byte{},
-		v1Cert:            &dummyCert{version: cert.Version1},
+		v1Cert:           &dummyCert{version: cert.Version1},
-		v1HandshakeBytes:  []byte{},
+		v1HandshakeBytes: []byte{},
 	}
 
 	lh := newTestLighthouse()
 	ifce := &Interface{
 		hostMap:          hostMap,
-		inside:           &overlay.NoopTun{},
+		inside:           &test.NoopTun{},
 		outside:          &udp.NoopConn{},
 		firewall:         &Firewall{},
 		lightHouse:       lh,
@@ -147,10 +146,10 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 	ifce.pki.cs.Store(cs)
 
 	// Create manager
-	conf := config.NewC(l)
+	ctx, cancel := context.WithCancel(context.Background())
-	punchy := NewPunchyFromConfig(l, conf)
+	defer cancel()
-	nc := newConnectionManagerFromConfig(l, conf, hostMap, punchy)
+	punchy := NewPunchyFromConfig(l, config.NewC(l))
-	nc.intf = ifce
+	nc := newConnectionManager(ctx, l, ifce, 5, 10, punchy)
 	p := []byte("")
 	nb := make([]byte, 12, 12)
 	out := make([]byte, mtu)
@@ -168,129 +167,33 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 	nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
 
 	// We saw traffic out to vpnIp
-	nc.Out(hostinfo)
+	nc.Out(hostinfo.localIndexId)
-	nc.In(hostinfo)
+	nc.In(hostinfo.localIndexId)
-	assert.True(t, hostinfo.in.Load())
+	assert.NotContains(t, nc.pendingDeletion, hostinfo.vpnAddrs[0])
-	assert.True(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.pendingDeletion.Load())
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 
 	// Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded
 	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
-	assert.False(t, hostinfo.pendingDeletion.Load())
+	assert.NotContains(t, nc.pendingDeletion, hostinfo.localIndexId)
-	assert.False(t, hostinfo.out.Load())
+	assert.NotContains(t, nc.out, hostinfo.localIndexId)
-	assert.False(t, hostinfo.in.Load())
+	assert.NotContains(t, nc.in, hostinfo.localIndexId)
 
 	// Do another traffic check tick, this host should be pending deletion now
-	nc.Out(hostinfo)
+	nc.Out(hostinfo.localIndexId)
 	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
-	assert.True(t, hostinfo.pendingDeletion.Load())
+	assert.Contains(t, nc.pendingDeletion, hostinfo.localIndexId)
-	assert.False(t, hostinfo.out.Load())
+	assert.NotContains(t, nc.out, hostinfo.localIndexId)
-	assert.False(t, hostinfo.in.Load())
+	assert.NotContains(t, nc.in, hostinfo.localIndexId)
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
 
 	// We saw traffic, should no longer be pending deletion
-	nc.In(hostinfo)
+	nc.In(hostinfo.localIndexId)
 	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
-	assert.False(t, hostinfo.pendingDeletion.Load())
+	assert.NotContains(t, nc.pendingDeletion, hostinfo.localIndexId)
-	assert.False(t, hostinfo.out.Load())
+	assert.NotContains(t, nc.out, hostinfo.localIndexId)
-	assert.False(t, hostinfo.in.Load())
+	assert.NotContains(t, nc.in, hostinfo.localIndexId)
-	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
-	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
-}
-
-func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
-	l := test.NewLogger()
-	localrange := netip.MustParsePrefix("10.1.1.1/24")
-	vpnAddrs := []netip.Addr{netip.MustParseAddr("172.1.1.2")}
-	preferredRanges := []netip.Prefix{localrange}
-
-	// Very incomplete mock objects
-	hostMap := newHostMap(l)
-	hostMap.preferredRanges.Store(&preferredRanges)
-
-	cs := &CertState{
-		initiatingVersion: cert.Version1,
-		privateKey:        []byte{},
-		v1Cert:            &dummyCert{version: cert.Version1},
-		v1HandshakeBytes:  []byte{},
-	}
-
-	lh := newTestLighthouse()
-	ifce := &Interface{
-		hostMap:          hostMap,
-		inside:           &overlay.NoopTun{},
-		outside:          &udp.NoopConn{},
-		firewall:         &Firewall{},
-		lightHouse:       lh,
-		pki:              &PKI{},
-		handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
-		l:                l,
-	}
-	ifce.pki.cs.Store(cs)
-
-	// Create manager
-	conf := config.NewC(l)
-	conf.Settings["tunnels"] = map[string]any{
-		"drop_inactive": true,
-	}
-	punchy := NewPunchyFromConfig(l, conf)
-	nc := newConnectionManagerFromConfig(l, conf, hostMap, punchy)
-	assert.True(t, nc.dropInactive.Load())
-	nc.intf = ifce
-
-	// Add an ip we have established a connection w/ to hostmap
-	hostinfo := &HostInfo{
-		vpnAddrs:      vpnAddrs,
-		localIndexId:  1099,
-		remoteIndexId: 9901,
-	}
-	hostinfo.ConnectionState = &ConnectionState{
-		myCert: &dummyCert{version: cert.Version1},
-		H:      &noise.HandshakeState{},
-	}
-	nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
-
-	// Do a traffic check tick, in and out should be cleared but should not be pending deletion
-	nc.Out(hostinfo)
-	nc.In(hostinfo)
-	assert.True(t, hostinfo.out.Load())
-	assert.True(t, hostinfo.in.Load())
-
-	now := time.Now()
-	decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, now)
-	assert.Equal(t, tryRehandshake, decision)
-	assert.Equal(t, now, hostinfo.lastUsed)
-	assert.False(t, hostinfo.pendingDeletion.Load())
-	assert.False(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.in.Load())
-
-	decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*5))
-	assert.Equal(t, doNothing, decision)
-	assert.Equal(t, now, hostinfo.lastUsed)
-	assert.False(t, hostinfo.pendingDeletion.Load())
-	assert.False(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.in.Load())
-
-	// Do another traffic check tick, should still not be pending deletion
-	decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Second*10))
-	assert.Equal(t, doNothing, decision)
-	assert.Equal(t, now, hostinfo.lastUsed)
-	assert.False(t, hostinfo.pendingDeletion.Load())
-	assert.False(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.in.Load())
-	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
-	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
-
-	// Finally advance beyond the inactivity timeout
-	decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, now.Add(time.Minute*10))
-	assert.Equal(t, closeTunnel, decision)
-	assert.Equal(t, now, hostinfo.lastUsed)
-	assert.False(t, hostinfo.pendingDeletion.Load())
-	assert.False(t, hostinfo.out.Load())
-	assert.False(t, hostinfo.in.Load())
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
 }
@@ -348,7 +251,7 @@ func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) {
 	lh := newTestLighthouse()
 	ifce := &Interface{
 		hostMap:          hostMap,
-		inside:           &overlay.NoopTun{},
+		inside:           &test.NoopTun{},
 		outside:          &udp.NoopConn{},
 		firewall:         &Firewall{},
 		lightHouse:       lh,
@@ -361,10 +264,10 @@ func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) {
 	ifce.disconnectInvalid.Store(true)
 
 	// Create manager
-	conf := config.NewC(l)
+	ctx, cancel := context.WithCancel(context.Background())
-	punchy := NewPunchyFromConfig(l, conf)
+	defer cancel()
-	nc := newConnectionManagerFromConfig(l, conf, hostMap, punchy)
+	punchy := NewPunchyFromConfig(l, config.NewC(l))
-	nc.intf = ifce
+	nc := newConnectionManager(ctx, l, ifce, 5, 10, punchy)
 	ifce.connectionManager = nc
 
 	hostinfo := &HostInfo{
@@ -447,10 +350,6 @@ func (d *dummyCert) PublicKey() []byte {
 	return d.publicKey
 }
 
-func (d *dummyCert) MarshalPublicKeyPEM() []byte {
-	return cert.MarshalPublicKeyToPEM(d.curve, d.publicKey)
-}
-
 func (d *dummyCert) Signature() []byte {
 	return d.signature
 }

connection_state.go

@@ -27,7 +27,7 @@ type ConnectionState struct {
 	writeLock      sync.Mutex
 }
 
-func NewConnectionState(l *logrus.Logger, cs *CertState, crt cert.Certificate, initiator bool, pattern noise.HandshakePattern) (*ConnectionState, error) {
+func NewConnectionState(l *logrus.Logger, cs *CertState, crt cert.Certificate, initiator bool, pattern noise.HandshakePattern, psk []byte) (*ConnectionState, error) {
 	var dhFunc noise.DHFunc
 	switch crt.Curve() {
 	case cert.Curve_CURVE25519:
@@ -50,14 +50,18 @@ func NewConnectionState(l *logrus.Logger, cs *CertState, crt cert.Certificate, i
 	}
 
 	static := noise.DHKey{Private: cs.privateKey, Public: crt.PublicKey()}
+
+	b := NewBits(ReplayWindow)
+	// Clear out bit 0, we never transmit it, and we don't want it showing as packet loss
+	b.Update(l, 0)
+
 	hs, err := noise.NewHandshakeState(noise.Config{
-		CipherSuite:   ncs,
+		CipherSuite:           ncs,
-		Random:        rand.Reader,
+		Random:                rand.Reader,
-		Pattern:       pattern,
+		Pattern:               pattern,
-		Initiator:     initiator,
+		Initiator:             initiator,
-		StaticKeypair: static,
+		StaticKeypair:         static,
-		//NOTE: These should come from CertState (pki.go) when we finally implement it
+		PresharedKey:          psk,
-		PresharedKey:          []byte{},
 		PresharedKeyPlacement: 0,
 	})
 	if err != nil {
@@ -69,7 +73,7 @@ func NewConnectionState(l *logrus.Logger, cs *CertState, crt cert.Certificate, i
 	ci := &ConnectionState{
 		H:         hs,
 		initiator: initiator,
-		window:    NewBits(ReplayWindow),
+		window:    b,
 		myCert:    crt,
 	}
 	// always start the counter from 2, as packet 1 and packet 2 are handshake packets.

control.go

@@ -2,11 +2,9 @@ package nebula
 
 import (
 	"context"
-	"errors"
 	"net/netip"
 	"os"
 	"os/signal"
-	"sync"
 	"syscall"
 
 	"github.com/sirupsen/logrus"
@@ -15,20 +13,6 @@ import (
 	"github.com/slackhq/nebula/overlay"
 )
 
-type RunState int
-
-const (
-	StateUnknown RunState = iota
-	StateReady
-	StateStarted
-	StateStopping
-	StateStopped
-)
-
-var ErrAlreadyStarted = errors.New("nebula is already started")
-var ErrAlreadyStopped = errors.New("nebula cannot be restarted")
-var ErrUnknownState = errors.New("nebula state is invalid")
-
 // Every interaction here needs to take extra care to copy memory and not return or use arguments "as is" when touching
 // core. This means copying IP objects, slices, de-referencing pointers and taking the actual value, etc
 
@@ -42,18 +26,14 @@ type controlHostLister interface {
 }
 
 type Control struct {
-	stateLock sync.Mutex
+	f               *Interface
-	state     RunState
+	l               *logrus.Logger
-
+	ctx             context.Context
-	f                      *Interface
+	cancel          context.CancelFunc
-	l                      *logrus.Logger
+	sshStart        func()
-	ctx                    context.Context
+	statsStart      func()
-	cancel                 context.CancelFunc
+	dnsStart        func()
-	sshStart               func()
+	lighthouseStart func()
-	statsStart             func()
-	dnsStart               func()
-	lighthouseStart        func()
-	connectionManagerStart func(context.Context)
 }
 
 type ControlHostInfo struct {
@@ -68,31 +48,10 @@ type ControlHostInfo struct {
 	CurrentRelaysThroughMe []netip.Addr     `json:"currentRelaysThroughMe"`
 }
 
-// Start actually runs nebula, this is a nonblocking call.
+// Start actually runs nebula, this is a nonblocking call. To block use Control.ShutdownBlock()
-// The returned function blocks until nebula has fully stopped and returns the
+func (c *Control) Start() {
-// first fatal reader error (if any). A nil error means nebula shut down
-// gracefully; a non-nil error means a reader hit an unexpected failure that
-// triggered the shutdown.
-func (c *Control) Start() (func() error, error) {
-	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
-	switch c.state {
-	case StateReady:
-		//yay!
-	case StateStopped, StateStopping:
-		return nil, ErrAlreadyStopped
-	case StateStarted:
-		return nil, ErrAlreadyStarted
-	default:
-		return nil, ErrUnknownState
-	}
-
 	// Activate the interface
-	err := c.f.activate()
+	c.f.activate()
-	if err != nil {
-		c.state = StateStopped
-		return nil, err
-	}
 
 	// Call all the delayed funcs that waited patiently for the interface to be created.
 	if c.sshStart != nil {
@@ -104,47 +63,20 @@ func (c *Control) Start() (func() error, error) {
 	if c.dnsStart != nil {
 		go c.dnsStart()
 	}
-	if c.connectionManagerStart != nil {
-		go c.connectionManagerStart(c.ctx)
-	}
 	if c.lighthouseStart != nil {
 		c.lighthouseStart()
 	}
 
-	c.f.triggerShutdown = c.Stop
-
 	// Start reading packets.
-	out, err := c.f.run()
+	c.f.run()
-	if err != nil {
-		c.state = StateStopped
-		return nil, err
-	}
-	c.state = StateStarted
-	return out, nil
-}
-
-func (c *Control) State() RunState {
-	c.stateLock.Lock()
-	defer c.stateLock.Unlock()
-	return c.state
 }
 
 func (c *Control) Context() context.Context {
 	return c.ctx
 }
 
-// Stop is a non-blocking call that signals nebula to close all tunnels and shut down
+// Stop signals nebula to shutdown and close all tunnels, returns after the shutdown is complete
 func (c *Control) Stop() {
-	c.stateLock.Lock()
-	if c.state != StateStarted {
-		c.stateLock.Unlock()
-		// We are stopping or stopped already
-		return
-	}
-
-	c.state = StateStopping
-	c.stateLock.Unlock()
-
 	// Stop the handshakeManager (and other services), to prevent new tunnels from
 	// being created while we're shutting them all down.
 	c.cancel()
@@ -153,9 +85,7 @@ func (c *Control) Stop() {
 	if err := c.f.Close(); err != nil {
 		c.l.WithError(err).Error("Close interface failed")
 	}
-	c.stateLock.Lock()
+	c.l.Info("Goodbye")
-	c.state = StateStopped
-	c.stateLock.Unlock()
 }
 
 // ShutdownBlock will listen for and block on term and interrupt signals, calling Control.Stop() once signalled
@@ -201,7 +131,8 @@ func (c *Control) ListHostmapIndexes(pendingMap bool) []ControlHostInfo {
 
 // GetCertByVpnIp returns the authenticated certificate of the given vpn IP, or nil if not found
 func (c *Control) GetCertByVpnIp(vpnIp netip.Addr) cert.Certificate {
-	if c.f.myVpnAddrsTable.Contains(vpnIp) {
+	_, found := c.f.myVpnAddrsTable.Lookup(vpnIp)
+	if found {
 		// Only returning the default certificate since its impossible
 		// for any other host but ourselves to have more than 1
 		return c.f.pki.getCertState().GetDefaultCertificate().Copy()

control_test.go

@@ -53,7 +53,7 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) {
 		localIndexId:  201,
 		vpnAddrs:      []netip.Addr{vpnIp},
 		relayState: RelayState{
-			relays:         nil,
+			relays:         map[netip.Addr]struct{}{},
 			relayForByAddr: map[netip.Addr]*Relay{},
 			relayForByIdx:  map[uint32]*Relay{},
 		},
@@ -72,14 +72,13 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) {
 		localIndexId:  201,
 		vpnAddrs:      []netip.Addr{vpnIp2},
 		relayState: RelayState{
-			relays:         nil,
+			relays:         map[netip.Addr]struct{}{},
 			relayForByAddr: map[netip.Addr]*Relay{},
 			relayForByIdx:  map[uint32]*Relay{},
 		},
 	}, &Interface{})
 
 	c := Control{
-		state: StateReady,
 		f: &Interface{
 			hostMap: hm,
 		},
@@ -102,7 +101,7 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) {
 
 	// Make sure we don't have any unexpected fields
 	assertFields(t, []string{"VpnAddrs", "LocalIndex", "RemoteIndex", "RemoteAddrs", "Cert", "MessageCounter", "CurrentRemote", "CurrentRelaysToMe", "CurrentRelaysThroughMe"}, thi)
-	assert.Equal(t, &expectedInfo, thi)
+	assert.EqualValues(t, &expectedInfo, thi)
 	test.AssertDeepCopyEqual(t, &expectedInfo, thi)
 
 	// Make sure we don't panic if the host info doesn't have a cert yet
@@ -111,7 +110,7 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) {
 	})
 }
 
-func assertFields(t *testing.T, expected []string, actualStruct any) {
+func assertFields(t *testing.T, expected []string, actualStruct interface{}) {
 	val := reflect.ValueOf(actualStruct).Elem()
 	fields := make([]string, val.NumField())
 	for i := 0; i < val.NumField(); i++ {

control_tester.go

@@ -1,4 +1,5 @@
 //go:build e2e_testing
+// +build e2e_testing
 
 package nebula
 
@@ -173,10 +174,6 @@ func (c *Control) GetHostmap() *HostMap {
 	return c.f.hostMap
 }
 
-func (c *Control) GetF() *Interface {
-	return c.f
-}
-
 func (c *Control) GetCertState() *CertState {
 	return c.f.pki.getCertState()
 }

dist/wireshark/nebula.lua

@@ -84,24 +84,30 @@ end
 
 function nebula.prefs_changed()
     if default_settings.all_ports == nebula.prefs.all_ports and default_settings.port == nebula.prefs.port then
+        -- Nothing changed, bail
         return
     end
 
-    -- Remove all existing registrations
+    -- Remove our old dissector
     DissectorTable.get("udp.port"):remove_all(nebula)
 
-    if nebula.prefs.all_ports then
+    if nebula.prefs.all_ports and default_settings.all_ports ~= nebula.prefs.all_ports then
-        -- Register on every port for hole punch capture
+        default_settings.all_port = nebula.prefs.all_ports
+
         for i=0, 65535 do
             DissectorTable.get("udp.port"):add(i, nebula)
         end
-    else
+
-        -- Register on the configured port only
+        -- no need to establish again on specific ports
-        DissectorTable.get("udp.port"):add(nebula.prefs.port, nebula)
+        return
     end
 
-    default_settings.all_ports = nebula.prefs.all_ports
+
-    default_settings.port = nebula.prefs.port
+    if default_settings.all_ports ~= nebula.prefs.all_ports then
+        -- Add our new port dissector
+        default_settings.port = nebula.prefs.port
+        DissectorTable.get("udp.port"):add(default_settings.port, nebula)
+    end
 end
 
 DissectorTable.get("udp.port"):add(default_settings.port, nebula)

dns_server.go

@@ -26,7 +26,7 @@ type dnsRecords struct {
 	dnsMap4         map[string]netip.Addr
 	dnsMap6         map[string]netip.Addr
 	hostMap         *HostMap
-	myVpnAddrsTable *bart.Lite
+	myVpnAddrsTable *bart.Table[struct{}]
 }
 
 func newDnsRecords(l *logrus.Logger, cs *CertState, hostMap *HostMap) *dnsRecords {
@@ -112,8 +112,8 @@ func (d *dnsRecords) isSelfNebulaOrLocalhost(addr string) bool {
 		return true
 	}
 
-	//if we found it in this table, it's good
+	_, found := d.myVpnAddrsTable.Lookup(b)
-	return d.myVpnAddrsTable.Contains(b)
+	return found //if we found it in this table, it's good
 }
 
 func (d *dnsRecords) parseQuery(m *dns.Msg, w dns.ResponseWriter) {

dns_server_test.go

@@ -38,24 +38,24 @@ func TestParsequery(t *testing.T) {
 func Test_getDnsServerAddr(t *testing.T) {
 	c := config.NewC(nil)
 
-	c.Settings["lighthouse"] = map[string]any{
+	c.Settings["lighthouse"] = map[interface{}]interface{}{
-		"dns": map[string]any{
+		"dns": map[interface{}]interface{}{
 			"host": "0.0.0.0",
 			"port": "1",
 		},
 	}
 	assert.Equal(t, "0.0.0.0:1", getDnsServerAddr(c))
 
-	c.Settings["lighthouse"] = map[string]any{
+	c.Settings["lighthouse"] = map[interface{}]interface{}{
-		"dns": map[string]any{
+		"dns": map[interface{}]interface{}{
 			"host": "::",
 			"port": "1",
 		},
 	}
 	assert.Equal(t, "[::]:1", getDnsServerAddr(c))
 
-	c.Settings["lighthouse"] = map[string]any{
+	c.Settings["lighthouse"] = map[interface{}]interface{}{
-		"dns": map[string]any{
+		"dns": map[interface{}]interface{}{
 			"host": "[::]",
 			"port": "1",
 		},
@@ -63,8 +63,8 @@ func Test_getDnsServerAddr(t *testing.T) {
 	assert.Equal(t, "[::]:1", getDnsServerAddr(c))
 
 	// Make sure whitespace doesn't mess us up
-	c.Settings["lighthouse"] = map[string]any{
+	c.Settings["lighthouse"] = map[interface{}]interface{}{
-		"dns": map[string]any{
+		"dns": map[interface{}]interface{}{
 			"host": "[::] ",
 			"port": "1",
 		},

e2e/handshake_manager_test.go

@@ -1,565 +0,0 @@
-//go:build e2e_testing
-// +build e2e_testing
-
-package e2e
-
-import (
-	"net/netip"
-	"testing"
-	"time"
-
-	"github.com/slackhq/nebula"
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/e2e/router"
-	"github.com/slackhq/nebula/header"
-	"github.com/slackhq/nebula/udp"
-	"github.com/stretchr/testify/assert"
-)
-
-// makeHandshakePacket creates a handshake packet with the given parameters.
-func makeHandshakePacket(from, to netip.AddrPort, subtype header.MessageSubType, remoteIndex uint32, counter uint64) *udp.Packet {
-	data := make([]byte, 200)
-	header.Encode(data, header.Version, header.Handshake, subtype, remoteIndex, counter)
-	for i := header.Len; i < len(data); i++ {
-		data[i] = byte(i)
-	}
-	return &udp.Packet{To: to, From: from, Data: data}
-}
-
-func TestHandshakeRetransmitDuplicate(t *testing.T) {
-	// Verify the responder correctly handles receiving the same msg1 multiple times
-	// (retransmission). The duplicate goes through CheckAndComplete -> ErrAlreadySeen
-	// and the cached response is resent.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	t.Log("Trigger handshake from me to them")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
-
-	t.Log("Grab my msg1")
-	msg1 := myControl.GetFromUDP(true)
-
-	t.Log("Inject msg1 into them, first time")
-	theirControl.InjectUDPPacket(msg1)
-	_ = theirControl.GetFromUDP(true)
-
-	t.Log("Inject the SAME msg1 again, tests ErrAlreadySeen path")
-	theirControl.InjectUDPPacket(msg1)
-	resp2 := theirControl.GetFromUDP(true)
-	assert.NotNil(t, resp2, "should get cached response on duplicate msg1")
-
-	t.Log("Complete handshake with cached response")
-	myControl.InjectUDPPacket(resp2)
-	myControl.WaitForType(1, 0, theirControl)
-
-	t.Log("Drain cached packet and verify tunnel works")
-	cachedPacket := theirControl.GetFromTun(true)
-	assertUdpPacket(t, []byte("Hi"), cachedPacket, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	t.Log("Verify only one tunnel exists on each side")
-	assert.Len(t, myControl.ListHostmapHosts(false), 1)
-	assert.Len(t, theirControl.ListHostmapHosts(false), 1)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestHandshakeTruncatedPacketRecovery(t *testing.T) {
-	// Verify that a truncated handshake packet is ignored and the real
-	// packet can still complete the handshake.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	t.Log("Trigger handshake")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
-
-	t.Log("Get msg1 and deliver to responder")
-	msg1 := myControl.GetFromUDP(true)
-	theirControl.InjectUDPPacket(msg1)
-
-	t.Log("Get the real response")
-	realResp := theirControl.GetFromUDP(true)
-
-	t.Log("Truncate the response and inject, should be ignored")
-	truncResp := realResp.Copy()
-	truncResp.Data = truncResp.Data[:header.Len]
-	myControl.InjectUDPPacket(truncResp)
-
-	t.Log("Verify pending handshake survived the truncated packet")
-	assert.NotEmpty(t, myControl.ListHostmapHosts(true), "pending handshake should still exist")
-
-	t.Log("Inject real response, should complete handshake")
-	myControl.InjectUDPPacket(realResp)
-	myControl.WaitForType(1, 0, theirControl)
-
-	t.Log("Drain and verify tunnel")
-	cachedPacket := theirControl.GetFromTun(true)
-	assertUdpPacket(t, []byte("Hi"), cachedPacket, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestHandshakeOrphanedMsg2Dropped(t *testing.T) {
-	// A msg2 arriving with no matching pending index should be silently dropped
-	// with no response sent and no state changes.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	t.Log("Complete a normal handshake")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
-	r.RouteForAllUntilTxTun(theirControl)
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	t.Log("Record hostmap state")
-	myIndexes := len(myControl.ListHostmapIndexes(false))
-
-	t.Log("Inject a fake msg2 with unknown RemoteIndex")
-	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0xDEADBEEF, 2))
-
-	t.Log("Verify no new indexes created")
-	assert.Equal(t, myIndexes, len(myControl.ListHostmapIndexes(false)))
-
-	t.Log("Verify no UDP response was sent")
-	time.Sleep(100 * time.Millisecond)
-	assert.Nil(t, myControl.GetFromUDP(false), "should not send a response to orphaned msg2")
-
-	t.Log("Verify existing tunnel still works")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestHandshakeUnknownMessageCounter(t *testing.T) {
-	// A handshake packet with an unexpected message counter should be silently
-	// dropped with no side effects and no UDP response.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	myControl.Start()
-	theirControl.Start()
-
-	t.Log("Inject handshake with MessageCounter=3")
-	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 3))
-
-	t.Log("Inject handshake with MessageCounter=99")
-	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 99))
-
-	t.Log("Verify no tunnels or pending handshakes")
-	assert.Empty(t, myControl.ListHostmapHosts(false))
-	assert.Empty(t, myControl.ListHostmapHosts(true))
-
-	t.Log("Verify no UDP response was sent")
-	time.Sleep(100 * time.Millisecond)
-	assert.Nil(t, myControl.GetFromUDP(false))
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestHandshakeUnknownSubtype(t *testing.T) {
-	// A handshake packet with an unknown subtype should be silently dropped.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
-	theirControl, _, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.Start()
-	theirControl.Start()
-
-	t.Log("Inject handshake with unknown subtype 99")
-	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.MessageSubType(99), 0, 1))
-
-	t.Log("Verify no tunnels or pending handshakes")
-	assert.Empty(t, myControl.ListHostmapHosts(false))
-	assert.Empty(t, myControl.ListHostmapHosts(true))
-
-	t.Log("Verify no UDP response was sent")
-	time.Sleep(100 * time.Millisecond)
-	assert.Nil(t, myControl.GetFromUDP(false))
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestHandshakeLateResponse(t *testing.T) {
-	// After a handshake times out, a late response should be silently ignored
-	// with no new tunnels created.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{
-		"handshakes": m{
-			"try_interval": "200ms",
-			"retries":      2,
-		},
-	})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	myControl.Start()
-	theirControl.Start()
-
-	t.Log("Trigger handshake from me")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
-
-	t.Log("Grab msg1 but don't deliver")
-	msg1 := myControl.GetFromUDP(true)
-
-	t.Log("Wait for handshake to time out")
-	for i := 0; i < 5; i++ {
-		time.Sleep(300 * time.Millisecond)
-		myControl.GetFromUDP(false)
-	}
-
-	t.Log("Confirm no pending handshakes remain")
-	assert.Empty(t, myControl.ListHostmapHosts(true))
-
-	t.Log("Deliver old msg1 to them, they create a tunnel")
-	theirControl.InjectUDPPacket(msg1)
-	resp := theirControl.GetFromUDP(true)
-	assert.NotNil(t, resp)
-
-	t.Log("Inject late response into me, should be ignored")
-	myControl.InjectUDPPacket(resp)
-
-	t.Log("No tunnel should exist on my side")
-	assert.Empty(t, myControl.ListHostmapHosts(false))
-	assert.Empty(t, myControl.ListHostmapHosts(true))
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestHandshakeSelfConnectionRejected(t *testing.T) {
-	// Verify that a node rejects a handshake containing its own VPN IP in the
-	// peer cert. We do this by sending the initiator's own msg1 back to itself.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
-
-	// Need a lighthouse entry to trigger a handshake
-	myControl.InjectLightHouseAddr(netip.MustParseAddr("10.128.0.2"), netip.MustParseAddrPort("10.0.0.2:4242"))
-
-	myControl.Start()
-
-	t.Log("Trigger handshake from me")
-	myControl.InjectTunUDPPacket(netip.MustParseAddr("10.128.0.2"), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
-	msg1 := myControl.GetFromUDP(true)
-
-	t.Log("Drain any handshake retransmits before injecting")
-	time.Sleep(100 * time.Millisecond)
-	for myControl.GetFromUDP(false) != nil {
-	}
-
-	t.Log("Feed my own msg1 back to me as if it came from someone else")
-	selfMsg := msg1.Copy()
-	selfMsg.From = netip.MustParseAddrPort("10.0.0.99:4242")
-	selfMsg.To = myUdpAddr
-	myControl.InjectUDPPacket(selfMsg)
-
-	t.Log("Verify no response was sent (self-connection rejected)")
-	time.Sleep(100 * time.Millisecond)
-	// Drain any further retransmits from the original handshake, then check
-	// that none of them are a handshake response (MessageCounter=2)
-	h := &header.H{}
-	for {
-		p := myControl.GetFromUDP(false)
-		if p == nil {
-			break
-		}
-		_ = h.Parse(p.Data)
-		assert.NotEqual(t, uint64(2), h.MessageCounter,
-			"should not send a stage 2 response to self-connection")
-	}
-
-	t.Log("Verify no tunnel to myself was created")
-	assert.Nil(t, myControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false))
-
-	myControl.Stop()
-}
-
-func TestHandshakeMessageCounter0Dropped(t *testing.T) {
-	// MessageCounter=0 is not a valid handshake message and should be dropped.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, _, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
-	_, _, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.Start()
-
-	t.Log("Inject handshake with MessageCounter=0")
-	myControl.InjectUDPPacket(makeHandshakePacket(theirUdpAddr, myUdpAddr, header.HandshakeIXPSK0, 0, 0))
-
-	time.Sleep(100 * time.Millisecond)
-	assert.Empty(t, myControl.ListHostmapHosts(false))
-	assert.Empty(t, myControl.ListHostmapHosts(true))
-	assert.Nil(t, myControl.GetFromUDP(false))
-
-	myControl.Stop()
-}
-
-func TestHandshakeRemoteAllowList(t *testing.T) {
-	// Verify that a handshake from a blocked underlay IP is dropped with no
-	// response and no state changes. Then verify the same packet from an
-	// allowed IP succeeds.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{
-		"lighthouse": m{
-			"remote_allow_list": m{
-				"10.0.0.0/8": true,
-				"0.0.0.0/0":  false,
-			},
-		},
-	})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	t.Log("Trigger handshake from them")
-	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi"))
-	msg1 := theirControl.GetFromUDP(true)
-
-	t.Log("Rewrite the source to a blocked IP and inject")
-	blockedMsg := msg1.Copy()
-	blockedMsg.From = netip.MustParseAddrPort("192.168.1.1:4242")
-	myControl.InjectUDPPacket(blockedMsg)
-
-	t.Log("Verify no tunnel, no pending, no response from blocked source")
-	time.Sleep(100 * time.Millisecond)
-	assert.Empty(t, myControl.ListHostmapHosts(false))
-	assert.Empty(t, myControl.ListHostmapHosts(true))
-	assert.Nil(t, myControl.GetFromUDP(false), "should not respond to blocked source")
-
-	t.Log("Now inject the real packet from the allowed source")
-	myControl.InjectUDPPacket(msg1)
-
-	t.Log("Verify handshake completes from allowed source")
-	resp := myControl.GetFromUDP(true)
-	assert.NotNil(t, resp)
-	theirControl.InjectUDPPacket(resp)
-	theirControl.WaitForType(1, 0, myControl)
-
-	t.Log("Drain cached packet and verify tunnel works")
-	cachedPacket := myControl.GetFromTun(true)
-	assertUdpPacket(t, []byte("Hi"), cachedPacket, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), 80, 80)
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestHandshakeAlreadySeenPreferredRemote(t *testing.T) {
-	// When a duplicate msg1 arrives via ErrAlreadySeen, verify the tunnel
-	// remains functional and hostmap index count is stable.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	t.Log("Complete a normal handshake via the router")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi"))
-	r.RouteForAllUntilTxTun(theirControl)
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	t.Log("Record hostmap state")
-	theirIndexes := len(theirControl.ListHostmapIndexes(false))
-	hi := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-	assert.NotNil(t, hi)
-	originalRemote := hi.CurrentRemote
-
-	t.Log("Re-trigger traffic to cause a new handshake attempt (ErrAlreadySeen)")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("roam"))
-	r.RouteForAllUntilTxTun(theirControl)
-
-	t.Log("Verify tunnel still works")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	t.Log("Verify remote is still valid and index count is stable")
-	hi2 := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-	assert.NotNil(t, hi2)
-	assert.Equal(t, originalRemote, hi2.CurrentRemote)
-	assert.Equal(t, theirIndexes, len(theirControl.ListHostmapIndexes(false)),
-		"no extra indexes should be created from ErrAlreadySeen")
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestHandshakeWrongResponderPacketStore(t *testing.T) {
-	// Verify that when the wrong host responds, the cached packets are
-	// transferred to the new handshake, the evil tunnel is closed, evil's
-	// address is blocked, and the correct tunnel is eventually established.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.100/24", nil)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.99/24", nil)
-	evilControl, evilVpnIpNet, evilUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "evil", "10.128.0.2/24", nil)
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), evilUdpAddr)
-
-	r := router.NewR(t, myControl, theirControl, evilControl)
-	defer r.RenderFlow()
-
-	myControl.Start()
-	theirControl.Start()
-	evilControl.Start()
-
-	t.Log("Send multiple packets to them (cached during handshake)")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet1"))
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("packet2"))
-
-	t.Log("Route until evil tunnel is closed")
-	h := &header.H{}
-	r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType {
-		if err := h.Parse(p.Data); err != nil {
-			panic(err)
-		}
-		if h.Type == header.CloseTunnel && p.To == evilUdpAddr {
-			return router.RouteAndExit
-		}
-		return router.KeepRouting
-	})
-
-	t.Log("Verify evil's address is blocked in the new pending handshake")
-	pendingHI := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), true)
-	if pendingHI != nil {
-		assert.NotContains(t, pendingHI.RemoteAddrs, evilUdpAddr,
-			"evil's address should be blocked")
-	}
-
-	t.Log("Inject correct lighthouse addr for them")
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	t.Log("Route until cached packets arrive at the real them")
-	p := r.RouteForAllUntilTxTun(theirControl)
-	assert.NotNil(t, p, "a cached packet should be delivered to the correct host")
-
-	t.Log("Verify the correct host has a tunnel")
-	assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet, theirVpnIpNet, myControl, theirControl)
-
-	t.Log("Verify no hostinfo artifacts from evil remain")
-	assert.Nil(t, myControl.GetHostInfoByVpnAddr(evilVpnIpNet[0].Addr(), true),
-		"no pending hostinfo for evil")
-	assert.Nil(t, myControl.GetHostInfoByVpnAddr(evilVpnIpNet[0].Addr(), false),
-		"no main hostinfo for evil")
-
-	myControl.Stop()
-	theirControl.Stop()
-	evilControl.Stop()
-}
-
-func TestHandshakeRelayComplete(t *testing.T) {
-	// Verify that a relay handshake completes correctly and relay state is
-	// properly maintained on all three nodes.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
-	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
-
-	myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
-	myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
-	relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	r := router.NewR(t, myControl, relayControl, theirControl)
-	defer r.RenderFlow()
-
-	myControl.Start()
-	relayControl.Start()
-	theirControl.Start()
-
-	t.Log("Trigger handshake via relay")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi via relay"))
-
-	p := r.RouteForAllUntilTxTun(theirControl)
-	assertUdpPacket(t, []byte("Hi via relay"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
-
-	t.Log("Verify bidirectional tunnel via relay")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	t.Log("Verify relay state on my side shows relay-to-me")
-	myHI := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
-	assert.NotNil(t, myHI)
-	assert.NotEmpty(t, myHI.CurrentRelaysToMe, "should have relay-to-me for them")
-
-	t.Log("Verify relay state on their side shows relay-to-me")
-	theirHI := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-	assert.NotNil(t, theirHI)
-	assert.NotEmpty(t, theirHI.CurrentRelaysToMe, "should have relay-to-me for me")
-
-	t.Log("Verify relay node shows through-me relays")
-	relayHI := relayControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-	assert.NotNil(t, relayHI)
-
-	myControl.Stop()
-	relayControl.Stop()
-	theirControl.Stop()
-}
-
-// NOTE: Relay V1 cert + IPv6 rejection is not tested here because
-// InjectTunUDPPacket from a V4 node to a V6 address panics in the test
-// framework. The check is in handshake_manager.go handleOutbound relay
-// logic (lines ~304-313): if the relay host has a V1 cert and either
-// address is IPv6, the relay is skipped.
-
-// NOTE: Relay reestablishment (Disestablished state transition) is covered
-// by the existing TestReestablishRelays in handshakes_test.go.

e2e/handshakes_test.go

@@ -20,17 +20,16 @@ import (
 	"github.com/slackhq/nebula/udp"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.yaml.in/yaml/v3"
+	"gopkg.in/yaml.v2"
 )
 
 func BenchmarkHotPath(b *testing.B) {
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
+	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
 	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", nil)
 
 	// Put their info in our lighthouse
 	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
 
 	// Start the servers
 	myControl.Start()
@@ -39,9 +38,6 @@ func BenchmarkHotPath(b *testing.B) {
 	r := router.NewR(b, myControl, theirControl)
 	r.CancelFlowLogs()
 
-	assertTunnel(b, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-	b.ResetTimer()
-
 	for n := 0; n < b.N; n++ {
 		myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
 		_ = r.RouteForAllUntilTxTun(theirControl)
@@ -51,39 +47,6 @@ func BenchmarkHotPath(b *testing.B) {
 	theirControl.Stop()
 }
 
-func BenchmarkHotPathRelay(b *testing.B) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
-	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay  ", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them   ", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
-
-	// Teach my how to get to the relay and that their can be reached via the relay
-	myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
-	myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
-	relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	// Build a router so we don't have to reason who gets which packet
-	r := router.NewR(b, myControl, relayControl, theirControl)
-	r.CancelFlowLogs()
-
-	// Start the servers
-	myControl.Start()
-	relayControl.Start()
-	theirControl.Start()
-
-	assertTunnel(b, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
-	b.ResetTimer()
-
-	for n := 0; n < b.N; n++ {
-		myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-		_ = r.RouteForAllUntilTxTun(theirControl)
-	}
-
-	myControl.Stop()
-	theirControl.Stop()
-	relayControl.Stop()
-}
-
 func TestGoodHandshake(t *testing.T) {
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)
@@ -134,41 +97,6 @@ func TestGoodHandshake(t *testing.T) {
 	theirControl.Stop()
 }
 
-func TestGoodHandshakeNoOverlap(t *testing.T) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.1/24", nil)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "2001::69/24", nil) //look ma, cross-stack!
-
-	// Put their info in our lighthouse
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	empty := []byte{}
-	t.Log("do something to cause a handshake")
-	myControl.GetF().SendMessageToVpnAddr(header.Test, header.MessageNone, theirVpnIpNet[0].Addr(), empty, empty, empty)
-
-	t.Log("Have them consume my stage 0 packet. They have a tunnel now")
-	theirControl.InjectUDPPacket(myControl.GetFromUDP(true))
-
-	t.Log("Get their stage 1 packet")
-	stage1Packet := theirControl.GetFromUDP(true)
-
-	t.Log("Have me consume their stage 1 packet. I have a tunnel now")
-	myControl.InjectUDPPacket(stage1Packet)
-
-	t.Log("Wait until we see a test packet come through to make sure we give the tunnel time to complete")
-	myControl.WaitForType(header.Test, 0, theirControl)
-
-	t.Log("Make sure our host infos are correct")
-	assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet, theirVpnIpNet, myControl, theirControl)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
 func TestWrongResponderHandshake(t *testing.T) {
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 
@@ -536,35 +464,6 @@ func TestRelays(t *testing.T) {
 	r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
 }
 
-func TestRelaysDontCareAboutIps(t *testing.T) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
-	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "relay  ", "2001::9999/24", m{"relay": m{"am_relay": true}})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them   ", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
-
-	// Teach my how to get to the relay and that their can be reached via the relay
-	myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
-	myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
-	relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-
-	// Build a router so we don't have to reason who gets which packet
-	r := router.NewR(t, myControl, relayControl, theirControl)
-	defer r.RenderFlow()
-
-	// Start the servers
-	myControl.Start()
-	relayControl.Start()
-	theirControl.Start()
-
-	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-
-	p := r.RouteForAllUntilTxTun(theirControl)
-	r.Log("Assert the tunnel works")
-	assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
-	r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
-}
-
 func TestReestablishRelays(t *testing.T) {
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me     ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
@@ -607,7 +506,7 @@ func TestReestablishRelays(t *testing.T) {
 	curIndexes := len(myControl.GetHostmap().Indexes)
 	for curIndexes >= start {
 		curIndexes = len(myControl.GetHostmap().Indexes)
-		r.Logf("Wait for the dead index to go away:start=%v indexes, current=%v indexes", start, curIndexes)
+		r.Logf("Wait for the dead index to go away:start=%v indexes, currnet=%v indexes", start, curIndexes)
 		myControl.InjectTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me should fail"))
 
 		r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType {
@@ -1092,7 +991,7 @@ func TestRehandshaking(t *testing.T) {
 	require.NoError(t, err)
 	var theirNewConfig m
 	require.NoError(t, yaml.Unmarshal(rc, &theirNewConfig))
-	theirFirewall := theirNewConfig["firewall"].(map[string]any)
+	theirFirewall := theirNewConfig["firewall"].(map[interface{}]interface{})
 	theirFirewall["inbound"] = []m{{
 		"proto": "any",
 		"port":  "any",
@@ -1153,9 +1052,6 @@ func TestRehandshakingLoser(t *testing.T) {
 	t.Log("Stand up a tunnel between me and them")
 	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
 
-	myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
-	theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-
 	r.RenderHostmaps("Starting hostmaps", myControl, theirControl)
 
 	r.Log("Renew their certificate and spin until mine sees it")
@@ -1191,7 +1087,7 @@ func TestRehandshakingLoser(t *testing.T) {
 	require.NoError(t, err)
 	var myNewConfig m
 	require.NoError(t, yaml.Unmarshal(rc, &myNewConfig))
-	theirFirewall := myNewConfig["firewall"].(map[string]any)
+	theirFirewall := myNewConfig["firewall"].(map[interface{}]interface{})
 	theirFirewall["inbound"] = []m{{
 		"proto": "any",
 		"port":  "any",
@@ -1329,108 +1225,134 @@ func TestV2NonPrimaryWithLighthouse(t *testing.T) {
 	theirControl.Stop()
 }
 
-func TestV2NonPrimaryWithOffNetLighthouse(t *testing.T) {
+func TestPSK(t *testing.T) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+	tests := []struct {
-	lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh  ", "2001::1/64", m{"lighthouse": m{"am_lighthouse": true}})
+		name         string
-
+		myPskMode    nebula.PskMode
-	o := m{
+		theirPskMode nebula.PskMode
-		"static_host_map": m{
+	}{
-			lhVpnIpNet[0].Addr().String(): []string{lhUdpAddr.String()},
+		// All accepting
+		{
+			name:         "both accepting",
+			myPskMode:    nebula.PskAccepting,
+			theirPskMode: nebula.PskAccepting,
 		},
-		"lighthouse": m{
+
-			"hosts": []string{lhVpnIpNet[0].Addr().String()},
+		// accepting and sending both ways
-			"local_allow_list": m{
+		{
-				// Try and block our lighthouse updates from using the actual addresses assigned to this computer
+			name:         "accepting to sending",
-				// If we start discovering addresses the test router doesn't know about then test traffic cant flow
+			myPskMode:    nebula.PskAccepting,
-				"10.0.0.0/24": true,
+			theirPskMode: nebula.PskSending,
-				"::/0":        false,
+		},
-			},
+		{
+			name:         "sending to accepting",
+			myPskMode:    nebula.PskSending,
+			theirPskMode: nebula.PskAccepting,
+		},
+
+		// All sending
+		{
+			name:         "sending to sending",
+			myPskMode:    nebula.PskSending,
+			theirPskMode: nebula.PskSending,
+		},
+
+		// enforced and sending both ways
+		{
+			name:         "enforced to sending",
+			myPskMode:    nebula.PskEnforced,
+			theirPskMode: nebula.PskSending,
+		},
+		{
+			name:         "sending to enforced",
+			myPskMode:    nebula.PskSending,
+			theirPskMode: nebula.PskEnforced,
+		},
+
+		// All enforced
+		{
+			name:         "both enforced",
+			myPskMode:    nebula.PskEnforced,
+			theirPskMode: nebula.PskEnforced,
+		},
+
+		// Enforced can technically handshake with an accepting node, but it is bad to be in this state
+		{
+			name:         "enforced to accepting",
+			myPskMode:    nebula.PskEnforced,
+			theirPskMode: nebula.PskAccepting,
 		},
 	}
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me  ", "10.128.0.2/24, ff::2/64", o)
-	theirControl, theirVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.3/24, ff::3/64", o)
 
-	// Build a router so we don't have to reason who gets which packet
+	for _, test := range tests {
-	r := router.NewR(t, lhControl, myControl, theirControl)
+		t.Run(test.name, func(t *testing.T) {
-	defer r.RenderFlow()
+			var myPskSettings, theirPskSettings m
 
-	// Start the servers
+			switch test.myPskMode {
-	lhControl.Start()
+			case nebula.PskAccepting:
-	myControl.Start()
+				myPskSettings = m{"psk": &m{"mode": "accepting", "keys": []string{"garbage0", "this is a key"}}}
-	theirControl.Start()
+			case nebula.PskSending:
+				myPskSettings = m{"psk": &m{"mode": "sending", "keys": []string{"this is a key", "garbage1"}}}
+			case nebula.PskEnforced:
+				myPskSettings = m{"psk": &m{"mode": "enforced", "keys": []string{"this is a key", "garbage2"}}}
+			}
 
-	t.Log("Stand up an ipv6 tunnel between me and them")
+			switch test.theirPskMode {
-	assert.True(t, myVpnIpNet[1].Addr().Is6())
+			case nebula.PskAccepting:
-	assert.True(t, theirVpnIpNet[1].Addr().Is6())
+				theirPskSettings = m{"psk": &m{"mode": "accepting", "keys": []string{"garbage3", "this is a key"}}}
-	assertTunnel(t, myVpnIpNet[1].Addr(), theirVpnIpNet[1].Addr(), myControl, theirControl, r)
+			case nebula.PskSending:
+				theirPskSettings = m{"psk": &m{"mode": "sending", "keys": []string{"this is a key", "garbage4"}}}
+			case nebula.PskEnforced:
+				theirPskSettings = m{"psk": &m{"mode": "enforced", "keys": []string{"this is a key", "garbage5"}}}
+			}
 
-	lhControl.Stop()
+			ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, nil)
-	myControl.Stop()
+			myControl, myVpnIp, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.0.0.1/24", myPskSettings)
-	theirControl.Stop()
+			theirControl, theirVpnIp, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.0.0.2/24", theirPskSettings)
-}
 
-func TestGoodHandshakeUnsafeDest(t *testing.T) {
+			myControl.InjectLightHouseAddr(theirVpnIp[0].Addr(), theirUdpAddr)
-	unsafePrefix := "192.168.6.0/24"
+			r := router.NewR(t, myControl, theirControl)
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServerWithUdpAndUnsafeNetworks(cert.Version2, ca, caKey, "spooky", "10.128.0.2/24", netip.MustParseAddrPort("10.64.0.2:4242"), unsafePrefix, nil)
+			// Start the servers
-	route := m{"route": unsafePrefix, "via": theirVpnIpNet[0].Addr().String()}
+			myControl.Start()
-	myCfg := m{
+			theirControl.Start()
-		"tun": m{
+
-			"unsafe_routes": []m{route},
+			t.Log("Route until we see our cached packet flow")
-		},
+			myControl.InjectTunUDPPacket(theirVpnIp[0].Addr(), 80, myVpnIp[0].Addr(), 80, []byte("Hi from me"))
+			r.RouteForAllExitFunc(func(p *udp.Packet, c *nebula.Control) router.ExitType {
+				h := &header.H{}
+				err := h.Parse(p.Data)
+				if err != nil {
+					panic(err)
+				}
+
+				// If this is the stage 1 handshake packet and I am configured to send with a psk, my cert name should
+				// not appear. It would likely be more obvious to unmarshal the payload and check but this works fine for now
+				if test.myPskMode == nebula.PskEnforced || test.myPskMode == nebula.PskSending {
+					if h.Type == 0 && h.MessageCounter == 1 {
+						assert.NotContains(t, string(p.Data), "test me")
+					}
+				}
+
+				if p.To == theirUdpAddr && h.Type == 1 {
+					return router.RouteAndExit
+				}
+
+				return router.KeepRouting
+			})
+
+			t.Log("My cached packet should be received by them")
+			myCachedPacket := theirControl.GetFromTun(true)
+			assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIp[0].Addr(), theirVpnIp[0].Addr(), 80, 80)
+
+			t.Log("Test the tunnel with them")
+			assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIp, theirVpnIp, myControl, theirControl)
+			assertTunnel(t, myVpnIp[0].Addr(), theirVpnIp[0].Addr(), myControl, theirControl, r)
+
+			myControl.Stop()
+			theirControl.Stop()
+			//TODO: assert hostmaps
+		})
 	}
-	myControl, myVpnIpNet, myUdpAddr, myConfig := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.1/24", myCfg)
-	t.Logf("my config %v", myConfig)
-	// Put their info in our lighthouse
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
 
-	spookyDest := netip.MustParseAddr("192.168.6.4")
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	t.Log("Send a udp packet through to begin standing up the tunnel, this should come out the other side")
-	myControl.InjectTunUDPPacket(spookyDest, 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
-
-	t.Log("Have them consume my stage 0 packet. They have a tunnel now")
-	theirControl.InjectUDPPacket(myControl.GetFromUDP(true))
-
-	t.Log("Get their stage 1 packet so that we can play with it")
-	stage1Packet := theirControl.GetFromUDP(true)
-
-	t.Log("I consume a garbage packet with a proper nebula header for our tunnel")
-	// this should log a statement and get ignored, allowing the real handshake packet to complete the tunnel
-	badPacket := stage1Packet.Copy()
-	badPacket.Data = badPacket.Data[:len(badPacket.Data)-header.Len]
-	myControl.InjectUDPPacket(badPacket)
-
-	t.Log("Have me consume their real stage 1 packet. I have a tunnel now")
-	myControl.InjectUDPPacket(stage1Packet)
-
-	t.Log("Wait until we see my cached packet come through")
-	myControl.WaitForType(1, 0, theirControl)
-
-	t.Log("Make sure our host infos are correct")
-	assertHostInfoPair(t, myUdpAddr, theirUdpAddr, myVpnIpNet, theirVpnIpNet, myControl, theirControl)
-
-	t.Log("Get that cached packet and make sure it looks right")
-	myCachedPacket := theirControl.GetFromTun(true)
-	assertUdpPacket(t, []byte("Hi from me"), myCachedPacket, myVpnIpNet[0].Addr(), spookyDest, 80, 80)
-
-	//reply
-	theirControl.InjectTunUDPPacket(myVpnIpNet[0].Addr(), 80, spookyDest, 80, []byte("Hi from the spookyman"))
-	//wait for reply
-	theirControl.WaitForType(1, 0, myControl)
-	theirCachedPacket := myControl.GetFromTun(true)
-	assertUdpPacket(t, []byte("Hi from the spookyman"), theirCachedPacket, spookyDest, myVpnIpNet[0].Addr(), 80, 80)
-
-	t.Log("Do a bidirectional tunnel test")
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	r.RenderHostmaps("Final hostmaps", myControl, theirControl)
-	myControl.Stop()
-	theirControl.Stop()
 }

e2e/helpers_test.go

@@ -22,14 +22,15 @@ import (
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/e2e/router"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
+	"gopkg.in/yaml.v2"
-	"go.yaml.in/yaml/v3"
 )
 
-type m = map[string]any
+type m map[string]interface{}
 
 // newSimpleServer creates a nebula instance with many assumptions
 func newSimpleServer(v cert.Version, caCrt cert.Certificate, caKey []byte, name string, sVpnNetworks string, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) {
+	l := NewTestLogger()
+
 	var vpnNetworks []netip.Prefix
 	for _, sn := range strings.Split(sVpnNetworks, ",") {
 		vpnIpNet, err := netip.ParsePrefix(strings.TrimSpace(sn))
@@ -55,54 +56,7 @@ func newSimpleServer(v cert.Version, caCrt cert.Certificate, caKey []byte, name
 		budpIp[3] = 239
 		udpAddr = netip.AddrPortFrom(netip.AddrFrom16(budpIp), 4242)
 	}
-	return newSimpleServerWithUdp(v, caCrt, caKey, name, sVpnNetworks, udpAddr, overrides)
+	_, _, myPrivKey, myPEM := cert_test.NewTestCert(v, cert.Curve_CURVE25519, caCrt, caKey, name, time.Now(), time.Now().Add(5*time.Minute), vpnNetworks, nil, []string{})
-}
-
-func newSimpleServerWithUdp(v cert.Version, caCrt cert.Certificate, caKey []byte, name string, sVpnNetworks string, udpAddr netip.AddrPort, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) {
-	return newSimpleServerWithUdpAndUnsafeNetworks(v, caCrt, caKey, name, sVpnNetworks, udpAddr, "", overrides)
-}
-
-func newSimpleServerWithUdpAndUnsafeNetworks(v cert.Version, caCrt cert.Certificate, caKey []byte, name string, sVpnNetworks string, udpAddr netip.AddrPort, sUnsafeNetworks string, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) {
-	l := NewTestLogger()
-
-	var vpnNetworks []netip.Prefix
-	for _, sn := range strings.Split(sVpnNetworks, ",") {
-		vpnIpNet, err := netip.ParsePrefix(strings.TrimSpace(sn))
-		if err != nil {
-			panic(err)
-		}
-		vpnNetworks = append(vpnNetworks, vpnIpNet)
-	}
-
-	if len(vpnNetworks) == 0 {
-		panic("no vpn networks")
-	}
-
-	firewallInbound := []m{{
-		"proto": "any",
-		"port":  "any",
-		"host":  "any",
-	}}
-
-	var unsafeNetworks []netip.Prefix
-	if sUnsafeNetworks != "" {
-		firewallInbound = []m{{
-			"proto":      "any",
-			"port":       "any",
-			"host":       "any",
-			"local_cidr": "0.0.0.0/0",
-		}}
-
-		for _, sn := range strings.Split(sUnsafeNetworks, ",") {
-			x, err := netip.ParsePrefix(strings.TrimSpace(sn))
-			if err != nil {
-				panic(err)
-			}
-			unsafeNetworks = append(unsafeNetworks, x)
-		}
-	}
-
-	_, _, myPrivKey, myPEM := cert_test.NewTestCert(v, cert.Curve_CURVE25519, caCrt, caKey, name, time.Now(), time.Now().Add(5*time.Minute), vpnNetworks, unsafeNetworks, []string{})
 
 	caB, err := caCrt.MarshalPEM()
 	if err != nil {
@@ -122,7 +76,11 @@ func newSimpleServerWithUdpAndUnsafeNetworks(v cert.Version, caCrt cert.Certific
 				"port":  "any",
 				"host":  "any",
 			}},
-			"inbound": firewallInbound,
+			"inbound": []m{{
+				"proto": "any",
+				"port":  "any",
+				"host":  "any",
+			}},
 		},
 		//"handshakes": m{
 		//	"try_interval": "1s",
@@ -171,109 +129,6 @@ func newSimpleServerWithUdpAndUnsafeNetworks(v cert.Version, caCrt cert.Certific
 	return control, vpnNetworks, udpAddr, c
 }
 
-// newServer creates a nebula instance with fewer assumptions
-func newServer(caCrt []cert.Certificate, certs []cert.Certificate, key []byte, overrides m) (*nebula.Control, []netip.Prefix, netip.AddrPort, *config.C) {
-	l := NewTestLogger()
-
-	vpnNetworks := certs[len(certs)-1].Networks()
-
-	var udpAddr netip.AddrPort
-	if vpnNetworks[0].Addr().Is4() {
-		budpIp := vpnNetworks[0].Addr().As4()
-		budpIp[1] -= 128
-		udpAddr = netip.AddrPortFrom(netip.AddrFrom4(budpIp), 4242)
-	} else {
-		budpIp := vpnNetworks[0].Addr().As16()
-		// beef for funsies
-		budpIp[2] = 190
-		budpIp[3] = 239
-		udpAddr = netip.AddrPortFrom(netip.AddrFrom16(budpIp), 4242)
-	}
-
-	caStr := ""
-	for _, ca := range caCrt {
-		x, err := ca.MarshalPEM()
-		if err != nil {
-			panic(err)
-		}
-		caStr += string(x)
-	}
-	certStr := ""
-	for _, c := range certs {
-		x, err := c.MarshalPEM()
-		if err != nil {
-			panic(err)
-		}
-		certStr += string(x)
-	}
-
-	mc := m{
-		"pki": m{
-			"ca":   caStr,
-			"cert": certStr,
-			"key":  string(key),
-		},
-		//"tun": m{"disabled": true},
-		"firewall": m{
-			"outbound": []m{{
-				"proto": "any",
-				"port":  "any",
-				"host":  "any",
-			}},
-			"inbound": []m{{
-				"proto": "any",
-				"port":  "any",
-				"host":  "any",
-			}},
-		},
-		//"handshakes": m{
-		//	"try_interval": "1s",
-		//},
-		"listen": m{
-			"host": udpAddr.Addr().String(),
-			"port": udpAddr.Port(),
-		},
-		"logging": m{
-			"timestamp_format": fmt.Sprintf("%v 15:04:05.000000", certs[0].Name()),
-			"level":            l.Level.String(),
-		},
-		"timers": m{
-			"pending_deletion_interval": 2,
-			"connection_alive_interval": 2,
-		},
-	}
-
-	if overrides != nil {
-		final := m{}
-		err := mergo.Merge(&final, overrides, mergo.WithAppendSlice)
-		if err != nil {
-			panic(err)
-		}
-		err = mergo.Merge(&final, mc, mergo.WithAppendSlice)
-		if err != nil {
-			panic(err)
-		}
-		mc = final
-	}
-
-	cb, err := yaml.Marshal(mc)
-	if err != nil {
-		panic(err)
-	}
-
-	c := config.NewC(l)
-	cStr := string(cb)
-	c.LoadString(cStr)
-
-	control, err := nebula.Main(c, false, "e2e-test", l, nil)
-
-	if err != nil {
-		panic(err)
-	}
-
-	return control, vpnNetworks, udpAddr, c
-}
-
 type doneCb func()
 
 func deadline(t *testing.T, seconds time.Duration) doneCb {
@@ -292,7 +147,7 @@ func deadline(t *testing.T, seconds time.Duration) doneCb {
 	}
 }
 
-func assertTunnel(t testing.TB, vpnIpA, vpnIpB netip.Addr, controlA, controlB *nebula.Control, r *router.R) {
+func assertTunnel(t *testing.T, vpnIpA, vpnIpB netip.Addr, controlA, controlB *nebula.Control, r *router.R) {
 	// Send a packet from them to me
 	controlB.InjectTunUDPPacket(vpnIpA, 80, vpnIpB, 90, []byte("Hi from B"))
 	bPacket := r.RouteForAllUntilTxTun(controlA)
@@ -304,14 +159,14 @@ func assertTunnel(t testing.TB, vpnIpA, vpnIpB netip.Addr, controlA, controlB *n
 	assertUdpPacket(t, []byte("Hello from A"), aPacket, vpnIpA, vpnIpB, 90, 80)
 }
 
-func assertHostInfoPair(t testing.TB, addrA, addrB netip.AddrPort, vpnNetsA, vpnNetsB []netip.Prefix, controlA, controlB *nebula.Control) {
+func assertHostInfoPair(t *testing.T, addrA, addrB netip.AddrPort, vpnNetsA, vpnNetsB []netip.Prefix, controlA, controlB *nebula.Control) {
 	// Get both host infos
 	//TODO: CERT-V2 we may want to loop over each vpnAddr and assert all the things
 	hBinA := controlA.GetHostInfoByVpnAddr(vpnNetsB[0].Addr(), false)
-	require.NotNil(t, hBinA, "Host B was not found by vpnAddr in controlA")
+	assert.NotNil(t, hBinA, "Host B was not found by vpnAddr in controlA")
 
 	hAinB := controlB.GetHostInfoByVpnAddr(vpnNetsA[0].Addr(), false)
-	require.NotNil(t, hAinB, "Host A was not found by vpnAddr in controlB")
+	assert.NotNil(t, hAinB, "Host A was not found by vpnAddr in controlB")
 
 	// Check that both vpn and real addr are correct
 	assert.EqualValues(t, getAddrs(vpnNetsB), hBinA.VpnAddrs, "Host B VpnIp is wrong in control A")
@@ -325,7 +180,7 @@ func assertHostInfoPair(t testing.TB, addrA, addrB netip.AddrPort, vpnNetsA, vpn
 	assert.Equal(t, hBinA.RemoteIndex, hAinB.LocalIndex, "Host B remote index does not match host A local index")
 }
 
-func assertUdpPacket(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
+func assertUdpPacket(t *testing.T, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
 	if toIp.Is6() {
 		assertUdpPacket6(t, expected, b, fromIp, toIp, fromPort, toPort)
 	} else {
@@ -333,7 +188,7 @@ func assertUdpPacket(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr,
 	}
 }
 
-func assertUdpPacket6(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
+func assertUdpPacket6(t *testing.T, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
 	packet := gopacket.NewPacket(b, layers.LayerTypeIPv6, gopacket.Lazy)
 	v6 := packet.Layer(layers.LayerTypeIPv6).(*layers.IPv6)
 	assert.NotNil(t, v6, "No ipv6 data found")
@@ -352,7 +207,7 @@ func assertUdpPacket6(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr,
 	assert.Equal(t, expected, data.Payload(), "Data was incorrect")
 }
 
-func assertUdpPacket4(t testing.TB, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
+func assertUdpPacket4(t *testing.T, expected, b []byte, fromIp, toIp netip.Addr, fromPort, toPort uint16) {
 	packet := gopacket.NewPacket(b, layers.LayerTypeIPv4, gopacket.Lazy)
 	v4 := packet.Layer(layers.LayerTypeIPv4).(*layers.IPv4)
 	assert.NotNil(t, v4, "No ipv4 data found")

e2e/router/router.go

@@ -111,10 +111,6 @@ type ExitFunc func(packet *udp.Packet, receiver *nebula.Control) ExitType
 func NewR(t testing.TB, controls ...*nebula.Control) *R {
 	ctx, cancel := context.WithCancel(context.Background())
 
-	if err := os.MkdirAll("mermaid", 0755); err != nil {
-		panic(err)
-	}
-
 	r := &R{
 		controls:     make(map[netip.AddrPort]*nebula.Control),
 		vpnControls:  make(map[netip.Addr]*nebula.Control),
@@ -194,6 +190,9 @@ func (r *R) renderFlow() {
 		return
 	}
 
+	if err := os.MkdirAll(filepath.Dir(r.fn), 0755); err != nil {
+		panic(err)
+	}
 	f, err := os.OpenFile(r.fn, os.O_CREATE|os.O_TRUNC|os.O_RDWR, 0644)
 	if err != nil {
 		panic(err)
@@ -700,7 +699,6 @@ func (r *R) FlushAll() {
 			r.Unlock()
 			panic("Can't FlushAll for host: " + p.To.String())
 		}
-		receiver.InjectUDPPacket(p)
 		r.Unlock()
 	}
 }

e2e/tunnels_test.go

@@ -1,472 +0,0 @@
-//go:build e2e_testing
-// +build e2e_testing
-
-package e2e
-
-import (
-	"fmt"
-	"net/netip"
-	"testing"
-	"time"
-
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/e2e/router"
-	"github.com/slackhq/nebula/header"
-	"github.com/slackhq/nebula/udp"
-	"github.com/stretchr/testify/assert"
-	"gopkg.in/yaml.v3"
-)
-
-func TestDropInactiveTunnels(t *testing.T) {
-	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
-	// under ideal conditions
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "5s"}})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "10m"}})
-
-	// Share our underlay information
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-
-	r.Log("Assert the tunnel between me and them works")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	r.Log("Go inactive and wait for the tunnels to get dropped")
-	waitStart := time.Now()
-	for {
-		myIndexes := len(myControl.GetHostmap().Indexes)
-		theirIndexes := len(theirControl.GetHostmap().Indexes)
-		if myIndexes == 0 && theirIndexes == 0 {
-			break
-		}
-
-		since := time.Since(waitStart)
-		r.Logf("my tunnels: %v; their tunnels: %v; duration: %v", myIndexes, theirIndexes, since)
-		if since > time.Second*30 {
-			t.Fatal("Tunnel should have been declared inactive after 5 seconds and before 30 seconds")
-		}
-
-		time.Sleep(1 * time.Second)
-		r.FlushAll()
-	}
-
-	r.Logf("Inactive tunnels were dropped within %v", time.Since(waitStart))
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestCertUpgrade(t *testing.T) {
-	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
-	// under ideal conditions
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	caB, err := ca.MarshalPEM()
-	if err != nil {
-		panic(err)
-	}
-	ca2, _, caKey2, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	ca2B, err := ca2.MarshalPEM()
-	if err != nil {
-		panic(err)
-	}
-	caStr := fmt.Sprintf("%s\n%s", caB, ca2B)
-
-	myCert, _, myPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "me", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.1/24")}, nil, []string{})
-	_, myCert2Pem := cert_test.NewTestCertDifferentVersion(myCert, cert.Version2, ca2, caKey2)
-
-	theirCert, _, theirPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "them", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.2/24")}, nil, []string{})
-	theirCert2, _ := cert_test.NewTestCertDifferentVersion(theirCert, cert.Version2, ca2, caKey2)
-
-	myControl, myVpnIpNet, myUdpAddr, myC := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{myCert}, myPrivKey, m{})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{theirCert, theirCert2}, theirPrivKey, m{})
-
-	// Share our underlay information
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	r.Log("Assert the tunnel between me and them works")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-	r.Log("yay")
-	//todo ???
-	time.Sleep(1 * time.Second)
-	r.FlushAll()
-
-	mc := m{
-		"pki": m{
-			"ca":   caStr,
-			"cert": string(myCert2Pem),
-			"key":  string(myPrivKey),
-		},
-		//"tun": m{"disabled": true},
-		"firewall": myC.Settings["firewall"],
-		//"handshakes": m{
-		//	"try_interval": "1s",
-		//},
-		"listen":  myC.Settings["listen"],
-		"logging": myC.Settings["logging"],
-		"timers":  myC.Settings["timers"],
-	}
-
-	cb, err := yaml.Marshal(mc)
-	if err != nil {
-		panic(err)
-	}
-
-	r.Logf("reload new v2-only config")
-	err = myC.ReloadConfigString(string(cb))
-	assert.NoError(t, err)
-	r.Log("yay, spin until their sees it")
-	waitStart := time.Now()
-	for {
-		assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-		c := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-		if c == nil {
-			r.Log("nil")
-		} else {
-			version := c.Cert.Version()
-			r.Logf("version %d", version)
-			if version == cert.Version2 {
-				break
-			}
-		}
-		since := time.Since(waitStart)
-		if since > time.Second*10 {
-			t.Fatal("Cert should be new by now")
-		}
-		time.Sleep(time.Second)
-	}
-
-	r.RenderHostmaps("Final hostmaps", myControl, theirControl)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestCertDowngrade(t *testing.T) {
-	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
-	// under ideal conditions
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	caB, err := ca.MarshalPEM()
-	if err != nil {
-		panic(err)
-	}
-	ca2, _, caKey2, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	ca2B, err := ca2.MarshalPEM()
-	if err != nil {
-		panic(err)
-	}
-	caStr := fmt.Sprintf("%s\n%s", caB, ca2B)
-
-	myCert, _, myPrivKey, myCertPem := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "me", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.1/24")}, nil, []string{})
-	myCert2, _ := cert_test.NewTestCertDifferentVersion(myCert, cert.Version2, ca2, caKey2)
-
-	theirCert, _, theirPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "them", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.2/24")}, nil, []string{})
-	theirCert2, _ := cert_test.NewTestCertDifferentVersion(theirCert, cert.Version2, ca2, caKey2)
-
-	myControl, myVpnIpNet, myUdpAddr, myC := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{myCert2}, myPrivKey, m{})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{theirCert, theirCert2}, theirPrivKey, m{})
-
-	// Share our underlay information
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	r.Log("Assert the tunnel between me and them works")
-	//assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
-	//r.Log("yay")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-	r.Log("yay")
-	//todo ???
-	time.Sleep(1 * time.Second)
-	r.FlushAll()
-
-	mc := m{
-		"pki": m{
-			"ca":   caStr,
-			"cert": string(myCertPem),
-			"key":  string(myPrivKey),
-		},
-		"firewall": myC.Settings["firewall"],
-		"listen":   myC.Settings["listen"],
-		"logging":  myC.Settings["logging"],
-		"timers":   myC.Settings["timers"],
-	}
-
-	cb, err := yaml.Marshal(mc)
-	if err != nil {
-		panic(err)
-	}
-
-	r.Logf("reload new v1-only config")
-	err = myC.ReloadConfigString(string(cb))
-	assert.NoError(t, err)
-	r.Log("yay, spin until their sees it")
-	waitStart := time.Now()
-	for {
-		assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-		c := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-		c2 := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
-		if c == nil || c2 == nil {
-			r.Log("nil")
-		} else {
-			version := c.Cert.Version()
-			theirVersion := c2.Cert.Version()
-			r.Logf("version %d,%d", version, theirVersion)
-			if version == cert.Version1 {
-				break
-			}
-		}
-		since := time.Since(waitStart)
-		if since > time.Second*5 {
-			r.Log("it is unusual that the cert is not new yet, but not a failure yet")
-		}
-		if since > time.Second*10 {
-			r.Log("wtf")
-			t.Fatal("Cert should be new by now")
-		}
-		time.Sleep(time.Second)
-	}
-
-	r.RenderHostmaps("Final hostmaps", myControl, theirControl)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestCertMismatchCorrection(t *testing.T) {
-	// The goal of this test is to ensure the shortest inactivity timeout will close the tunnel on both sides
-	// under ideal conditions
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	ca2, _, caKey2, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-
-	myCert, _, myPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "me", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.1/24")}, nil, []string{})
-	myCert2, _ := cert_test.NewTestCertDifferentVersion(myCert, cert.Version2, ca2, caKey2)
-
-	theirCert, _, theirPrivKey, _ := cert_test.NewTestCert(cert.Version1, cert.Curve_CURVE25519, ca, caKey, "them", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{netip.MustParsePrefix("10.128.0.2/24")}, nil, []string{})
-	theirCert2, _ := cert_test.NewTestCertDifferentVersion(theirCert, cert.Version2, ca2, caKey2)
-
-	myControl, myVpnIpNet, myUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{myCert2}, myPrivKey, m{})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newServer([]cert.Certificate{ca, ca2}, []cert.Certificate{theirCert, theirCert2}, theirPrivKey, m{})
-
-	// Share our underlay information
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	r.Log("Assert the tunnel between me and them works")
-	//assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
-	//r.Log("yay")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-	r.Log("yay")
-	//todo ???
-	time.Sleep(1 * time.Second)
-	r.FlushAll()
-
-	waitStart := time.Now()
-	for {
-		assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-		c := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-		c2 := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
-		if c == nil || c2 == nil {
-			r.Log("nil")
-		} else {
-			version := c.Cert.Version()
-			theirVersion := c2.Cert.Version()
-			r.Logf("version %d,%d", version, theirVersion)
-			if version == theirVersion {
-				break
-			}
-		}
-		since := time.Since(waitStart)
-		if since > time.Second*5 {
-			r.Log("wtf")
-		}
-		if since > time.Second*10 {
-			r.Log("wtf")
-			t.Fatal("Cert should be new by now")
-		}
-		time.Sleep(time.Second)
-	}
-
-	r.RenderHostmaps("Final hostmaps", myControl, theirControl)
-
-	myControl.Stop()
-	theirControl.Stop()
-}
-
-func TestCrossStackRelaysWork(t *testing.T) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me     ", "10.128.0.1/24,fc00::1/64", m{"relay": m{"use_relays": true}})
-	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "relay  ", "10.128.0.128/24,fc00::128/64", m{"relay": m{"am_relay": true}})
-	theirUdp := netip.MustParseAddrPort("10.0.0.2:4242")
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServerWithUdp(cert.Version2, ca, caKey, "them   ", "fc00::2/64", theirUdp, m{"relay": m{"use_relays": true}})
-
-	//myVpnV4 := myVpnIpNet[0]
-	myVpnV6 := myVpnIpNet[1]
-	relayVpnV4 := relayVpnIpNet[0]
-	relayVpnV6 := relayVpnIpNet[1]
-	theirVpnV6 := theirVpnIpNet[0]
-
-	// Teach my how to get to the relay and that their can be reached via the relay
-	myControl.InjectLightHouseAddr(relayVpnV4.Addr(), relayUdpAddr)
-	myControl.InjectLightHouseAddr(relayVpnV6.Addr(), relayUdpAddr)
-	myControl.InjectRelays(theirVpnV6.Addr(), []netip.Addr{relayVpnV6.Addr()})
-	relayControl.InjectLightHouseAddr(theirVpnV6.Addr(), theirUdpAddr)
-
-	// Build a router so we don't have to reason who gets which packet
-	r := router.NewR(t, myControl, relayControl, theirControl)
-	defer r.RenderFlow()
-
-	// Start the servers
-	myControl.Start()
-	relayControl.Start()
-	theirControl.Start()
-
-	t.Log("Trigger a handshake from me to them via the relay")
-	myControl.InjectTunUDPPacket(theirVpnV6.Addr(), 80, myVpnV6.Addr(), 80, []byte("Hi from me"))
-
-	p := r.RouteForAllUntilTxTun(theirControl)
-	r.Log("Assert the tunnel works")
-	assertUdpPacket(t, []byte("Hi from me"), p, myVpnV6.Addr(), theirVpnV6.Addr(), 80, 80)
-
-	t.Log("reply?")
-	theirControl.InjectTunUDPPacket(myVpnV6.Addr(), 80, theirVpnV6.Addr(), 80, []byte("Hi from them"))
-	p = r.RouteForAllUntilTxTun(myControl)
-	assertUdpPacket(t, []byte("Hi from them"), p, theirVpnV6.Addr(), myVpnV6.Addr(), 80, 80)
-
-	r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
-	//t.Log("finish up")
-	//myControl.Stop()
-	//theirControl.Stop()
-	//relayControl.Stop()
-}
-
-func TestCloseTunnelAuthenticated(t *testing.T) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "5s"}})
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them", "10.128.0.2/24", m{"tunnels": m{"drop_inactive": true, "inactivity_timeout": "10m"}})
-
-	// Share our underlay information
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	// Start the servers
-	myControl.Start()
-	theirControl.Start()
-
-	r := router.NewR(t, myControl, theirControl)
-
-	r.Log("Assert the tunnel between me and them works")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-
-	r.Log("Close the tunnel")
-	myControl.CloseTunnel(theirVpnIpNet[0].Addr(), false)
-	r.FlushAll()
-
-	waitStart := time.Now()
-	for {
-		myIndexes := len(myControl.GetHostmap().Indexes)
-		theirIndexes := len(theirControl.GetHostmap().Indexes)
-		if myIndexes == 0 && theirIndexes == 0 {
-			break
-		}
-
-		since := time.Since(waitStart)
-		r.Logf("my tunnels: %v; their tunnels: %v; duration: %v", myIndexes, theirIndexes, since)
-		if since > time.Second*6 {
-			t.Fatal("Tunnel should have been declared inactive after 2 seconds and before 6 seconds")
-		}
-
-		time.Sleep(1 * time.Second)
-		//r.FlushAll()
-	}
-
-	r.Logf("Happy path success, tunnels were dropped within %v", time.Since(waitStart))
-
-	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-	r.Log("Assert another tunnel between me and them works")
-	assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
-	hi := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
-	if hi == nil {
-		t.Fatal("There is no hostinfo for this tunnel")
-	}
-	myHi := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
-	if myHi == nil {
-		t.Fatal("There is no hostinfo for my tunnel")
-	}
-	r.Log("It does")
-
-	buf := make([]byte, 1024)
-	hdr := header.H{
-		Version:        1,
-		Type:           header.CloseTunnel,
-		Subtype:        0,
-		Reserved:       0,
-		RemoteIndex:    hi.RemoteIndex,
-		MessageCounter: 5,
-	}
-	out, err := hdr.Encode(buf)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	pkt := &udp.Packet{
-		To:   hi.CurrentRemote,
-		From: myHi.CurrentRemote,
-		Data: out,
-	}
-	r.InjectUDPPacket(myControl, theirControl, pkt)
-	r.Log("Injected bogus close tunnel. Let's see!")
-	waitStart = time.Now()
-	for {
-		myIndexes := len(myControl.GetHostmap().Indexes)
-		theirIndexes := len(theirControl.GetHostmap().Indexes)
-		if myIndexes == 0 {
-			t.Fatal("myIndexes should not be 0")
-		}
-		if theirIndexes == 0 {
-			t.Fatal("theirIndexes should not be 0, they should have rejected this bogus packet")
-		}
-
-		since := time.Since(waitStart)
-		r.Logf("my tunnels: %v; their tunnels: %v; duration: %v", myIndexes, theirIndexes, since)
-		if since > time.Second*4 {
-			t.Log("The tunnel would have been gone by now")
-			break
-		}
-
-		time.Sleep(1 * time.Second)
-		r.FlushAll()
-	}
-
-	myControl.Stop()
-	theirControl.Stop()
-}

examples/config.yml

@@ -13,11 +13,43 @@ pki:
   # disconnect_invalid is a toggle to force a client to be disconnected if the certificate is expired or invalid.
   #disconnect_invalid: true
 
-  # initiating_version controls which certificate version is used when initiating handshakes.
+  # default_version controls which certificate version is used in handshakes.
   # This setting only applies if both a v1 and a v2 certificate are configured, in which case it will default to `1`.
   # Once all hosts in the mesh are configured with both a v1 and v2 certificate then this should be changed to `2`.
   # After all hosts in the mesh are using a v2 certificate then v1 certificates are no longer needed.
-  # initiating_version: 1
+  # default_version: 1
+
+  # psk can be used to mask the contents of handshakes.
+  psk:
+    # `mode` defines how the pre shared keys can be used in a handshake.
+    # `accepting` (the default) will initiate handshakes using an empty key and will try to use any keys provided when
+    # receiving handshakes, including an empty key.
+    # `sending` will initiate handshakes with the first key provided and will try to use any keys provided when
+    # receiving handshakes, including an empty key.
+    # `enforced` will initiate handshakes with the first psk key provided and will try to use any keys provided when
+    # responding to handshakes. An empty key will not be allowed.
+    #
+    # To change a mesh from not using a psk to enforcing psk:
+    # 1. Leave `mode` as `accepting` and configure `psk.keys` to match on all nodes in the mesh and reload.
+    # 2. Change `mode` to `sending` on all nodes in the mesh and reload.
+    # 3. Change `mode` to `enforced` on all nodes in the mesh and reload.
+    #mode: accepting
+
+    # The keys provided are sent through hkdf to ensure the shared secret used in the noise protocol is the
+    # correct byte length.
+    #
+    # Only the first key is used for outbound handshakes but all keys provided will be tried in the order specified, on
+    # incoming handshakes. This is to allow for psk rotation.
+    #
+    # To rotate a primary key:
+    # 1. Put the new key in the 2nd slot on every node in the mesh and reload.
+    # 2. Move the key from the 2nd slot to the 1st slot, the old primary key is now in the 2nd slot, reload.
+    # 3. Remove the old primary key once it is no longer in use on every node in the mesh and reload.
+    #keys:
+    # - shared secret string, this one is used in all outbound handshakes # This is the primary key used when sending handshakes
+    # - this is a fallback key, received handshakes can use this
+    # - another fallback, received handshakes can use this one too
+    # - "\x68\x65\x6c\x6c\x6f\x20\x66\x72\x69\x65\x6e\x64\x73" # for raw bytes if you desire
 
 # The static host map defines a set of hosts with fixed IP addresses on the internet (or any network).
 # A host can have multiple fixed IP addresses defined here, and nebula will try each when establishing a tunnel.
@@ -144,10 +176,6 @@ listen:
   # valid values: always, never, private
   # This setting is reloadable.
   #send_recv_error: always
-  # Similar to send_recv_error, this option lets you configure if you want to accept "recv_error" packets from remote hosts.
-  # valid values: always, never, private
-  # This setting is reloadable.
-  #accept_recv_error: always
   # The so_sock option is a Linux-specific feature that allows all outgoing Nebula packets to be tagged with a specific identifier.
   # This tagging enables IP rule-based filtering. For example, it supports 0.0.0.0/0 unsafe_routes,
   # allowing for more precise routing decisions based on the packet tags. Default is 0 meaning no mark is set.
@@ -204,12 +232,6 @@ punchy:
   # Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access.
   #trusted_cas:
     #- "ssh public key string"
-  # sandbox_dir restricts file paths for profiling commands (start-cpu-profile, save-heap-profile,
-  # save-mutex-profile) to the specified directory. Relative paths will be resolved within this directory,
-  # and absolute paths outside of it will be rejected. Default is $TMP/nebula-debug.
-  # The directory is NOT automatically created.
-  # Overriding this to "" is the same as "/" and will allow overwriting any path on the host.
-  #sandbox_dir: /var/tmp/nebula-debug
 
 # EXPERIMENTAL: relay support for networks that can't establish direct connections.
 relay:
@@ -249,28 +271,7 @@ tun:
 
   # Unsafe routes allows you to route traffic over nebula to non-nebula nodes
   # Unsafe routes should be avoided unless you have hosts/services that cannot run nebula
-  # Supports weighted ECMP if you define a list of gateways, this can be used for load balancing or redundancy to hosts outside of nebula
+  # NOTE: The nebula certificate of the "via" node *MUST* have the "route" defined as a subnet in its certificate
-  # NOTES:
-  # * You will only see a single gateway in the routing table if you are not on linux
-  # * If a gateway is not reachable through the overlay another gateway will be selected to send the traffic through, ignoring weights
-  #
-  # unsafe_routes:
-  # # Multiple gateways without defining a weight defaults to a weight of 1, this will balance traffic equally between the three gateways
-  # - route: 192.168.87.0/24
-  #   via:
-  #     - gateway: 10.0.0.1
-  #     - gateway: 10.0.0.2
-  #     - gateway: 10.0.0.3
-  # # Multiple gateways with a weight, this will balance traffic accordingly
-  # - route: 192.168.87.0/24
-  #   via:
-  #     - gateway: 10.0.0.1
-  #       weight: 10
-  #     - gateway: 10.0.0.2
-  #       weight: 5
-  #
-  # NOTE: The nebula certificate of the "via" node(s) *MUST* have the "route" defined as a subnet in its certificate
-  # `via`: single node or list of gateways to use for this route
   # `mtu`: will default to tun mtu if this option is not specified
   # `metric`: will default to 0 if this option is not specified
   # `install`: will default to true, controls whether this route is installed in the systems routing table.
@@ -285,10 +286,6 @@ tun:
   # On linux only, set to true to manage unsafe routes directly on the system route table with gateway routes instead of
   # in nebula configuration files. Default false, not reloadable.
   #use_system_route_table: false
-  # Buffer size for reading routes updates. 0 means default system buffer size. (/proc/sys/net/core/rmem_default).
-  # If using massive routes updates, for example BGP, you may need to increase this value to avoid packet loss.
-  # SO_RCVBUFFORCE is used to avoid having to raise the system wide max
-  #use_system_route_table_buffer_size: 0
 
 # Configure logging level
 logging:
@@ -348,19 +345,6 @@ logging:
   # after receiving the response for lighthouse queries
   #trigger_buffer: 64
 
-# Tunnel manager settings
-#tunnels:
-  # drop_inactive controls whether inactive tunnels are maintained or dropped after the inactive_timeout period has
-  # elapsed.
-  # In general, it is a good idea to enable this setting. It will be enabled by default in a future release.
-  # This setting is reloadable
-  #drop_inactive: false
-
-  # inactivity_timeout controls how long a tunnel MUST NOT see any inbound or outbound traffic before being considered
-  # inactive and eligible to be dropped.
-  # This setting is reloadable
-  #inactivity_timeout: 10m
-
 # Nebula security group configuration
 firewall:
   # Action to take when a packet is not allowed by the firewall rules.
@@ -372,11 +356,11 @@ firewall:
   outbound_action: drop
   inbound_action: drop
 
-  # THIS FLAG IS DEPRECATED AND WILL BE REMOVED IN A FUTURE RELEASE. (Defaults to false.)
+  # Controls the default value for local_cidr. Default is true, will be deprecated after v1.9 and defaulted to false.
-  # This setting only affects nebula hosts exposing unsafe_routes. When set to false, each inbound rule must contain a
+  # This setting only affects nebula hosts with subnets encoded in their certificate. A nebula host acting as an
-  # `local_cidr` if the intention is to allow traffic to flow to an unsafe route. When set to true, every firewall rule
+  # unsafe router with `default_local_cidr_any: true` will expose their unsafe routes to every inbound rule regardless
-  # will apply to all configured unsafe_routes regardless of the actual destination of the packet, unless `local_cidr`
+  # of the actual destination for the packet. Setting this to false requires each inbound rule to contain a `local_cidr`
-  # is explicitly defined. This is usually not the desired behavior and should be avoided!
+  # if the intention is to allow traffic to flow to an unsafe route.
   #default_local_cidr_any: false
 
   conntrack:
@@ -388,16 +372,17 @@ firewall:
   # Rules are comprised of a protocol, port, and one or more of host, group, or CIDR
   # Logical evaluation is roughly: port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND (local cidr)
   # - port: Takes `0` or `any` as any, a single number `80`, a range `200-901`, or `fragment` to match second and further fragments of fragmented packets (since there is no port available).
+  #   code: same as port but makes more sense when talking about ICMP, TODO: this is not currently implemented in a way that works, use `any`
   #   proto: `any`, `tcp`, `udp`, or `icmp`
-  #          a port specification is ignored if proto is `icmp`
   #   host: `any` or a literal hostname, ie `test-host`
   #   group: `any` or a literal group name, ie `default-group`
   #   groups: Same as group but accepts a list of values. Multiple values are AND'd together and a certificate would have to contain all groups to pass
-  #   cidr: a remote CIDR, `0.0.0.0/0` is any ipv4 and `::/0` is any ipv6. `any` means any ip family and address.
+  #   cidr: a remote CIDR, `0.0.0.0/0` is any ipv4 and `::/0` is any ipv6.
-  #   local_cidr: a local CIDR, `0.0.0.0/0` is any ipv4 and `::/0` is any ipv6. `any` means any ip family and address.
+  #   local_cidr: a local CIDR, `0.0.0.0/0` is any ipv4 and `::/0` is any ipv6. This could be used to filter destinations when using unsafe_routes.
-  #     This can be used to filter destinations when using unsafe_routes.
+  #     If no unsafe networks are present in the certificate(s) or `default_local_cidr_any` is true then the default is any ipv4 or ipv6 network.
-  #     By default, this is set to only the VPN (overlay) networks assigned via the certificate networks field unless `default_local_cidr_any` is set to true.
+  #     Otherwise the default is any vpn network assigned to via the certificate.
-  #     If there are unsafe_routes present in this config file, `local_cidr` should be set appropriately for the intended us case.
+  #     `default_local_cidr_any` defaults to false and is deprecated, it will be removed in a future release.
+  #     If there are unsafe routes present its best to set `local_cidr` to whatever best fits the situation.
   #   ca_name: An issuing CA name
   #   ca_sha: An issuing CA shasum
 

examples/go_service/main.go

@@ -5,12 +5,8 @@ import (
 	"fmt"
 	"log"
 	"net"
-	"os"
 
-	"github.com/sirupsen/logrus"
-	"github.com/slackhq/nebula"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/overlay"
 	"github.com/slackhq/nebula/service"
 )
 
@@ -63,16 +59,7 @@ pki:
 	if err := cfg.LoadString(configStr); err != nil {
 		return err
 	}
-
+	svc, err := service.New(&cfg)
-	logger := logrus.New()
-	logger.Out = os.Stdout
-
-	ctrl, err := nebula.Main(&cfg, false, "custom-app", logger, overlay.NewUserDeviceFromConfig)
-	if err != nil {
-		return err
-	}
-
-	svc, err := service.New(ctrl)
 	if err != nil {
 		return err
 	}

firewall.go

@@ -8,7 +8,6 @@ import (
 	"hash/fnv"
 	"net/netip"
 	"reflect"
-	"slices"
 	"strconv"
 	"strings"
 	"sync"
@@ -23,7 +22,7 @@ import (
 )
 
 type FirewallInterface interface {
-	AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr string, caName string, caSha string) error
+	AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, addr, localAddr netip.Prefix, caName string, caSha string) error
 }
 
 type conn struct {
@@ -54,7 +53,7 @@ type Firewall struct {
 
 	// routableNetworks describes the vpn addresses as well as any unsafe networks issued to us in the certificate.
 	// The vpn addresses are a full bit match while the unsafe networks only match the prefix
-	routableNetworks *bart.Lite
+	routableNetworks *bart.Table[struct{}]
 
 	// assignedNetworks is a list of vpn networks assigned to us in the certificate.
 	assignedNetworks  []netip.Prefix
@@ -126,7 +125,7 @@ type firewallPort map[int32]*FirewallCA
 
 type firewallLocalCIDR struct {
 	Any       bool
-	LocalCIDR *bart.Lite
+	LocalCIDR *bart.Table[struct{}]
 }
 
 // NewFirewall creates a new Firewall object. A TimerWheel is created for you from the provided timeouts.
@@ -149,17 +148,17 @@ func NewFirewall(l *logrus.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.D
 		tmax = defaultTimeout
 	}
 
-	routableNetworks := new(bart.Lite)
+	routableNetworks := new(bart.Table[struct{}])
 	var assignedNetworks []netip.Prefix
 	for _, network := range c.Networks() {
 		nprefix := netip.PrefixFrom(network.Addr(), network.Addr().BitLen())
-		routableNetworks.Insert(nprefix)
+		routableNetworks.Insert(nprefix, struct{}{})
 		assignedNetworks = append(assignedNetworks, network)
 	}
 
 	hasUnsafeNetworks := false
 	for _, n := range c.UnsafeNetworks() {
-		routableNetworks.Insert(n)
+		routableNetworks.Insert(n, struct{}{})
 		hasUnsafeNetworks = true
 	}
 
@@ -230,7 +229,7 @@ func NewFirewallFromConfig(l *logrus.Logger, cs *CertState, c *config.C) (*Firew
 	case "drop":
 		fw.OutSendReject = false
 	default:
-		l.WithField("action", outboundAction).Warn("invalid firewall.outbound_action, defaulting to `drop`")
+		l.WithField("action", inboundAction).Warn("invalid firewall.outbound_action, defaulting to `drop`")
 		fw.OutSendReject = false
 	}
 
@@ -248,7 +247,32 @@ func NewFirewallFromConfig(l *logrus.Logger, cs *CertState, c *config.C) (*Firew
 }
 
 // AddRule properly creates the in memory rule structure for a firewall table.
-func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error {
+func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, ip, localIp netip.Prefix, caName string, caSha string) error {
+	// Under gomobile, stringing a nil pointer with fmt causes an abort in debug mode for iOS
+	// https://github.com/golang/go/issues/14131
+	sIp := ""
+	if ip.IsValid() {
+		sIp = ip.String()
+	}
+	lIp := ""
+	if localIp.IsValid() {
+		lIp = localIp.String()
+	}
+
+	// We need this rule string because we generate a hash. Removing this will break firewall reload.
+	ruleString := fmt.Sprintf(
+		"incoming: %v, proto: %v, startPort: %v, endPort: %v, groups: %v, host: %v, ip: %v, localIp: %v, caName: %v, caSha: %s",
+		incoming, proto, startPort, endPort, groups, host, sIp, lIp, caName, caSha,
+	)
+	f.rules += ruleString + "\n"
+
+	direction := "incoming"
+	if !incoming {
+		direction = "outgoing"
+	}
+	f.l.WithField("firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "ip": sIp, "localIp": lIp, "caName": caName, "caSha": caSha}).
+		Info("Firewall rule added")
+
 	var (
 		ft *FirewallTable
 		fp firewallPort
@@ -266,12 +290,6 @@ func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort
 	case firewall.ProtoUDP:
 		fp = ft.UDP
 	case firewall.ProtoICMP, firewall.ProtoICMPv6:
-		//ICMP traffic doesn't have ports, so we always coerce to "any", even if a value is provided
-		if startPort != firewall.PortAny {
-			f.l.WithField("startPort", startPort).Warn("ignoring port specification for ICMP firewall rule")
-		}
-		startPort = firewall.PortAny
-		endPort = firewall.PortAny
 		fp = ft.ICMP
 	case firewall.ProtoAny:
 		fp = ft.AnyProto
@@ -279,21 +297,7 @@ func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort
 		return fmt.Errorf("unknown protocol %v", proto)
 	}
 
-	// We need this rule string because we generate a hash. Removing this will break firewall reload.
+	return fp.addRule(f, startPort, endPort, groups, host, ip, localIp, caName, caSha)
-	ruleString := fmt.Sprintf(
-		"incoming: %v, proto: %v, startPort: %v, endPort: %v, groups: %v, host: %v, ip: %v, localIp: %v, caName: %v, caSha: %s",
-		incoming, proto, startPort, endPort, groups, host, cidr, localCidr, caName, caSha,
-	)
-	f.rules += ruleString + "\n"
-
-	direction := "incoming"
-	if !incoming {
-		direction = "outgoing"
-	}
-	f.l.WithField("firewallRule", m{"direction": direction, "proto": proto, "startPort": startPort, "endPort": endPort, "groups": groups, "host": host, "cidr": cidr, "localCidr": localCidr, "caName": caName, "caSha": caSha}).
-		Info("Firewall rule added")
-
-	return fp.addRule(f, startPort, endPort, groups, host, cidr, localCidr, caName, caSha)
 }
 
 // GetRuleHash returns a hash representation of all inbound and outbound rules
@@ -327,12 +331,13 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw
 		return nil
 	}
 
-	rs, ok := r.([]any)
+	rs, ok := r.([]interface{})
 	if !ok {
 		return fmt.Errorf("%s failed to parse, should be an array of rules", table)
 	}
 
 	for i, t := range rs {
+		var groups []string
 		r, err := convertRule(l, t, table, i)
 		if err != nil {
 			return fmt.Errorf("%s rule #%v; %s", table, i, err)
@@ -342,10 +347,23 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw
 			return fmt.Errorf("%s rule #%v; only one of port or code should be provided", table, i)
 		}
 
-		if r.Host == "" && len(r.Groups) == 0 && r.Cidr == "" && r.LocalCidr == "" && r.CAName == "" && r.CASha == "" {
+		if r.Host == "" && len(r.Groups) == 0 && r.Group == "" && r.Cidr == "" && r.LocalCidr == "" && r.CAName == "" && r.CASha == "" {
 			return fmt.Errorf("%s rule #%v; at least one of host, group, cidr, local_cidr, ca_name, or ca_sha must be provided", table, i)
 		}
 
+		if len(r.Groups) > 0 {
+			groups = r.Groups
+		}
+
+		if r.Group != "" {
+			// Check if we have both groups and group provided in the rule config
+			if len(groups) > 0 {
+				return fmt.Errorf("%s rule #%v; only one of group or groups should be defined, both provided", table, i)
+			}
+
+			groups = []string{r.Group}
+		}
+
 		var sPort, errPort string
 		if r.Code != "" {
 			errPort = "code"
@@ -355,51 +373,42 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw
 			sPort = r.Port
 		}
 
-		var proto uint8
+		startPort, endPort, err := parsePort(sPort)
-		var startPort, endPort int32
-		switch r.Proto {
-		case "any":
-			proto = firewall.ProtoAny
-			startPort, endPort, err = parsePort(sPort)
-		case "tcp":
-			proto = firewall.ProtoTCP
-			startPort, endPort, err = parsePort(sPort)
-		case "udp":
-			proto = firewall.ProtoUDP
-			startPort, endPort, err = parsePort(sPort)
-		case "icmp":
-			proto = firewall.ProtoICMP
-			startPort = firewall.PortAny
-			endPort = firewall.PortAny
-			if sPort != "" {
-				l.WithField("port", sPort).Warn("ignoring port specification for ICMP firewall rule")
-			}
-		default:
-			return fmt.Errorf("%s rule #%v; proto was not understood; `%s`", table, i, r.Proto)
-		}
 		if err != nil {
 			return fmt.Errorf("%s rule #%v; %s %s", table, i, errPort, err)
 		}
 
-		if r.Cidr != "" && r.Cidr != "any" {
+		var proto uint8
-			_, err = netip.ParsePrefix(r.Cidr)
+		switch r.Proto {
+		case "any":
+			proto = firewall.ProtoAny
+		case "tcp":
+			proto = firewall.ProtoTCP
+		case "udp":
+			proto = firewall.ProtoUDP
+		case "icmp":
+			proto = firewall.ProtoICMP
+		default:
+			return fmt.Errorf("%s rule #%v; proto was not understood; `%s`", table, i, r.Proto)
+		}
+
+		var cidr netip.Prefix
+		if r.Cidr != "" {
+			cidr, err = netip.ParsePrefix(r.Cidr)
 			if err != nil {
 				return fmt.Errorf("%s rule #%v; cidr did not parse; %s", table, i, err)
 			}
 		}
 
-		if r.LocalCidr != "" && r.LocalCidr != "any" {
+		var localCidr netip.Prefix
-			_, err = netip.ParsePrefix(r.LocalCidr)
+		if r.LocalCidr != "" {
+			localCidr, err = netip.ParsePrefix(r.LocalCidr)
 			if err != nil {
 				return fmt.Errorf("%s rule #%v; local_cidr did not parse; %s", table, i, err)
 			}
 		}
 
-		if warning := r.sanity(); warning != nil {
+		err = fw.AddRule(inbound, proto, startPort, endPort, groups, r.Host, cidr, localCidr, r.CAName, r.CASha)
-			l.Warnf("%s rule #%v; %s", table, i, warning)
-		}
-
-		err = fw.AddRule(inbound, proto, startPort, endPort, r.Groups, r.Host, r.Cidr, r.LocalCidr, r.CAName, r.CASha)
 		if err != nil {
 			return fmt.Errorf("%s rule #%v; `%s`", table, i, err)
 		}
@@ -408,10 +417,8 @@ func AddFirewallRulesFromConfig(l *logrus.Logger, inbound bool, c *config.C, fw
 	return nil
 }
 
-var ErrUnknownNetworkType = errors.New("unknown network type")
+var ErrInvalidRemoteIP = errors.New("remote IP is not in remote certificate subnets")
-var ErrPeerRejected = errors.New("remote address is not within a network that we handle")
+var ErrInvalidLocalIP = errors.New("local IP is not in list of handled local IPs")
-var ErrInvalidRemoteIP = errors.New("remote address is not in remote certificate networks")
-var ErrInvalidLocalIP = errors.New("local address is not in list of handled local addresses")
 var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
 
 // Drop returns an error if the packet should be dropped, explaining why. It
@@ -422,35 +429,24 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
 		return nil
 	}
 
-	// Make sure remote address matches nebula certificate, and determine how to treat it
+	// Make sure remote address matches nebula certificate
-	if h.networks == nil {
+	if h.networks != nil {
+		_, ok := h.networks.Lookup(fp.RemoteAddr)
+		if !ok {
+			f.metrics(incoming).droppedRemoteAddr.Inc(1)
+			return ErrInvalidRemoteIP
+		}
+	} else {
 		// Simple case: Certificate has one address and no unsafe networks
 		if h.vpnAddrs[0] != fp.RemoteAddr {
 			f.metrics(incoming).droppedRemoteAddr.Inc(1)
 			return ErrInvalidRemoteIP
 		}
-	} else {
-		nwType, ok := h.networks.Lookup(fp.RemoteAddr)
-		if !ok {
-			f.metrics(incoming).droppedRemoteAddr.Inc(1)
-			return ErrInvalidRemoteIP
-		}
-		switch nwType {
-		case NetworkTypeVPN:
-			break // nothing special
-		case NetworkTypeVPNPeer:
-			f.metrics(incoming).droppedRemoteAddr.Inc(1)
-			return ErrPeerRejected // reject for now, one day this may have different FW rules
-		case NetworkTypeUnsafe:
-			break // nothing special, one day this may have different FW rules
-		default:
-			f.metrics(incoming).droppedRemoteAddr.Inc(1)
-			return ErrUnknownNetworkType //should never happen
-		}
 	}
 
 	// Make sure we are supposed to be handling this local ip address
-	if !f.routableNetworks.Contains(fp.LocalAddr) {
+	_, ok := f.routableNetworks.Lookup(fp.LocalAddr)
+	if !ok {
 		f.metrics(incoming).droppedLocalAddr.Inc(1)
 		return ErrInvalidLocalIP
 	}
@@ -480,7 +476,7 @@ func (f *Firewall) metrics(incoming bool) firewallMetrics {
 	}
 }
 
-// Destroy cleans up any known cyclical references so the object can be freed by GC. This should be called if a new
+// Destroy cleans up any known cyclical references so the object can be free'd my GC. This should be called if a new
 // firewall object is created
 func (f *Firewall) Destroy() {
 	//TODO: clean references if/when needed
@@ -646,7 +642,7 @@ func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.CachedC
 	return false
 }
 
-func (fp firewallPort) addRule(f *Firewall, startPort int32, endPort int32, groups []string, host string, cidr, localCidr, caName string, caSha string) error {
+func (fp firewallPort) addRule(f *Firewall, startPort int32, endPort int32, groups []string, host string, ip, localIp netip.Prefix, caName string, caSha string) error {
 	if startPort > endPort {
 		return fmt.Errorf("start port was lower than end port")
 	}
@@ -659,7 +655,7 @@ func (fp firewallPort) addRule(f *Firewall, startPort int32, endPort int32, grou
 			}
 		}
 
-		if err := fp[i].addRule(f, groups, host, cidr, localCidr, caName, caSha); err != nil {
+		if err := fp[i].addRule(f, groups, host, ip, localIp, caName, caSha); err != nil {
 			return err
 		}
 	}
@@ -673,13 +669,6 @@ func (fp firewallPort) match(p firewall.Packet, incoming bool, c *cert.CachedCer
 		return false
 	}
 
-	// this branch is here to catch traffic from FirewallTable.Any.match and FirewallTable.ICMP.match
-	if p.Protocol == firewall.ProtoICMP || p.Protocol == firewall.ProtoICMPv6 {
-		// port numbers are re-used for connection tracking of ICMP,
-		// but we don't want to actually filter on them.
-		return fp[firewall.PortAny].match(p, c, caPool)
-	}
-
 	var port int32
 
 	if p.Fragment {
@@ -697,7 +686,7 @@ func (fp firewallPort) match(p firewall.Packet, incoming bool, c *cert.CachedCer
 	return fp[firewall.PortAny].match(p, c, caPool)
 }
 
-func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, cidr, localCidr, caName, caSha string) error {
+func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, ip, localIp netip.Prefix, caName, caSha string) error {
 	fr := func() *FirewallRule {
 		return &FirewallRule{
 			Hosts:  make(map[string]*firewallLocalCIDR),
@@ -711,14 +700,14 @@ func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, cidr, l
 			fc.Any = fr()
 		}
 
-		return fc.Any.addRule(f, groups, host, cidr, localCidr)
+		return fc.Any.addRule(f, groups, host, ip, localIp)
 	}
 
 	if caSha != "" {
 		if _, ok := fc.CAShas[caSha]; !ok {
 			fc.CAShas[caSha] = fr()
 		}
-		err := fc.CAShas[caSha].addRule(f, groups, host, cidr, localCidr)
+		err := fc.CAShas[caSha].addRule(f, groups, host, ip, localIp)
 		if err != nil {
 			return err
 		}
@@ -728,7 +717,7 @@ func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, cidr, l
 		if _, ok := fc.CANames[caName]; !ok {
 			fc.CANames[caName] = fr()
 		}
-		err := fc.CANames[caName].addRule(f, groups, host, cidr, localCidr)
+		err := fc.CANames[caName].addRule(f, groups, host, ip, localIp)
 		if err != nil {
 			return err
 		}
@@ -760,24 +749,24 @@ func (fc *FirewallCA) match(p firewall.Packet, c *cert.CachedCertificate, caPool
 	return fc.CANames[s.Certificate.Name()].match(p, c)
 }
 
-func (fr *FirewallRule) addRule(f *Firewall, groups []string, host, cidr, localCidr string) error {
+func (fr *FirewallRule) addRule(f *Firewall, groups []string, host string, ip, localCIDR netip.Prefix) error {
 	flc := func() *firewallLocalCIDR {
 		return &firewallLocalCIDR{
-			LocalCIDR: new(bart.Lite),
+			LocalCIDR: new(bart.Table[struct{}]),
 		}
 	}
 
-	if fr.isAny(groups, host, cidr) {
+	if fr.isAny(groups, host, ip) {
 		if fr.Any == nil {
 			fr.Any = flc()
 		}
 
-		return fr.Any.addRule(f, localCidr)
+		return fr.Any.addRule(f, localCIDR)
 	}
 
 	if len(groups) > 0 {
 		nlc := flc()
-		err := nlc.addRule(f, localCidr)
+		err := nlc.addRule(f, localCIDR)
 		if err != nil {
 			return err
 		}
@@ -793,46 +782,44 @@ func (fr *FirewallRule) addRule(f *Firewall, groups []string, host, cidr, localC
 		if nlc == nil {
 			nlc = flc()
 		}
-		err := nlc.addRule(f, localCidr)
+		err := nlc.addRule(f, localCIDR)
 		if err != nil {
 			return err
 		}
 		fr.Hosts[host] = nlc
 	}
 
-	if cidr != "" {
+	if ip.IsValid() {
-		c, err := netip.ParsePrefix(cidr)
+		nlc, _ := fr.CIDR.Get(ip)
-		if err != nil {
-			return err
-		}
-		nlc, _ := fr.CIDR.Get(c)
 		if nlc == nil {
 			nlc = flc()
 		}
-		err = nlc.addRule(f, localCidr)
+		err := nlc.addRule(f, localCIDR)
 		if err != nil {
 			return err
 		}
-		fr.CIDR.Insert(c, nlc)
+		fr.CIDR.Insert(ip, nlc)
 	}
 
 	return nil
 }
 
-func (fr *FirewallRule) isAny(groups []string, host string, cidr string) bool {
+func (fr *FirewallRule) isAny(groups []string, host string, ip netip.Prefix) bool {
-	if len(groups) == 0 && host == "" && cidr == "" {
+	if len(groups) == 0 && host == "" && !ip.IsValid() {
 		return true
 	}
 
-	if slices.Contains(groups, "any") {
+	for _, group := range groups {
-		return true
+		if group == "any" {
+			return true
+		}
 	}
 
 	if host == "any" {
 		return true
 	}
 
-	if cidr == "any" {
+	if ip.IsValid() && ip.Bits() == 0 {
 		return true
 	}
 
@@ -884,30 +871,24 @@ func (fr *FirewallRule) match(p firewall.Packet, c *cert.CachedCertificate) bool
 	return false
 }
 
-func (flc *firewallLocalCIDR) addRule(f *Firewall, localCidr string) error {
+func (flc *firewallLocalCIDR) addRule(f *Firewall, localIp netip.Prefix) error {
-	if localCidr == "any" {
+	if !localIp.IsValid() {
-		flc.Any = true
-		return nil
-	}
-
-	if localCidr == "" {
 		if !f.hasUnsafeNetworks || f.defaultLocalCIDRAny {
 			flc.Any = true
 			return nil
 		}
 
 		for _, network := range f.assignedNetworks {
-			flc.LocalCIDR.Insert(network)
+			flc.LocalCIDR.Insert(network, struct{}{})
 		}
 		return nil
 
+	} else if localIp.Bits() == 0 {
+		flc.Any = true
+		return nil
 	}
 
-	c, err := netip.ParsePrefix(localCidr)
+	flc.LocalCIDR.Insert(localIp, struct{}{})
-	if err != nil {
-		return err
-	}
-	flc.LocalCIDR.Insert(c)
 	return nil
 }
 
@@ -920,7 +901,8 @@ func (flc *firewallLocalCIDR) match(p firewall.Packet, c *cert.CachedCertificate
 		return true
 	}
 
-	return flc.LocalCIDR.Contains(p.LocalAddr)
+	_, ok := flc.LocalCIDR.Lookup(p.LocalAddr)
+	return ok
 }
 
 type rule struct {
@@ -928,6 +910,7 @@ type rule struct {
 	Code      string
 	Proto     string
 	Host      string
+	Group     string
 	Groups    []string
 	Cidr      string
 	LocalCidr string
@@ -935,15 +918,15 @@ type rule struct {
 	CASha     string
 }
 
-func convertRule(l *logrus.Logger, p any, table string, i int) (rule, error) {
+func convertRule(l *logrus.Logger, p interface{}, table string, i int) (rule, error) {
 	r := rule{}
 
-	m, ok := p.(map[string]any)
+	m, ok := p.(map[interface{}]interface{})
 	if !ok {
 		return r, errors.New("could not parse rule")
 	}
 
-	toString := func(k string, m map[string]any) string {
+	toString := func(k string, m map[interface{}]interface{}) string {
 		v, ok := m[k]
 		if !ok {
 			return ""
@@ -961,7 +944,7 @@ func convertRule(l *logrus.Logger, p any, table string, i int) (rule, error) {
 	r.CASha = toString("ca_sha", m)
 
 	// Make sure group isn't an array
-	if v, ok := m["group"].([]any); ok {
+	if v, ok := m["group"].([]interface{}); ok {
 		if len(v) > 1 {
 			return r, errors.New("group should contain a single value, an array with more than one entry was provided")
 		}
@@ -969,8 +952,7 @@ func convertRule(l *logrus.Logger, p any, table string, i int) (rule, error) {
 		l.Warnf("%s rule #%v; group was an array with a single value, converting to simple value", table, i)
 		m["group"] = v[0]
 	}
-
+	r.Group = toString("group", m)
-	singleGroup := toString("group", m)
 
 	if rg, ok := m["groups"]; ok {
 		switch reflect.TypeOf(rg).Kind() {
@@ -987,105 +969,52 @@ func convertRule(l *logrus.Logger, p any, table string, i int) (rule, error) {
 		}
 	}
 
-	//flatten group vs groups
-	if singleGroup != "" {
-		// Check if we have both groups and group provided in the rule config
-		if len(r.Groups) > 0 {
-			return r, fmt.Errorf("only one of group or groups should be defined, both provided")
-		}
-		r.Groups = []string{singleGroup}
-	}
-
 	return r, nil
 }
 
-// sanity returns an error if the rule would be evaluated in a way that would short-circuit a configured check on a wildcard value
+func parsePort(s string) (startPort, endPort int32, err error) {
-// rules are evaluated as "port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND local_cidr"
-func (r *rule) sanity() error {
-	//port, proto, local_cidr are AND, no need to check here
-	//ca_sha and ca_name don't have a wildcard value, no need to check here
-	groupsEmpty := len(r.Groups) == 0
-	hostEmpty := r.Host == ""
-	cidrEmpty := r.Cidr == ""
-
-	if (groupsEmpty && hostEmpty && cidrEmpty) == true {
-		return nil //no content!
-	}
-
-	groupsHasAny := slices.Contains(r.Groups, "any")
-	if groupsHasAny && len(r.Groups) > 1 {
-		return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the other groups specified", r.Groups)
-	}
-
-	if r.Host == "any" {
-		if !groupsEmpty {
-			return fmt.Errorf("groups specified as %s, but host=any will match any host, regardless of groups", r.Groups)
-		}
-
-		if !cidrEmpty {
-			return fmt.Errorf("cidr specified as %s, but host=any will match any host, regardless of cidr", r.Cidr)
-		}
-	}
-
-	if groupsHasAny {
-		if !hostEmpty && r.Host != "any" {
-			return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified host %s", r.Groups, r.Host)
-		}
-		if !cidrEmpty {
-			return fmt.Errorf("groups spec [%s] contains the group '\"any\". This rule will ignore the specified cidr %s", r.Groups, r.Cidr)
-		}
-	}
-
-	if r.Code != "" {
-		return fmt.Errorf("code specified as [%s]. Support for 'code' will be dropped in a future release, as it has never been functional", r.Code)
-	}
-
-	//todo alert on cidr-any
-
-	return nil
-}
-
-func parsePort(s string) (int32, int32, error) {
-	var err error
-	const notAPort int32 = -2
 	if s == "any" {
-		return firewall.PortAny, firewall.PortAny, nil
+		startPort = firewall.PortAny
-	}
+		endPort = firewall.PortAny
-	if s == "fragment" {
+
-		return firewall.PortFragment, firewall.PortFragment, nil
+	} else if s == "fragment" {
-	}
+		startPort = firewall.PortFragment
-	if !strings.Contains(s, `-`) {
+		endPort = firewall.PortFragment
+
+	} else if strings.Contains(s, `-`) {
+		sPorts := strings.SplitN(s, `-`, 2)
+		sPorts[0] = strings.Trim(sPorts[0], " ")
+		sPorts[1] = strings.Trim(sPorts[1], " ")
+
+		if len(sPorts) != 2 || sPorts[0] == "" || sPorts[1] == "" {
+			return 0, 0, fmt.Errorf("appears to be a range but could not be parsed; `%s`", s)
+		}
+
+		rStartPort, err := strconv.Atoi(sPorts[0])
+		if err != nil {
+			return 0, 0, fmt.Errorf("beginning range was not a number; `%s`", sPorts[0])
+		}
+
+		rEndPort, err := strconv.Atoi(sPorts[1])
+		if err != nil {
+			return 0, 0, fmt.Errorf("ending range was not a number; `%s`", sPorts[1])
+		}
+
+		startPort = int32(rStartPort)
+		endPort = int32(rEndPort)
+
+		if startPort == firewall.PortAny {
+			endPort = firewall.PortAny
+		}
+
+	} else {
 		rPort, err := strconv.Atoi(s)
 		if err != nil {
-			return notAPort, notAPort, fmt.Errorf("was not a number; `%s`", s)
+			return 0, 0, fmt.Errorf("was not a number; `%s`", s)
 		}
-		return int32(rPort), int32(rPort), nil
+		startPort = int32(rPort)
+		endPort = startPort
 	}
 
-	sPorts := strings.SplitN(s, `-`, 2)
+	return
-	for i := range sPorts {
-		sPorts[i] = strings.Trim(sPorts[i], " ")
-	}
-	if len(sPorts) != 2 || sPorts[0] == "" || sPorts[1] == "" {
-		return notAPort, notAPort, fmt.Errorf("appears to be a range but could not be parsed; `%s`", s)
-	}
-
-	rStartPort, err := strconv.Atoi(sPorts[0])
-	if err != nil {
-		return notAPort, notAPort, fmt.Errorf("beginning range was not a number; `%s`", sPorts[0])
-	}
-
-	rEndPort, err := strconv.Atoi(sPorts[1])
-	if err != nil {
-		return notAPort, notAPort, fmt.Errorf("ending range was not a number; `%s`", sPorts[1])
-	}
-
-	startPort := int32(rStartPort)
-	endPort := int32(rEndPort)
-
-	if startPort == firewall.PortAny {
-		endPort = firewall.PortAny
-	}
-
-	return startPort, endPort, nil
 }

firewall/packet.go

@@ -6,7 +6,7 @@ import (
 	"net/netip"
 )
 
-type m = map[string]any
+type m map[string]interface{}
 
 const (
 	ProtoAny    = 0 // When we want to handle HOPOPT (0) we can change this, if ever
@@ -22,10 +22,7 @@ const (
 type Packet struct {
 	LocalAddr  netip.Addr
 	RemoteAddr netip.Addr
-	// LocalPort is the destination port for incoming traffic, or the source port for outgoing. Zero for ICMP.
+	LocalPort  uint16
-	LocalPort uint16
-	// RemotePort is the source port for incoming traffic, or the destination port for outgoing.
-	// For ICMP, it's the "identifier". This is only used for connection tracking, actual firewall rules will not filter on ICMP identifier
 	RemotePort uint16
 	Protocol   uint8
 	Fragment   bool
@@ -49,8 +46,6 @@ func (fp Packet) MarshalJSON() ([]byte, error) {
 		proto = "tcp"
 	case ProtoICMP:
 		proto = "icmp"
-	case ProtoICMPv6:
-		proto = "icmpv6"
 	case ProtoUDP:
 		proto = "udp"
 	default:

go.mod

@@ -1,40 +1,40 @@
 module github.com/slackhq/nebula
 
-go 1.25.0
+go 1.23.6
+
+toolchain go1.23.7
 
 require (
-	dario.cat/mergo v1.0.2
+	dario.cat/mergo v1.0.1
-	filippo.io/bigmod v0.1.0
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
 	github.com/armon/go-radix v1.0.0
 	github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432
 	github.com/flynn/noise v1.1.0
-	github.com/gaissmai/bart v0.26.0
+	github.com/gaissmai/bart v0.18.1
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/gopacket v1.1.19
-	github.com/kardianos/service v1.2.4
+	github.com/kardianos/service v1.2.2
-	github.com/miekg/dns v1.1.72
+	github.com/miekg/dns v1.1.63
-	github.com/miekg/pkcs11 v1.1.2
+	github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b
 	github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f
-	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/client_golang v1.21.1
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
-	github.com/sirupsen/logrus v1.9.4
+	github.com/sirupsen/logrus v1.9.3
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
-	github.com/stretchr/testify v1.11.1
+	github.com/stretchr/testify v1.10.0
-	github.com/vishvananda/netlink v1.3.1
+	github.com/vishvananda/netlink v1.3.0
-	go.yaml.in/yaml/v3 v3.0.4
+	golang.org/x/crypto v0.36.0
-	golang.org/x/crypto v0.50.0
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
-	golang.org/x/net v0.52.0
+	golang.org/x/net v0.37.0
-	golang.org/x/sync v0.20.0
+	golang.org/x/sync v0.12.0
-	golang.org/x/sys v0.43.0
+	golang.org/x/sys v0.31.0
-	golang.org/x/term v0.42.0
+	golang.org/x/term v0.30.0
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
 	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
-	golang.zx2c4.com/wireguard/windows v0.6.1
+	golang.zx2c4.com/wireguard/windows v0.5.3
-	google.golang.org/protobuf v1.36.11
+	google.golang.org/protobuf v1.36.5
-	gopkg.in/yaml.v3 v3.0.1
+	gopkg.in/yaml.v2 v2.4.0
 	gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
 )
 
@@ -43,14 +43,15 @@ require (
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/google/btree v1.1.2 // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
-	github.com/prometheus/common v0.66.1 // indirect
+	github.com/prometheus/common v0.62.0 // indirect
-	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
-	github.com/vishvananda/netns v0.0.5 // indirect
+	github.com/vishvananda/netns v0.0.4 // indirect
-	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	golang.org/x/mod v0.18.0 // indirect
-	golang.org/x/mod v0.34.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.43.0 // indirect
+	golang.org/x/tools v0.22.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,8 +1,6 @@
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
+dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=
-dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
+dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
-filippo.io/bigmod v0.1.0 h1:UNzDk7y9ADKST+axd9skUpBQeW7fG2KrTZyOE4uGQy8=
-filippo.io/bigmod v0.1.0/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -26,8 +24,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg=
 github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
-github.com/gaissmai/bart v0.26.0 h1:xOZ57E9hJLBiQaSyeZa9wgWhGuzfGACgqp4BE77OkO0=
+github.com/gaissmai/bart v0.18.1 h1:bX2j560JC1MJpoEDevBGvXL5OZ1mkls320Vl8Igb5QQ=
-github.com/gaissmai/bart v0.26.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
+github.com/gaissmai/bart v0.18.1/go.mod h1:JJzMAhNF5Rjo4SF4jWBrANuJfqY+FvsFhW7t1UZJ+XY=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -55,8 +53,8 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
@@ -66,12 +64,12 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/kardianos/service v1.2.4 h1:XNlGtZOYNx2u91urOdg/Kfmc+gfmuIo1Dd3rEi2OgBk=
+github.com/kardianos/service v1.2.2 h1:ZvePhAHfvo0A7Mftk/tEzqEZ7Q4lgnR8sGz4xu1YX60=
-github.com/kardianos/service v1.2.4/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc=
+github.com/kardianos/service v1.2.2/go.mod h1:CIMRFEJVL+0DS1a3Nx06NaMn4Dz63Ng6O7dl0qH0zVM=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
-github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -85,10 +83,10 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
+github.com/miekg/dns v1.1.63 h1:8M5aAw6OMZfFXTT7K5V0Eu5YiiL8l7nUAkyN6C9YwaY=
-github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
+github.com/miekg/dns v1.1.63/go.mod h1:6NGHfjhpmr5lt3XPLuyfDJi5AXbNIPM9PY6H6sF1Nfs=
-github.com/miekg/pkcs11 v1.1.2 h1:/VxmeAX5qU6Q3EwafypogwWbYryHFmF2RpkJmw3m4MQ=
+github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b h1:J/AzCvg5z0Hn1rqZUJjpbzALUmkKX0Zwbc/i4fw7Sfk=
-github.com/miekg/pkcs11 v1.1.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
+github.com/miekg/pkcs11 v1.1.2-0.20231115102856-9078ad6b9d4b/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -108,24 +106,24 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.23.2 h1:Je96obch5RDVy3FDMndoUsjAhG5Edi49h0RJWRi/o0o=
+github.com/prometheus/client_golang v1.21.1 h1:DOvXXTqVzvkIewV/CDPFdejpMCGeMcbGCQ8YOmu+Ibk=
-github.com/prometheus/client_golang v1.23.2/go.mod h1:Tb1a6LWHB3/SPIzCoaDXI4I8UHKeFTEQ1YCr+0Gyqmg=
+github.com/prometheus/client_golang v1.21.1/go.mod h1:U9NM32ykUErtVBxdvD3zfi+EuFkkaBvMb09mIfe0Zgg=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
+github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E=
-github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
+github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.66.1 h1:h5E0h5/Y8niHc5DlaLlWLArTQI7tMrsfQjHV+d9ZoGs=
+github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io=
-github.com/prometheus/common v0.66.1/go.mod h1:gcaUsgf3KfRSwHY4dIMXLPV0K/Wg1oZ8+SbZk/HH/dA=
+github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg=
+github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
-github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is=
+github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
@@ -133,8 +131,8 @@ github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncj
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
-github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
-github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
 github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 h1:pnnLyeX7o/5aX8qUQ69P/mLojDqwda8hFOCBTmP/6hw=
@@ -145,35 +143,29 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
+github.com/vishvananda/netlink v1.3.0 h1:X7l42GfcV4S6E4vHTsw48qbrV+9PVojNfIhZcwQdrZk=
-github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
+github.com/vishvananda/netlink v1.3.0/go.mod h1:i6NetklAujEcC6fK0JPjT8qSwWyO0HLn4UKG+hGqeJs=
-github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
+github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8=
-github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
+github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
-go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
-go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
-go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
-go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
-go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
+golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34=
-golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
+golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
+golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=
-golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
+golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -184,8 +176,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
+golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c=
-golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
+golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -193,8 +185,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
+golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw=
-golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -205,16 +197,18 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201015000850-e3ed0017c211/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
+golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
-golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
+golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y=
-golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
+golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -225,8 +219,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
+golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA=
-golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
+golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -235,8 +229,8 @@ golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeu
 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
 golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo=
 golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
-golang.zx2c4.com/wireguard/windows v0.6.1 h1:XMaKojH1Hs/raMrmnir4n35nTvzvWj7NmSYzHn2F4qU=
+golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE=
-golang.zx2c4.com/wireguard/windows v0.6.1/go.mod h1:04aqInu5GYuTFvMuDw/rKBAF7mHrltW/3rekpfbbZDM=
+golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
@@ -245,8 +239,8 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
+google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
-google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
+google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -257,6 +251,8 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

handshake_ix.go

@@ -1,8 +1,8 @@
 package nebula
 
 import (
-	"bytes"
 	"net/netip"
+	"slices"
 	"time"
 
 	"github.com/flynn/noise"
@@ -23,17 +23,13 @@ func ixHandshakeStage0(f *Interface, hh *HandshakeHostInfo) bool {
 		return false
 	}
 
+	// If we're connecting to a v6 address we must use a v2 cert
 	cs := f.pki.getCertState()
-	v := cs.initiatingVersion
+	v := cs.defaultVersion
-	if hh.initiatingVersionOverride != cert.VersionPre1 {
+	for _, a := range hh.hostinfo.vpnAddrs {
-		v = hh.initiatingVersionOverride
+		if a.Is6() {
-	} else if v < cert.Version2 {
+			v = cert.Version2
-		// If we're connecting to a v6 address we should encourage use of a V2 cert
+			break
-		for _, a := range hh.hostinfo.vpnAddrs {
-			if a.Is6() {
-				v = cert.Version2
-				break
-			}
 		}
 	}
 
@@ -52,10 +48,9 @@ func ixHandshakeStage0(f *Interface, hh *HandshakeHostInfo) bool {
 			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).
 			WithField("certVersion", v).
 			Error("Unable to handshake with host because no certificate handshake bytes is available")
-		return false
 	}
 
-	ci, err := NewConnectionState(f.l, cs, crt, true, noise.HandshakeIX)
+	ci, err := NewConnectionState(f.l, cs, crt, true, noise.HandshakeIX, cs.psk.primary)
 	if err != nil {
 		f.l.WithError(err).WithField("vpnAddrs", hh.hostinfo.vpnAddrs).
 			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).
@@ -76,8 +71,7 @@ func ixHandshakeStage0(f *Interface, hh *HandshakeHostInfo) bool {
 
 	hsBytes, err := hs.Marshal()
 	if err != nil {
-		f.l.WithError(err).WithField("vpnAddrs", hh.hostinfo.vpnAddrs).
+		f.l.WithError(err).WithField("vpnAddrs", hh.hostinfo.vpnAddrs).WithField("certVersion", v).
-			WithField("certVersion", v).
 			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).Error("Failed to marshal handshake message")
 		return false
 	}
@@ -100,48 +94,66 @@ func ixHandshakeStage0(f *Interface, hh *HandshakeHostInfo) bool {
 	return true
 }
 
-func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H) {
+func ixHandshakeStage1(f *Interface, addr netip.AddrPort, via *ViaSender, packet []byte, h *header.H) {
 	cs := f.pki.getCertState()
 	crt := cs.GetDefaultCertificate()
 	if crt == nil {
-		f.l.WithField("from", via).
+		f.l.WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 0, "style": "ix_psk0"}).
-			WithField("certVersion", cs.initiatingVersion).
+			WithField("certVersion", cs.defaultVersion).
 			Error("Unable to handshake with host because no certificate is available")
-		return
 	}
 
-	ci, err := NewConnectionState(f.l, cs, crt, false, noise.HandshakeIX)
+	var (
-	if err != nil {
+		err error
-		f.l.WithError(err).WithField("from", via).
+		ci  *ConnectionState
+		msg []byte
+	)
+
+	hs := &NebulaHandshake{}
+
+	for _, psk := range cs.psk.keys {
+		ci, err = NewConnectionState(f.l, cs, crt, false, noise.HandshakeIX, psk)
+		if err != nil {
+			//TODO: should be bother logging this, if we have multiple psks and the error is unrelated it will be verbose.
+			f.l.WithError(err).WithField("udpAddr", addr).
+				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+				Error("Failed to create connection state")
+			continue
+		}
+
+		msg, _, _, err = ci.H.ReadMessage(nil, packet[header.Len:])
+		if err != nil {
+			// Calls to ReadMessage with an incorrect psk should fail, try the next one if we have one
+			continue
+		}
+
+		// Sometimes ReadMessage returns fine with a nil psk even if the handshake is using a psk, ensure our protobuf
+		// comes out clean as well
+		err = hs.Unmarshal(msg)
+		if err == nil {
+			// There was no error, we can continue with this handshake
+			break
+		}
+
+		// The unmarshal failed, try the next psk if we have one
+	}
+
+	// We finished with an error, log it and get out
+	if err != nil || hs.Details == nil {
+		// We aren't logging the error here because we can't be sure of the failure when using psk
+		f.l.WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
-			Error("Failed to create connection state")
+			Error("Was unable to decrypt the handshake")
 		return
 	}
 
 	// Mark packet 1 as seen so it doesn't show up as missed
 	ci.window.Update(f.l, 1)
 
-	msg, _, _, err := ci.H.ReadMessage(nil, packet[header.Len:])
-	if err != nil {
-		f.l.WithError(err).WithField("from", via).
-			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
-			Error("Failed to call noise.ReadMessage")
-		return
-	}
-
-	hs := &NebulaHandshake{}
-	err = hs.Unmarshal(msg)
-	if err != nil || hs.Details == nil {
-		f.l.WithError(err).WithField("from", via).
-			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
-			Error("Failed unmarshal handshake message")
-		return
-	}
-
 	rc, err := cert.Recombine(cert.Version(hs.Details.CertVersion), hs.Details.Cert, ci.H.PeerStatic(), ci.Curve())
 	if err != nil {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
 			Info("Handshake did not contain a certificate")
 		return
@@ -149,12 +161,12 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 
 	remoteCert, err := f.pki.GetCAPool().VerifyCertificate(time.Now(), rc)
 	if err != nil {
-		fp, fperr := rc.Fingerprint()
+		fp, err := rc.Fingerprint()
-		if fperr != nil {
+		if err != nil {
 			fp = "<error generating certificate fingerprint>"
 		}
 
-		e := f.l.WithError(err).WithField("from", via).
+		e := f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
 			WithField("certVpnNetworks", rc.Networks()).
 			WithField("certFingerprint", fp)
@@ -167,76 +179,77 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 		return
 	}
 
-	if !bytes.Equal(remoteCert.Certificate.PublicKey(), ci.H.PeerStatic()) {
-		f.l.WithField("from", via).
-			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
-			WithField("cert", remoteCert).Info("public key mismatch between certificate and handshake")
-		return
-	}
-
 	if remoteCert.Certificate.Version() != ci.myCert.Version() {
 		// We started off using the wrong certificate version, lets see if we can match the version that was sent to us
-		myCertOtherVersion := cs.getCertificate(remoteCert.Certificate.Version())
+		rc := cs.getCertificate(remoteCert.Certificate.Version())
-		if myCertOtherVersion == nil {
+		if rc == nil {
-			if f.l.Level >= logrus.DebugLevel {
+			f.l.WithError(err).WithField("udpAddr", addr).
-				f.l.WithError(err).WithFields(m{
+				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).WithField("cert", remoteCert).
-					"from":      via,
+				Info("Unable to handshake with host due to missing certificate version")
-					"handshake": m{"stage": 1, "style": "ix_psk0"},
+			return
-					"cert":      remoteCert,
-				}).Debug("Might be unable to handshake with host due to missing certificate version")
-			}
-		} else {
-			// Record the certificate we are actually using
-			ci.myCert = myCertOtherVersion
 		}
+
+		// Record the certificate we are actually using
+		ci.myCert = rc
 	}
 
 	if len(remoteCert.Certificate.Networks()) == 0 {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("cert", remoteCert).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
 			Info("No networks in certificate")
 		return
 	}
 
+	var vpnAddrs []netip.Addr
+	var filteredNetworks []netip.Prefix
 	certName := remoteCert.Certificate.Name()
-	certVersion := remoteCert.Certificate.Version()
 	fingerprint := remoteCert.Fingerprint
 	issuer := remoteCert.Certificate.Issuer()
-	vpnNetworks := remoteCert.Certificate.Networks()
 
-	anyVpnAddrsInCommon := false
+	for _, network := range remoteCert.Certificate.Networks() {
-	vpnAddrs := make([]netip.Addr, len(vpnNetworks))
+		vpnAddr := network.Addr()
-	for i, network := range vpnNetworks {
+		_, found := f.myVpnAddrsTable.Lookup(vpnAddr)
-		if f.myVpnAddrsTable.Contains(network.Addr()) {
+		if found {
-			f.l.WithField("vpnNetworks", vpnNetworks).WithField("from", via).
+			f.l.WithField("vpnAddr", vpnAddr).WithField("udpAddr", addr).
 				WithField("certName", certName).
-				WithField("certVersion", certVersion).
 				WithField("fingerprint", fingerprint).
 				WithField("issuer", issuer).
 				WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Refusing to handshake with myself")
 			return
 		}
-		vpnAddrs[i] = network.Addr()
+
-		if f.myVpnNetworksTable.Contains(network.Addr()) {
+		// vpnAddrs outside our vpn networks are of no use to us, filter them out
-			anyVpnAddrsInCommon = true
+		if _, ok := f.myVpnNetworksTable.Lookup(vpnAddr); !ok {
+			continue
 		}
+
+		filteredNetworks = append(filteredNetworks, network)
+		vpnAddrs = append(vpnAddrs, vpnAddr)
 	}
 
-	if !via.IsRelayed {
+	if len(vpnAddrs) == 0 {
+		f.l.WithError(err).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("No usable vpn addresses from host, refusing handshake")
+		return
+	}
+
+	if addr.IsValid() {
+		// addr can be invalid when the tunnel is being relayed.
 		// We only want to apply the remote allow list for direct tunnels here
-		if !f.lightHouse.GetRemoteAllowList().AllowAll(vpnAddrs, via.UdpAddr.Addr()) {
+		if !f.lightHouse.GetRemoteAllowList().AllowAll(vpnAddrs, addr.Addr()) {
-			f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake")
-				Debug("lighthouse.remote_allow_list denied incoming handshake")
 			return
 		}
 	}
 
 	myIndex, err := generateIndex(f.l)
 	if err != nil {
-		f.l.WithError(err).WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+		f.l.WithError(err).WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
 			WithField("certName", certName).
-			WithField("certVersion", certVersion).
 			WithField("fingerprint", fingerprint).
 			WithField("issuer", issuer).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to generate index")
@@ -251,36 +264,30 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 		HandshakePacket:   make(map[uint8][]byte, 0),
 		lastHandshakeTime: hs.Details.Time,
 		relayState: RelayState{
-			relays:         nil,
+			relays:         map[netip.Addr]struct{}{},
 			relayForByAddr: map[netip.Addr]*Relay{},
 			relayForByIdx:  map[uint32]*Relay{},
 		},
 	}
 
-	msgRxL := f.l.WithFields(m{
+	f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
-		"vpnAddrs":       vpnAddrs,
+		WithField("certName", certName).
-		"from":           via,
+		WithField("fingerprint", fingerprint).
-		"certName":       certName,
+		WithField("issuer", issuer).
-		"certVersion":    certVersion,
+		WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
-		"fingerprint":    fingerprint,
+		WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
-		"issuer":         issuer,
+		Info("Handshake message received")
-		"initiatorIndex": hs.Details.InitiatorIndex,
-		"responderIndex": hs.Details.ResponderIndex,
-		"remoteIndex":    h.RemoteIndex,
-		"handshake":      m{"stage": 1, "style": "ix_psk0"},
-	})
-
-	if anyVpnAddrsInCommon {
-		msgRxL.Info("Handshake message received")
-	} else {
-		//todo warn if not lighthouse or relay?
-		msgRxL.Info("Handshake message received, but no vpnNetworks in common.")
-	}
 
 	hs.Details.ResponderIndex = myIndex
 	hs.Details.Cert = cs.getHandshakeBytes(ci.myCert.Version())
 	if hs.Details.Cert == nil {
-		msgRxL.WithField("myCertVersion", ci.myCert.Version()).
+		f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+			WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
+			WithField("certVersion", ci.myCert.Version()).
 			Error("Unable to handshake with host because no certificate handshake bytes is available")
 		return
 	}
@@ -291,9 +298,8 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 
 	hsBytes, err := hs.Marshal()
 	if err != nil {
-		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("certName", certName).
-			WithField("certVersion", certVersion).
 			WithField("fingerprint", fingerprint).
 			WithField("issuer", issuer).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to marshal handshake message")
@@ -303,17 +309,15 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 	nh := header.Encode(make([]byte, header.Len), header.Version, header.Handshake, header.HandshakeIXPSK0, hs.Details.InitiatorIndex, 2)
 	msg, dKey, eKey, err := ci.H.WriteMessage(nh, hsBytes)
 	if err != nil {
-		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("certName", certName).
-			WithField("certVersion", certVersion).
 			WithField("fingerprint", fingerprint).
 			WithField("issuer", issuer).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Failed to call noise.WriteMessage")
 		return
 	} else if dKey == nil || eKey == nil {
-		f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("certName", certName).
-			WithField("certVersion", certVersion).
 			WithField("fingerprint", fingerprint).
 			WithField("issuer", issuer).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).Error("Noise did not arrive at a key")
@@ -337,17 +341,15 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 	ci.eKey = NewNebulaCipherState(eKey)
 
 	hostinfo.remotes = f.lightHouse.QueryCache(vpnAddrs)
-	if !via.IsRelayed {
+	hostinfo.SetRemote(addr)
-		hostinfo.SetRemote(via.UdpAddr)
+	hostinfo.buildNetworks(filteredNetworks, remoteCert.Certificate.UnsafeNetworks())
-	}
-	hostinfo.buildNetworks(f.myVpnNetworksTable, remoteCert.Certificate)
 
 	existing, err := f.handshakeManager.CheckAndComplete(hostinfo, 0, f)
 	if err != nil {
 		switch err {
 		case ErrAlreadySeen:
 			// Update remote if preferred
-			if existing.SetRemoteIfPreferred(f.hostMap, via) {
+			if existing.SetRemoteIfPreferred(f.hostMap, addr) {
 				// Send a test packet to ensure the other side has also switched to
 				// the preferred remote
 				f.SendMessageToVpnAddr(header.Test, header.TestRequest, vpnAddrs[0], []byte(""), make([]byte, 12, 12), make([]byte, mtu))
@@ -355,21 +357,21 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 
 			msg = existing.HandshakePacket[2]
 			f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1)
-			if !via.IsRelayed {
+			if addr.IsValid() {
-				err := f.outside.WriteTo(msg, via.UdpAddr)
+				err := f.outside.WriteTo(msg, addr)
 				if err != nil {
-					f.l.WithField("vpnAddrs", existing.vpnAddrs).WithField("from", via).
+					f.l.WithField("vpnAddrs", existing.vpnAddrs).WithField("udpAddr", addr).
 						WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
 						WithError(err).Error("Failed to send handshake message")
 				} else {
-					f.l.WithField("vpnAddrs", existing.vpnAddrs).WithField("from", via).
+					f.l.WithField("vpnAddrs", existing.vpnAddrs).WithField("udpAddr", addr).
 						WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("cached", true).
 						Info("Handshake message sent")
 				}
 				return
 			} else {
-				if via.relay == nil {
+				if via == nil {
-					f.l.Error("Handshake send failed: both addr and via.relay are nil.")
+					f.l.Error("Handshake send failed: both addr and via are nil.")
 					return
 				}
 				hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
@@ -381,9 +383,8 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 			}
 		case ErrExistingHostInfo:
 			// This means there was an existing tunnel and this handshake was older than the one we are currently based on
-			f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
 				WithField("certName", certName).
-				WithField("certVersion", certVersion).
 				WithField("oldHandshakeTime", existing.lastHandshakeTime).
 				WithField("newHandshakeTime", hostinfo.lastHandshakeTime).
 				WithField("fingerprint", fingerprint).
@@ -397,9 +398,8 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 			return
 		case ErrLocalIndexCollision:
 			// This means we failed to insert because of collision on localIndexId. Just let the next handshake packet retry
-			f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
 				WithField("certName", certName).
-				WithField("certVersion", certVersion).
 				WithField("fingerprint", fingerprint).
 				WithField("issuer", issuer).
 				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
@@ -410,9 +410,8 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 		default:
 			// Shouldn't happen, but just in case someone adds a new error type to CheckAndComplete
 			// And we forget to update it here
-			f.l.WithError(err).WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+			f.l.WithError(err).WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
 				WithField("certName", certName).
-				WithField("certVersion", certVersion).
 				WithField("fingerprint", fingerprint).
 				WithField("issuer", issuer).
 				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
@@ -424,23 +423,28 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 
 	// Do the send
 	f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1)
-	if !via.IsRelayed {
+	if addr.IsValid() {
-		err = f.outside.WriteTo(msg, via.UdpAddr)
+		err = f.outside.WriteTo(msg, addr)
-		log := f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
-			WithField("certName", certName).
-			WithField("certVersion", certVersion).
-			WithField("fingerprint", fingerprint).
-			WithField("issuer", issuer).
-			WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
-			WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"})
 		if err != nil {
-			log.WithError(err).Error("Failed to send handshake")
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
+				WithField("certName", certName).
+				WithField("fingerprint", fingerprint).
+				WithField("issuer", issuer).
+				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+				WithError(err).Error("Failed to send handshake")
 		} else {
-			log.Info("Handshake message sent")
+			f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
+				WithField("certName", certName).
+				WithField("fingerprint", fingerprint).
+				WithField("issuer", issuer).
+				WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
+				WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+				Info("Handshake message sent")
 		}
 	} else {
-		if via.relay == nil {
+		if via == nil {
-			f.l.Error("Handshake send failed: both addr and via.relay are nil.")
+			f.l.Error("Handshake send failed: both addr and via are nil.")
 			return
 		}
 		hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
@@ -450,7 +454,6 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 		f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false)
 		f.l.WithField("vpnAddrs", vpnAddrs).WithField("relay", via.relayHI.vpnAddrs[0]).
 			WithField("certName", certName).
-			WithField("certVersion", certVersion).
 			WithField("fingerprint", fingerprint).
 			WithField("issuer", issuer).
 			WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
@@ -458,14 +461,14 @@ func ixHandshakeStage1(f *Interface, via ViaSender, packet []byte, h *header.H)
 			Info("Handshake message sent")
 	}
 
-	f.connectionManager.AddTrafficWatch(hostinfo)
+	f.connectionManager.AddTrafficWatch(hostinfo.localIndexId)
 
-	hostinfo.remotes.RefreshFromHandshake(vpnAddrs)
+	hostinfo.remotes.ResetBlockedRemotes()
 
 	return
 }
 
-func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packet []byte, h *header.H) bool {
+func ixHandshakeStage2(f *Interface, addr netip.AddrPort, via *ViaSender, hh *HandshakeHostInfo, packet []byte, h *header.H) bool {
 	if hh == nil {
 		// Nothing here to tear down, got a bogus stage 2 packet
 		return true
@@ -475,10 +478,10 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 	defer hh.Unlock()
 
 	hostinfo := hh.hostinfo
-	if !via.IsRelayed {
+	if addr.IsValid() {
 		// The vpnAddr we know about is the one we tried to handshake with, use it to apply the remote allow list.
-		if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, via.UdpAddr.Addr()) {
+		if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, addr.Addr()) {
-			f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).Debug("lighthouse.remote_allow_list denied incoming handshake")
+			f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake")
 			return false
 		}
 	}
@@ -486,7 +489,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 	ci := hostinfo.ConnectionState
 	msg, eKey, dKey, err := ci.H.ReadMessage(nil, packet[header.Len:])
 	if err != nil {
-		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).WithField("header", h).
 			Error("Failed to call noise.ReadMessage")
 
@@ -495,7 +498,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 		// near future
 		return false
 	} else if dKey == nil || eKey == nil {
-		f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
 			Error("Noise did not arrive at a key")
 
@@ -507,7 +510,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 	hs := &NebulaHandshake{}
 	err = hs.Unmarshal(msg)
 	if err != nil || hs.Details == nil {
-		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("from", via).
+		f.l.WithError(err).WithField("vpnAddrs", hostinfo.vpnAddrs).WithField("udpAddr", addr).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).Error("Failed unmarshal handshake message")
 
 		// The handshake state machine is complete, if things break now there is no chance to recover. Tear down and start again
@@ -516,7 +519,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 
 	rc, err := cert.Recombine(cert.Version(hs.Details.CertVersion), hs.Details.Cert, ci.H.PeerStatic(), ci.Curve())
 	if err != nil {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("vpnAddrs", hostinfo.vpnAddrs).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
 			Info("Handshake did not contain a certificate")
@@ -530,7 +533,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 			fp = "<error generating certificate fingerprint>"
 		}
 
-		e := f.l.WithError(err).WithField("from", via).
+		e := f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("vpnAddrs", hostinfo.vpnAddrs).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
 			WithField("certFingerprint", fp).
@@ -543,15 +546,9 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 		e.Info("Invalid certificate from host")
 		return true
 	}
-	if !bytes.Equal(remoteCert.Certificate.PublicKey(), ci.H.PeerStatic()) {
-		f.l.WithField("from", via).
-			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
-			WithField("cert", remoteCert).Info("public key mismatch between certificate and handshake")
-		return true
-	}
 
 	if len(remoteCert.Certificate.Networks()) == 0 {
-		f.l.WithError(err).WithField("from", via).
+		f.l.WithError(err).WithField("udpAddr", addr).
 			WithField("vpnAddrs", hostinfo.vpnAddrs).
 			WithField("cert", remoteCert).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
@@ -561,7 +558,6 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 
 	vpnNetworks := remoteCert.Certificate.Networks()
 	certName := remoteCert.Certificate.Name()
-	certVersion := remoteCert.Certificate.Version()
 	fingerprint := remoteCert.Fingerprint
 	issuer := remoteCert.Certificate.Issuer()
 
@@ -574,32 +570,38 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 	ci.eKey = NewNebulaCipherState(eKey)
 
 	// Make sure the current udpAddr being used is set for responding
-	if !via.IsRelayed {
+	if addr.IsValid() {
-		hostinfo.SetRemote(via.UdpAddr)
+		hostinfo.SetRemote(addr)
 	} else {
 		hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
 	}
 
-	correctHostResponded := false
+	var vpnAddrs []netip.Addr
-	anyVpnAddrsInCommon := false
+	var filteredNetworks []netip.Prefix
-	vpnAddrs := make([]netip.Addr, len(vpnNetworks))
+	for _, network := range vpnNetworks {
-	for i, network := range vpnNetworks {
+		// vpnAddrs outside our vpn networks are of no use to us, filter them out
-		vpnAddrs[i] = network.Addr()
+		vpnAddr := network.Addr()
-		if f.myVpnNetworksTable.Contains(network.Addr()) {
+		if _, ok := f.myVpnNetworksTable.Lookup(vpnAddr); !ok {
-			anyVpnAddrsInCommon = true
+			continue
-		}
-		if hostinfo.vpnAddrs[0] == network.Addr() {
-			// todo is it more correct to see if any of hostinfo.vpnAddrs are in the cert? it should have len==1, but one day it might not?
-			correctHostResponded = true
 		}
+
+		filteredNetworks = append(filteredNetworks, network)
+		vpnAddrs = append(vpnAddrs, vpnAddr)
+	}
+
+	if len(vpnAddrs) == 0 {
+		f.l.WithError(err).WithField("udpAddr", addr).
+			WithField("certName", certName).
+			WithField("fingerprint", fingerprint).
+			WithField("issuer", issuer).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).Error("No usable vpn addresses from host, refusing handshake")
+		return true
 	}
 
 	// Ensure the right host responded
-	if !correctHostResponded {
+	if !slices.Contains(vpnAddrs, hostinfo.vpnAddrs[0]) {
 		f.l.WithField("intendedVpnAddrs", hostinfo.vpnAddrs).WithField("haveVpnNetworks", vpnNetworks).
-			WithField("from", via).
+			WithField("udpAddr", addr).WithField("certName", certName).
-			WithField("certName", certName).
-			WithField("certVersion", certVersion).
 			WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
 			Info("Incorrect host responded to handshake")
 
@@ -607,11 +609,10 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 		f.handshakeManager.DeleteHostInfo(hostinfo)
 
 		// Create a new hostinfo/handshake for the intended vpn ip
-		//TODO is hostinfo.vpnAddrs[0] always the address to use?
 		f.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], func(newHH *HandshakeHostInfo) {
 			// Block the current used address
 			newHH.hostinfo.remotes = hostinfo.remotes
-			newHH.hostinfo.remotes.BlockRemote(via)
+			newHH.hostinfo.remotes.BlockRemote(addr)
 
 			f.l.WithField("blockedUdpAddrs", newHH.hostinfo.remotes.CopyBlockedRemotes()).
 				WithField("vpnNetworks", vpnNetworks).
@@ -634,29 +635,23 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 	ci.window.Update(f.l, 2)
 
 	duration := time.Since(hh.startTime).Nanoseconds()
-	msgRxL := f.l.WithField("vpnAddrs", vpnAddrs).WithField("from", via).
+	f.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", addr).
 		WithField("certName", certName).
-		WithField("certVersion", certVersion).
 		WithField("fingerprint", fingerprint).
 		WithField("issuer", issuer).
 		WithField("initiatorIndex", hs.Details.InitiatorIndex).WithField("responderIndex", hs.Details.ResponderIndex).
 		WithField("remoteIndex", h.RemoteIndex).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
 		WithField("durationNs", duration).
-		WithField("sentCachedPackets", len(hh.packetStore))
+		WithField("sentCachedPackets", len(hh.packetStore)).
-	if anyVpnAddrsInCommon {
+		Info("Handshake message received")
-		msgRxL.Info("Handshake message received")
-	} else {
-		//todo warn if not lighthouse or relay?
-		msgRxL.Info("Handshake message received, but no vpnNetworks in common.")
-	}
 
 	// Build up the radix for the firewall if we have subnets in the cert
 	hostinfo.vpnAddrs = vpnAddrs
-	hostinfo.buildNetworks(f.myVpnNetworksTable, remoteCert.Certificate)
+	hostinfo.buildNetworks(filteredNetworks, remoteCert.Certificate.UnsafeNetworks())
 
 	// Complete our handshake and update metrics, this will replace any existing tunnels for the vpnAddrs here
 	f.handshakeManager.Complete(hostinfo, f)
-	f.connectionManager.AddTrafficWatch(hostinfo)
+	f.connectionManager.AddTrafficWatch(hostinfo.localIndexId)
 
 	if f.l.Level >= logrus.DebugLevel {
 		hostinfo.logger(f.l).Debugf("Sending %d stored packets", len(hh.packetStore))
@@ -671,7 +666,7 @@ func ixHandshakeStage2(f *Interface, via ViaSender, hh *HandshakeHostInfo, packe
 		f.cachedPacketMetrics.sent.Inc(int64(len(hh.packetStore)))
 	}
 
-	hostinfo.remotes.RefreshFromHandshake(vpnAddrs)
+	hostinfo.remotes.ResetBlockedRemotes()
 	f.metricHandshakes.Update(duration)
 
 	return false

handshake_manager.go

@@ -68,12 +68,11 @@ type HandshakeManager struct {
 type HandshakeHostInfo struct {
 	sync.Mutex
 
-	startTime                 time.Time        // Time that we first started trying with this handshake
+	startTime   time.Time        // Time that we first started trying with this handshake
-	ready                     bool             // Is the handshake ready
+	ready       bool             // Is the handshake ready
-	initiatingVersionOverride cert.Version     // Should we use a non-default cert version for this handshake?
+	counter     int64            // How many attempts have we made so far
-	counter                   int64            // How many attempts have we made so far
+	lastRemotes []netip.AddrPort // Remotes that we sent to during the previous attempt
-	lastRemotes               []netip.AddrPort // Remotes that we sent to during the previous attempt
+	packetStore []*cachedPacket  // A set of packets to be transmitted once the handshake completes
-	packetStore               []*cachedPacket  // A set of packets to be transmitted once the handshake completes
 
 	hostinfo *HostInfo
 }
@@ -136,11 +135,11 @@ func (hm *HandshakeManager) Run(ctx context.Context) {
 	}
 }
 
-func (hm *HandshakeManager) HandleIncoming(via ViaSender, packet []byte, h *header.H) {
+func (hm *HandshakeManager) HandleIncoming(addr netip.AddrPort, via *ViaSender, packet []byte, h *header.H) {
 	// First remote allow list check before we know the vpnIp
-	if !via.IsRelayed {
+	if addr.IsValid() {
-		if !hm.lightHouse.GetRemoteAllowList().AllowUnknownVpnAddr(via.UdpAddr.Addr()) {
+		if !hm.lightHouse.GetRemoteAllowList().AllowUnknownVpnAddr(addr.Addr()) {
-			hm.l.WithField("from", via).Debug("lighthouse.remote_allow_list denied incoming handshake")
+			hm.l.WithField("udpAddr", addr).Debug("lighthouse.remote_allow_list denied incoming handshake")
 			return
 		}
 	}
@@ -149,11 +148,11 @@ func (hm *HandshakeManager) HandleIncoming(via ViaSender, packet []byte, h *head
 	case header.HandshakeIXPSK0:
 		switch h.MessageCounter {
 		case 1:
-			ixHandshakeStage1(hm.f, via, packet, h)
+			ixHandshakeStage1(hm.f, addr, via, packet, h)
 
 		case 2:
 			newHostinfo := hm.queryIndex(h.RemoteIndex)
-			tearDown := ixHandshakeStage2(hm.f, via, newHostinfo, packet, h)
+			tearDown := ixHandshakeStage2(hm.f, addr, via, newHostinfo, packet, h)
 			if tearDown && newHostinfo != nil {
 				hm.DeleteHostInfo(newHostinfo.hostinfo)
 			}
@@ -269,13 +268,14 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 		hostinfo.logger(hm.l).WithField("relays", hostinfo.remotes.relays).Info("Attempt to relay through hosts")
 		// Send a RelayRequest to all known Relay IP's
 		for _, relay := range hostinfo.remotes.relays {
-			// Don't relay through the host I'm trying to connect to
+			// Don't relay to myself
 			if relay == vpnIp {
 				continue
 			}
 
-			// Don't relay to myself
+			// Don't relay through the host I'm trying to connect to
-			if hm.f.myVpnAddrsTable.Contains(relay) {
+			_, found := hm.f.myVpnAddrsTable.Lookup(relay)
+			if found {
 				continue
 			}
 
@@ -451,7 +451,7 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han
 		vpnAddrs:        []netip.Addr{vpnAddr},
 		HandshakePacket: make(map[uint8][]byte, 0),
 		relayState: RelayState{
-			relays:         nil,
+			relays:         map[netip.Addr]struct{}{},
 			relayForByAddr: map[netip.Addr]*Relay{},
 			relayForByIdx:  map[uint32]*Relay{},
 		},
@@ -590,7 +590,7 @@ func (hm *HandshakeManager) allocateIndex(hh *HandshakeHostInfo) error {
 	hm.Lock()
 	defer hm.Unlock()
 
-	for range 32 {
+	for i := 0; i < 32; i++ {
 		index, err := generateIndex(hm.l)
 		if err != nil {
 			return err

handshake_manager_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/slackhq/nebula/test"
 	"github.com/slackhq/nebula/udp"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func Test_NewHandshakeManagerVpnIp(t *testing.T) {
@@ -23,11 +24,15 @@ func Test_NewHandshakeManagerVpnIp(t *testing.T) {
 
 	lh := newTestLighthouse()
 
+	psk, err := NewPsk(PskAccepting, nil)
+	require.NoError(t, err)
+
 	cs := &CertState{
-		initiatingVersion: cert.Version1,
+		defaultVersion:   cert.Version1,
-		privateKey:        []byte{},
+		privateKey:       []byte{},
-		v1Cert:            &dummyCert{version: cert.Version1},
+		v1Cert:           &dummyCert{version: cert.Version1},
-		v1HandshakeBytes:  []byte{},
+		v1HandshakeBytes: []byte{},
+		psk:              psk,
 	}
 
 	blah := NewHandshakeManager(l, mainHM, lh, &udp.NoopConn{}, defaultHandshakeConfig)
@@ -98,5 +103,5 @@ func (mw *mockEncWriter) GetHostInfo(_ netip.Addr) *HostInfo {
 }
 
 func (mw *mockEncWriter) GetCertState() *CertState {
-	return &CertState{initiatingVersion: cert.Version2}
+	return &CertState{defaultVersion: cert.Version2}
 }

header/header.go

@@ -19,7 +19,7 @@ import (
 // |-----------------------------------------------------------------------|
 // |                               payload...                              |
 
-type m = map[string]any
+type m map[string]interface{}
 
 const (
 	Version uint8 = 1

hostmap.go

@@ -1,12 +1,9 @@
 package nebula
 
 import (
-	"encoding/json"
 	"errors"
-	"fmt"
 	"net"
 	"net/netip"
-	"slices"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -19,10 +16,12 @@ import (
 	"github.com/slackhq/nebula/header"
 )
 
+// const ProbeLen = 100
 const defaultPromoteEvery = 1000       // Count of packets sent before we try moving a tunnel to a preferred underlay ip address
 const defaultReQueryEvery = 5000       // Count of packets sent before re-querying a hostinfo to the lighthouse
 const defaultReQueryWait = time.Minute // Minimum amount of seconds to wait before re-querying a hostinfo the lighthouse. Evaluated every ReQueryEvery
 const MaxRemotes = 10
+const maxRecvError = 4
 
 // MaxHostInfosPerVpnIp is the max number of hostinfos we will track for a given vpn ip
 // 5 allows for an initial handshake and each host pair re-handshaking twice
@@ -69,7 +68,7 @@ type HostMap struct {
 type RelayState struct {
 	sync.RWMutex
 
-	relays []netip.Addr // Ordered set of VpnAddrs of Hosts to use as relays to access this peer
+	relays map[netip.Addr]struct{} // Set of vpnAddr's of Hosts to use as relays to access this peer
 	// For data race avoidance, the contents of a *Relay are treated immutably. To update a *Relay, copy the existing data,
 	// modify what needs to be updated, and store the new modified copy in the relayForByIp and relayForByIdx maps (with
 	// the RelayState Lock held)
@@ -80,12 +79,7 @@ type RelayState struct {
 func (rs *RelayState) DeleteRelay(ip netip.Addr) {
 	rs.Lock()
 	defer rs.Unlock()
-	for idx, val := range rs.relays {
+	delete(rs.relays, ip)
-		if val == ip {
-			rs.relays = append(rs.relays[:idx], rs.relays[idx+1:]...)
-			return
-		}
-	}
 }
 
 func (rs *RelayState) UpdateRelayForByIpState(vpnIp netip.Addr, state int) {
@@ -130,16 +124,16 @@ func (rs *RelayState) GetRelayForByAddr(addr netip.Addr) (*Relay, bool) {
 func (rs *RelayState) InsertRelayTo(ip netip.Addr) {
 	rs.Lock()
 	defer rs.Unlock()
-	if !slices.Contains(rs.relays, ip) {
+	rs.relays[ip] = struct{}{}
-		rs.relays = append(rs.relays, ip)
-	}
 }
 
 func (rs *RelayState) CopyRelayIps() []netip.Addr {
-	ret := make([]netip.Addr, len(rs.relays))
 	rs.RLock()
 	defer rs.RUnlock()
-	copy(ret, rs.relays)
+	ret := make([]netip.Addr, 0, len(rs.relays))
+	for ip := range rs.relays {
+		ret = append(ret, ip)
+	}
 	return ret
 }
 
@@ -214,18 +208,6 @@ func (rs *RelayState) InsertRelay(ip netip.Addr, idx uint32, r *Relay) {
 	rs.relayForByIdx[idx] = r
 }
 
-type NetworkType uint8
-
-const (
-	NetworkTypeUnknown NetworkType = iota
-	// NetworkTypeVPN is a network that overlaps one or more of the vpnNetworks in our certificate
-	NetworkTypeVPN
-	// NetworkTypeVPNPeer is a network that does not overlap one of our networks
-	NetworkTypeVPNPeer
-	// NetworkTypeUnsafe is a network from Certificate.UnsafeNetworks()
-	NetworkTypeUnsafe
-)
-
 type HostInfo struct {
 	remote          netip.AddrPort
 	remotes         *RemoteList
@@ -237,10 +219,11 @@ type HostInfo struct {
 	// vpnAddrs is a list of vpn addresses assigned to this host that are within our own vpn networks
 	// The host may have other vpn addresses that are outside our
 	// vpn networks but were removed because they are not usable
-	vpnAddrs []netip.Addr
+	vpnAddrs  []netip.Addr
+	recvError atomic.Uint32
 
-	// networks is a combination of specific vpn addresses (not prefixes!) and full unsafe networks assigned to this host.
+	// networks are both all vpn and unsafe networks assigned to this host
-	networks   *bart.Table[NetworkType]
+	networks   *bart.Table[struct{}]
 	relayState RelayState
 
 	// HandshakePacket records the packets used to create this hostinfo
@@ -267,36 +250,12 @@ type HostInfo struct {
 	// Used to track other hostinfos for this vpn ip since only 1 can be primary
 	// Synchronised via hostmap lock and not the hostinfo lock.
 	next, prev *HostInfo
-
-	//TODO: in, out, and others might benefit from being an atomic.Int32. We could collapse connectionManager pendingDeletion, relayUsed, and in/out into this 1 thing
-	in, out, pendingDeletion atomic.Bool
-
-	// lastUsed tracks the last time ConnectionManager checked the tunnel and it was in use.
-	// This value will be behind against actual tunnel utilization in the hot path.
-	// This should only be used by the ConnectionManagers ticker routine.
-	lastUsed time.Time
 }
 
 type ViaSender struct {
-	UdpAddr   netip.AddrPort
 	relayHI   *HostInfo // relayHI is the host info object of the relay
 	remoteIdx uint32    // remoteIdx is the index included in the header of the received packet
 	relay     *Relay    // relay contains the rest of the relay information, including the PeerIP of the host trying to communicate with us.
-	IsRelayed bool      // IsRelayed is true if the packet was sent through a relay
-}
-
-func (v ViaSender) String() string {
-	if v.IsRelayed {
-		return fmt.Sprintf("%s (relayed)", v.UdpAddr)
-	}
-	return v.UdpAddr.String()
-}
-
-func (v ViaSender) MarshalJSON() ([]byte, error) {
-	if v.IsRelayed {
-		return json.Marshal(m{"relay": v.UdpAddr})
-	}
-	return json.Marshal(m{"direct": v.UdpAddr})
 }
 
 type cachedPacket struct {
@@ -712,7 +671,6 @@ func (i *HostInfo) GetCert() *cert.CachedCertificate {
 	return nil
 }
 
-// TODO: Maybe use ViaSender here?
 func (i *HostInfo) SetRemote(remote netip.AddrPort) {
 	// We copy here because we likely got this remote from a source that reuses the object
 	if i.remote != remote {
@@ -723,14 +681,14 @@ func (i *HostInfo) SetRemote(remote netip.AddrPort) {
 
 // SetRemoteIfPreferred returns true if the remote was changed. The lastRoam
 // time on the HostInfo will also be updated.
-func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, via ViaSender) bool {
+func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, newRemote netip.AddrPort) bool {
-	if via.IsRelayed {
+	if !newRemote.IsValid() {
+		// relays have nil udp Addrs
 		return false
 	}
-
 	currentRemote := i.remote
 	if !currentRemote.IsValid() {
-		i.SetRemote(via.UdpAddr)
+		i.SetRemote(newRemote)
 		return true
 	}
 
@@ -743,7 +701,7 @@ func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, via ViaSender) bool {
 			return false
 		}
 
-		if l.Contains(via.UdpAddr.Addr()) {
+		if l.Contains(newRemote.Addr()) {
 			newIsPreferred = true
 		}
 	}
@@ -753,7 +711,7 @@ func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, via ViaSender) bool {
 		i.lastRoam = time.Now()
 		i.lastRoamRemote = currentRemote
 
-		i.SetRemote(via.UdpAddr)
+		i.SetRemote(newRemote)
 
 		return true
 	}
@@ -761,26 +719,26 @@ func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, via ViaSender) bool {
 	return false
 }
 
-// buildNetworks fills in the networks field of HostInfo. It accepts a cert.Certificate so you never ever mix the network types up.
+func (i *HostInfo) RecvErrorExceeded() bool {
-func (i *HostInfo) buildNetworks(myVpnNetworksTable *bart.Lite, c cert.Certificate) {
+	if i.recvError.Add(1) >= maxRecvError {
-	if len(c.Networks()) == 1 && len(c.UnsafeNetworks()) == 0 {
+		return true
-		if myVpnNetworksTable.Contains(c.Networks()[0].Addr()) {
+	}
-			return // Simple case, no BART needed
+	return true
-		}
+}
+
+func (i *HostInfo) buildNetworks(networks, unsafeNetworks []netip.Prefix) {
+	if len(networks) == 1 && len(unsafeNetworks) == 0 {
+		// Simple case, no CIDRTree needed
+		return
 	}
 
-	i.networks = new(bart.Table[NetworkType])
+	i.networks = new(bart.Table[struct{}])
-	for _, network := range c.Networks() {
+	for _, network := range networks {
-		nprefix := netip.PrefixFrom(network.Addr(), network.Addr().BitLen())
+		i.networks.Insert(network, struct{}{})
-		if myVpnNetworksTable.Contains(network.Addr()) {
-			i.networks.Insert(nprefix, NetworkTypeVPN)
-		} else {
-			i.networks.Insert(nprefix, NetworkTypeVPNPeer)
-		}
 	}
 
-	for _, network := range c.UnsafeNetworks() {
+	for _, network := range unsafeNetworks {
-		i.networks.Insert(network, NetworkTypeUnsafe)
+		i.networks.Insert(network, struct{}{})
 	}
 }
 

hostmap_test.go

@@ -7,7 +7,6 @@ import (
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestHostMap_MakePrimary(t *testing.T) {
@@ -211,36 +210,8 @@ func TestHostMap_reload(t *testing.T) {
 	assert.Empty(t, hm.GetPreferredRanges())
 
 	c.ReloadConfigString("preferred_ranges: [1.1.1.0/24, 10.1.1.0/24]")
-	assert.Equal(t, []string{"1.1.1.0/24", "10.1.1.0/24"}, toS(hm.GetPreferredRanges()))
+	assert.EqualValues(t, []string{"1.1.1.0/24", "10.1.1.0/24"}, toS(hm.GetPreferredRanges()))
 
 	c.ReloadConfigString("preferred_ranges: [1.1.1.1/32]")
-	assert.Equal(t, []string{"1.1.1.1/32"}, toS(hm.GetPreferredRanges()))
+	assert.EqualValues(t, []string{"1.1.1.1/32"}, toS(hm.GetPreferredRanges()))
-}
-
-func TestHostMap_RelayState(t *testing.T) {
-	h1 := &HostInfo{vpnAddrs: []netip.Addr{netip.MustParseAddr("0.0.0.1")}, localIndexId: 1}
-	a1 := netip.MustParseAddr("::1")
-	a2 := netip.MustParseAddr("2001::1")
-
-	h1.relayState.InsertRelayTo(a1)
-	assert.Equal(t, []netip.Addr{a1}, h1.relayState.relays)
-	h1.relayState.InsertRelayTo(a2)
-	assert.Equal(t, []netip.Addr{a1, a2}, h1.relayState.relays)
-	// Ensure that the first relay added is the first one returned in the copy
-	currentRelays := h1.relayState.CopyRelayIps()
-	require.Len(t, currentRelays, 2)
-	assert.Equal(t, a1, currentRelays[0])
-
-	// Deleting the last one in the list works ok
-	h1.relayState.DeleteRelay(a2)
-	assert.Equal(t, []netip.Addr{a1}, h1.relayState.relays)
-
-	// Deleting an element not in the list works ok
-	h1.relayState.DeleteRelay(a2)
-	assert.Equal(t, []netip.Addr{a1}, h1.relayState.relays)
-
-	// Deleting the only element in the list works ok
-	h1.relayState.DeleteRelay(a1)
-	assert.Equal(t, []netip.Addr{}, h1.relayState.relays)
-
 }

hostmap_tester.go

@@ -1,4 +1,5 @@
 //go:build e2e_testing
+// +build e2e_testing
 
 package nebula
 

inside.go

@@ -8,10 +8,9 @@ import (
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/iputil"
 	"github.com/slackhq/nebula/noiseutil"
-	"github.com/slackhq/nebula/routing"
 )
 
-func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb []byte, batch *sendBatch, rejectBuf []byte, q int, localCache firewall.ConntrackCache) {
+func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache) {
 	err := newPacket(packet, false, fwPacket)
 	if err != nil {
 		if f.l.Level >= logrus.DebugLevel {
@@ -22,18 +21,20 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 
 	// Ignore local broadcast packets
 	if f.dropLocalBroadcast {
-		if f.myBroadcastAddrsTable.Contains(fwPacket.RemoteAddr) {
+		_, found := f.myBroadcastAddrsTable.Lookup(fwPacket.RemoteAddr)
+		if found {
 			return
 		}
 	}
 
-	if f.myVpnAddrsTable.Contains(fwPacket.RemoteAddr) {
+	_, found := f.myVpnAddrsTable.Lookup(fwPacket.RemoteAddr)
+	if found {
 		// Immediately forward packets from self to self.
 		// This should only happen on Darwin-based and FreeBSD hosts, which
 		// routes packets from the Nebula addr to the Nebula addr through the Nebula
 		// TUN device.
 		if immediatelyForwardToSelf {
-			_, err := f.readers[q].WriteReject(packet)
+			_, err := f.readers[q].Write(packet)
 			if err != nil {
 				f.l.WithError(err).Error("Failed to forward to tun")
 			}
@@ -48,12 +49,12 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 		return
 	}
 
-	hostinfo, ready := f.getOrHandshakeConsiderRouting(fwPacket, func(hh *HandshakeHostInfo) {
+	hostinfo, ready := f.getOrHandshake(fwPacket.RemoteAddr, func(hh *HandshakeHostInfo) {
 		hh.cachePacket(f.l, header.Message, 0, packet, f.sendMessageNow, f.cachedPacketMetrics)
 	})
 
 	if hostinfo == nil {
-		f.rejectInside(packet, rejectBuf, q)
+		f.rejectInside(packet, out, q)
 		if f.l.Level >= logrus.DebugLevel {
 			f.l.WithField("vpnAddr", fwPacket.RemoteAddr).
 				WithField("fwPacket", fwPacket).
@@ -68,10 +69,10 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 
 	dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason == nil {
-		f.sendInsideMessage(hostinfo, packet, nb, batch, rejectBuf, q)
+		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
 
 	} else {
-		f.rejectInside(packet, rejectBuf, q)
+		f.rejectInside(packet, out, q)
 		if f.l.Level >= logrus.DebugLevel {
 			hostinfo.logger(f.l).
 				WithField("fwPacket", fwPacket).
@@ -81,63 +82,6 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 	}
 }
 
-// sendInsideMessage encrypts a firewall-approved inside packet into the
-// caller's batch slot for later sendmmsg flush. When hostinfo.remote is not
-// valid we fall through to the relay slow path via the unbatched sendNoMetrics
-// so relay behavior is unchanged.
-func (f *Interface) sendInsideMessage(hostinfo *HostInfo, p, nb []byte, batch *sendBatch, rejectBuf []byte, q int) {
-	ci := hostinfo.ConnectionState
-	if ci.eKey == nil {
-		return
-	}
-
-	if !hostinfo.remote.IsValid() {
-		// Slow path: relay fallback. Reuse rejectBuf as the ciphertext
-		// scratch; sendNoMetrics arranges header space for SendVia.
-		f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, p, nb, rejectBuf, q)
-		return
-	}
-
-	scratch := batch.Next()
-	if scratch == nil {
-		// Batch full: bypass batching and send this packet directly so we
-		// never drop traffic on over-subscribed iterations.
-		f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, p, nb, rejectBuf, q)
-		return
-	}
-
-	if noiseutil.EncryptLockNeeded {
-		ci.writeLock.Lock()
-	}
-	c := ci.messageCounter.Add(1)
-
-	out := header.Encode(scratch, header.Version, header.Message, 0, hostinfo.remoteIndexId, c)
-	f.connectionManager.Out(hostinfo)
-
-	if hostinfo.lastRebindCount != f.rebindCount {
-		//NOTE: there is an update hole if a tunnel isn't used and exactly 256 rebinds occur before the tunnel is
-		// finally used again. This tunnel would eventually be torn down and recreated if this action didn't help.
-		f.lightHouse.QueryServer(hostinfo.vpnAddrs[0])
-		hostinfo.lastRebindCount = f.rebindCount
-		if f.l.Level >= logrus.DebugLevel {
-			f.l.WithField("vpnAddrs", hostinfo.vpnAddrs).Debug("Lighthouse update triggered for punch due to rebind counter")
-		}
-	}
-
-	out, err := ci.eKey.EncryptDanger(out, out, p, c, nb)
-	if noiseutil.EncryptLockNeeded {
-		ci.writeLock.Unlock()
-	}
-	if err != nil {
-		hostinfo.logger(f.l).WithError(err).
-			WithField("udpAddr", hostinfo.remote).WithField("counter", c).
-			Error("Failed to encrypt outgoing packet")
-		return
-	}
-
-	batch.Commit(len(out), hostinfo.remote)
-}
-
 func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
 	if !f.firewall.InSendReject {
 		return
@@ -148,7 +92,7 @@ func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
 		return
 	}
 
-	_, err := f.readers[q].WriteReject(out)
+	_, err := f.readers[q].Write(out)
 	if err != nil {
 		f.l.WithError(err).Error("Failed to write to tun")
 	}
@@ -177,93 +121,22 @@ func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *
 	f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, out, nb, packet, q)
 }
 
-// Handshake will attempt to initiate a tunnel with the provided vpn address. This is a no-op if the tunnel is already established or being established
-// it does not check if it is within our vpn networks!
 func (f *Interface) Handshake(vpnAddr netip.Addr) {
-	f.handshakeManager.GetOrHandshake(vpnAddr, nil)
+	f.getOrHandshake(vpnAddr, nil)
 }
 
-// getOrHandshakeNoRouting returns nil if the vpnAddr is not routable.
+// getOrHandshake returns nil if the vpnAddr is not routable.
 // If the 2nd return var is false then the hostinfo is not ready to be used in a tunnel
-func (f *Interface) getOrHandshakeNoRouting(vpnAddr netip.Addr, cacheCallback func(*HandshakeHostInfo)) (*HostInfo, bool) {
+func (f *Interface) getOrHandshake(vpnAddr netip.Addr, cacheCallback func(*HandshakeHostInfo)) (*HostInfo, bool) {
-	if f.myVpnNetworksTable.Contains(vpnAddr) {
+	_, found := f.myVpnNetworksTable.Lookup(vpnAddr)
-		return f.handshakeManager.GetOrHandshake(vpnAddr, cacheCallback)
+	if !found {
-	}
+		vpnAddr = f.inside.RouteFor(vpnAddr)
-
+		if !vpnAddr.IsValid() {
-	return nil, false
+			return nil, false
-}
+		}
-
-// getOrHandshakeConsiderRouting will try to find the HostInfo to handle this packet, starting a handshake if necessary.
-// If the 2nd return var is false then the hostinfo is not ready to be used in a tunnel.
-func (f *Interface) getOrHandshakeConsiderRouting(fwPacket *firewall.Packet, cacheCallback func(*HandshakeHostInfo)) (*HostInfo, bool) {
-	destinationAddr := fwPacket.RemoteAddr
-
-	hostinfo, ready := f.getOrHandshakeNoRouting(destinationAddr, cacheCallback)
-
-	// Host is inside the mesh, no routing required
-	if hostinfo != nil {
-		return hostinfo, ready
-	}
-
-	gateways := f.inside.RoutesFor(destinationAddr)
-
-	switch len(gateways) {
-	case 0:
-		return nil, false
-	case 1:
-		// Single gateway route
-		return f.handshakeManager.GetOrHandshake(gateways[0].Addr(), cacheCallback)
-	default:
-		// Multi gateway route, perform ECMP categorization
-		gatewayAddr, balancingOk := routing.BalancePacket(fwPacket, gateways)
-
-		if !balancingOk {
-			// This happens if the gateway buckets were not calculated, this _should_ never happen
-			f.l.Error("Gateway buckets not calculated, fallback from ECMP to random routing. Please report this bug.")
-		}
-
-		var handshakeInfoForChosenGateway *HandshakeHostInfo
-		var hhReceiver = func(hh *HandshakeHostInfo) {
-			handshakeInfoForChosenGateway = hh
-		}
-
-		// Store the handshakeHostInfo for later.
-		// If this node is not reachable we will attempt other nodes, if none are reachable we will
-		// cache the packet for this gateway.
-		if hostinfo, ready = f.handshakeManager.GetOrHandshake(gatewayAddr, hhReceiver); ready {
-			return hostinfo, true
-		}
-
-		// It appears the selected gateway cannot be reached, find another gateway to fallback on.
-		// The current implementation breaks ECMP but that seems better than no connectivity.
-		// If ECMP is also required when a gateway is down then connectivity status
-		// for each gateway needs to be kept and the weights recalculated when they go up or down.
-		// This would also need to interact with unsafe_route updates through reloading the config or
-		// use of the use_system_route_table option
-
-		if f.l.Level >= logrus.DebugLevel {
-			f.l.WithField("destination", destinationAddr).
-				WithField("originalGateway", gatewayAddr).
-				Debugln("Calculated gateway for ECMP not available, attempting other gateways")
-		}
-
-		for i := range gateways {
-			// Skip the gateway that failed previously
-			if gateways[i].Addr() == gatewayAddr {
-				continue
-			}
-
-			// We do not need the HandshakeHostInfo since we cache the packet in the originally chosen gateway
-			if hostinfo, ready = f.handshakeManager.GetOrHandshake(gateways[i].Addr(), nil); ready {
-				return hostinfo, true
-			}
-		}
-
-		// No gateways reachable, cache the packet in the originally chosen gateway
-		cacheCallback(handshakeInfoForChosenGateway)
-		return hostinfo, false
 	}
 
+	return f.handshakeManager.GetOrHandshake(vpnAddr, cacheCallback)
 }
 
 func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte) {
@@ -288,10 +161,9 @@ func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubTyp
 	f.sendNoMetrics(header.Message, st, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, p, nb, out, 0)
 }
 
-// SendMessageToVpnAddr handles real addr:port lookup and sends to the current best known address for vpnAddr.
+// SendMessageToVpnAddr handles real addr:port lookup and sends to the current best known address for vpnAddr
-// This function ignores myVpnNetworksTable, and will always attempt to treat the address as a vpnAddr
 func (f *Interface) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p, nb, out []byte) {
-	hostInfo, ready := f.handshakeManager.GetOrHandshake(vpnAddr, func(hh *HandshakeHostInfo) {
+	hostInfo, ready := f.getOrHandshake(vpnAddr, func(hh *HandshakeHostInfo) {
 		hh.cachePacket(f.l, t, st, p, f.SendMessageToHostInfo, f.cachedPacketMetrics)
 	})
 
@@ -346,7 +218,7 @@ func (f *Interface) SendVia(via *HostInfo,
 	c := via.ConnectionState.messageCounter.Add(1)
 
 	out = header.Encode(out, header.Version, header.Message, header.MessageRelay, relay.RemoteIndex, c)
-	f.connectionManager.Out(via)
+	f.connectionManager.Out(via.localIndexId)
 
 	// Authenticate the header and payload, but do not encrypt for this message type.
 	// The payload consists of the inner, unencrypted Nebula header, as well as the end-to-end encrypted payload.
@@ -414,7 +286,7 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 
 	//l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p)
 	out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
-	f.connectionManager.Out(hostinfo)
+	f.connectionManager.Out(hostinfo.localIndexId)
 
 	// Query our LH if we haven't since the last time we've been rebound, this will cause the remote to punch against
 	// all our addrs and enable a faster roaming.

inside_bsd.go

@@ -1,4 +1,5 @@
 //go:build darwin || dragonfly || freebsd || netbsd || openbsd
+// +build darwin dragonfly freebsd netbsd openbsd
 
 package nebula
 

inside_generic.go

@@ -1,4 +1,5 @@
 //go:build !darwin && !dragonfly && !freebsd && !netbsd && !openbsd
+// +build !darwin,!dragonfly,!freebsd,!netbsd,!openbsd
 
 package nebula
 

interface.go

@@ -4,8 +4,10 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"net/netip"
-	"sync"
+	"os"
+	"runtime"
 	"sync/atomic"
 	"time"
 
@@ -16,31 +18,29 @@ import (
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/overlay"
-	"github.com/slackhq/nebula/overlay/coalesce"
-	"github.com/slackhq/nebula/overlay/tio"
 	"github.com/slackhq/nebula/udp"
 )
 
 const mtu = 9001
 
 type InterfaceConfig struct {
-	HostMap            *HostMap
+	HostMap                 *HostMap
-	Outside            udp.Conn
+	Outside                 udp.Conn
-	Inside             overlay.Device
+	Inside                  overlay.Device
-	pki                *PKI
+	pki                     *PKI
-	Cipher             string
+	Firewall                *Firewall
-	Firewall           *Firewall
+	ServeDns                bool
-	ServeDns           bool
+	HandshakeManager        *HandshakeManager
-	HandshakeManager   *HandshakeManager
+	lightHouse              *LightHouse
-	lightHouse         *LightHouse
+	checkInterval           time.Duration
-	connectionManager  *connectionManager
+	pendingDeletionInterval time.Duration
-	DropLocalBroadcast bool
+	DropLocalBroadcast      bool
-	DropMulticast      bool
+	DropMulticast           bool
-	routines           int
+	routines                int
-	MessageMetrics     *MessageMetrics
+	MessageMetrics          *MessageMetrics
-	version            string
+	version                 string
-	relayManager       *relayManager
+	relayManager            *relayManager
-	punchy             *Punchy
+	punchy                  *Punchy
 
 	tryPromoteEvery uint32
 	reQueryEvery    uint32
@@ -61,11 +61,11 @@ type Interface struct {
 	serveDns              bool
 	createTime            time.Time
 	lightHouse            *LightHouse
-	myBroadcastAddrsTable *bart.Lite
+	myBroadcastAddrsTable *bart.Table[struct{}]
-	myVpnAddrs            []netip.Addr // A list of addresses assigned to us via our certificate
+	myVpnAddrs            []netip.Addr          // A list of addresses assigned to us via our certificate
-	myVpnAddrsTable       *bart.Lite
+	myVpnAddrsTable       *bart.Table[struct{}] // A table of addresses assigned to us via our certificate
-	myVpnNetworks         []netip.Prefix // A list of networks assigned to us via our certificate
+	myVpnNetworks         []netip.Prefix        // A list of networks assigned to us via our certificate
-	myVpnNetworksTable    *bart.Lite
+	myVpnNetworksTable    *bart.Table[struct{}] // A table of networks assigned to us via our certificate
 	dropLocalBroadcast    bool
 	dropMulticast         bool
 	routines              int
@@ -77,8 +77,7 @@ type Interface struct {
 	reQueryEvery    atomic.Uint32
 	reQueryWait     atomic.Int64
 
-	sendRecvErrorConfig   recvErrorConfig
+	sendRecvErrorConfig sendRecvErrorConfig
-	acceptRecvErrorConfig recvErrorConfig
 
 	// rebindCount is used to decide if an active tunnel should trigger a punch notification through a lighthouse
 	rebindCount int8
@@ -87,18 +86,7 @@ type Interface struct {
 	conntrackCacheTimeout time.Duration
 
 	writers []udp.Conn
-	readers []tio.Queue
+	readers []io.ReadWriteCloser
-	// tunCoalescers is one tcpCoalescer per tun queue, wrapping readers[i].
-	// decryptToTun sends plaintext into the coalescer; listenOut calls its
-	// Flush at the end of each UDP recvmmsg batch.
-	tunCoalescers []*coalesce.TCPCoalescer
-	wg            sync.WaitGroup
-
-	// fatalErr holds the first unexpected reader error that caused shutdown.
-	// nil means "no fatal error" (yet)
-	fatalErr atomic.Pointer[error]
-	// triggerShutdown is a function that will be run exactly once, when onFatal swaps something non-nil into fatalErr
-	triggerShutdown func()
 
 	metricHandshakes    metrics.Histogram
 	messageMetrics      *MessageMetrics
@@ -122,34 +110,34 @@ type EncWriter interface {
 	GetCertState() *CertState
 }
 
-type recvErrorConfig uint8
+type sendRecvErrorConfig uint8
 
 const (
-	recvErrorAlways recvErrorConfig = iota
+	sendRecvErrorAlways sendRecvErrorConfig = iota
-	recvErrorNever
+	sendRecvErrorNever
-	recvErrorPrivate
+	sendRecvErrorPrivate
 )
 
-func (s recvErrorConfig) ShouldRecvError(endpoint netip.AddrPort) bool {
+func (s sendRecvErrorConfig) ShouldSendRecvError(endpoint netip.AddrPort) bool {
 	switch s {
-	case recvErrorPrivate:
+	case sendRecvErrorPrivate:
 		return endpoint.Addr().IsPrivate()
-	case recvErrorAlways:
+	case sendRecvErrorAlways:
 		return true
-	case recvErrorNever:
+	case sendRecvErrorNever:
 		return false
 	default:
-		panic(fmt.Errorf("invalid recvErrorConfig value: %d", s))
+		panic(fmt.Errorf("invalid sendRecvErrorConfig value: %d", s))
 	}
 }
 
-func (s recvErrorConfig) String() string {
+func (s sendRecvErrorConfig) String() string {
 	switch s {
-	case recvErrorAlways:
+	case sendRecvErrorAlways:
 		return "always"
-	case recvErrorNever:
+	case sendRecvErrorNever:
 		return "never"
-	case recvErrorPrivate:
+	case sendRecvErrorPrivate:
 		return "private"
 	default:
 		return fmt.Sprintf("invalid(%d)", s)
@@ -169,9 +157,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 	if c.Firewall == nil {
 		return nil, errors.New("no firewall rules")
 	}
-	if c.connectionManager == nil {
-		return nil, errors.New("no connection manager")
-	}
 
 	cs := c.pki.getCertState()
 	ifce := &Interface{
@@ -189,15 +174,14 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 		routines:              c.routines,
 		version:               c.version,
 		writers:               make([]udp.Conn, c.routines),
-		readers:               make([]tio.Queue, c.routines),
+		readers:               make([]io.ReadWriteCloser, c.routines),
-		tunCoalescers:         make([]*coalesce.TCPCoalescer, c.routines),
 		myVpnNetworks:         cs.myVpnNetworks,
 		myVpnNetworksTable:    cs.myVpnNetworksTable,
 		myVpnAddrs:            cs.myVpnAddrs,
 		myVpnAddrsTable:       cs.myVpnAddrsTable,
 		myBroadcastAddrsTable: cs.myVpnBroadcastAddrsTable,
 		relayManager:          c.relayManager,
-		connectionManager:     c.connectionManager,
+
 		conntrackCacheTimeout: c.ConntrackCacheTimeout,
 
 		metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)),
@@ -214,7 +198,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 	ifce.reQueryEvery.Store(c.reQueryEvery)
 	ifce.reQueryWait.Store(int64(c.reQueryWait))
 
-	ifce.connectionManager.intf = ifce
+	ifce.connectionManager = newConnectionManager(ctx, c.l, ifce, c.checkInterval, c.pendingDeletionInterval, c.punchy)
 
 	return ifce, nil
 }
@@ -222,7 +206,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 // activate creates the interface on the host. After the interface is created, any
 // other services that want to bind listeners to its IP may do so successfully. However,
 // the interface isn't going to process anything until run() is called.
-func (f *Interface) activate() error {
+func (f *Interface) activate() {
 	// actually turn on tun dev
 
 	addr, err := f.outside.LocalAddr()
@@ -235,75 +219,41 @@ func (f *Interface) activate() error {
 		WithField("boringcrypto", boringEnabled()).
 		Info("Nebula interface is active")
 
-	if f.routines > 1 {
-		if !f.inside.SupportsMultiqueue() || !f.outside.SupportsMultipleReaders() {
-			f.routines = 1
-			f.l.Warn("routines is not supported on this platform, falling back to a single routine")
-		}
-	}
-
 	metrics.GetOrRegisterGauge("routines", nil).Update(int64(f.routines))
 
 	// Prepare n tun queues
+	var reader io.ReadWriteCloser = f.inside
 	for i := 0; i < f.routines; i++ {
 		if i > 0 {
-			err = f.inside.NewMultiQueueReader()
+			reader, err = f.inside.NewMultiQueueReader()
 			if err != nil {
-				return err
+				f.l.Fatal(err)
 			}
 		}
-	}
+		f.readers[i] = reader
-	f.readers = f.inside.Readers()
-	for i := range f.readers {
-		f.tunCoalescers[i] = coalesce.NewTCPCoalescer(f.readers[i]) //todo don't always do this
 	}
 
-	f.wg.Add(1) // for us to wait on Close() to return
+	if err := f.inside.Activate(); err != nil {
-	if err = f.inside.Activate(); err != nil {
-		f.wg.Done()
 		f.inside.Close()
-		return err
+		f.l.Fatal(err)
 	}
-
-	return nil
 }
 
-func (f *Interface) run() (func() error, error) {
+func (f *Interface) run() {
 	// Launch n queues to read packets from udp
 	for i := 0; i < f.routines; i++ {
-		f.wg.Go(func() {
+		go f.listenOut(i)
-			f.listenOut(i)
-		})
 	}
 
 	// Launch n queues to read packets from tun dev
 	for i := 0; i < f.routines; i++ {
-		f.wg.Go(func() {
+		go f.listenIn(f.readers[i], i)
-			f.listenIn(f.readers[i], i)
-		})
-	}
-
-	return func() error {
-		f.wg.Wait()
-		if e := f.fatalErr.Load(); e != nil {
-			return *e
-		}
-		return nil
-	}, nil
-}
-
-// onFatal stores the first fatal reader error, and calls triggerShutdown if it was the first one
-func (f *Interface) onFatal(err error) {
-	swapped := f.fatalErr.CompareAndSwap(nil, &err)
-	if !swapped {
-		return
-	}
-	if f.triggerShutdown != nil {
-		f.triggerShutdown()
 	}
 }
 
 func (f *Interface) listenOut(i int) {
+	runtime.LockOSThread()
+
 	var li udp.Conn
 	if i > 0 {
 		li = f.writers[i]
@@ -313,82 +263,45 @@ func (f *Interface) listenOut(i int) {
 
 	ctCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
 	lhh := f.lightHouse.NewRequestHandler()
+	plaintext := make([]byte, udp.MTU)
 	h := &header.H{}
 	fwPacket := &firewall.Packet{}
 	nb := make([]byte, 12, 12)
 
-	// plaintexts is a ring of decrypt scratches, one per packet in a UDP
+	li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) {
-	// recvmmsg batch. The coalescer borrows payload slices from here and
+		f.readOutsidePackets(fromUdpAddr, nil, plaintext[:0], payload, h, fwPacket, lhh, nb, i, ctCache.Get(f.l))
-	// requires they stay valid until Flush — so we rotate each packet and
-	// reset only in the batch-end flush callback.
-	var plaintexts [][]byte
-	idx := 0
-	coalescer := f.tunCoalescers[i]
-	err := li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) {
-		if idx >= len(plaintexts) {
-			plaintexts = append(plaintexts, make([]byte, udp.MTU))
-		}
-		f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, plaintexts[idx][:0], payload, h, fwPacket, lhh, nb, i, ctCache.Get(f.l))
-		idx++
-	}, func() {
-		if err := coalescer.Flush(); err != nil {
-			f.l.WithError(err).Error("Failed to flush tun coalescer")
-		}
-		idx = 0
 	})
-
-	if err != nil && !f.closed.Load() {
-		f.l.WithError(err).Error("Error while reading inbound packet, closing")
-		f.onFatal(err)
-	}
-
-	f.l.Debugf("underlay reader %v is done", i)
 }
 
-func (f *Interface) listenIn(reader tio.Queue, i int) {
+func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
-	rejectBuf := make([]byte, mtu)
+	runtime.LockOSThread()
-	batch := newSendBatch(sendBatchCap, udp.MTU+32)
+
+	packet := make([]byte, mtu)
+	out := make([]byte, mtu)
 	fwPacket := &firewall.Packet{}
 	nb := make([]byte, 12, 12)
 
 	conntrackCache := firewall.NewConntrackCacheTicker(f.conntrackCacheTimeout)
 
 	for {
-		pkts, err := reader.Read()
+		n, err := reader.Read(packet)
 		if err != nil {
-			if !f.closed.Load() {
+			if errors.Is(err, os.ErrClosed) && f.closed.Load() {
-				f.l.WithError(err).WithField("reader", i).Error("Error while reading outbound packet, closing")
+				return
-				f.onFatal(err)
 			}
-			break
+
+			f.l.WithError(err).Error("Error while reading outbound packet")
+			// This only seems to happen when something fatal happens to the fd, so exit.
+			os.Exit(2)
 		}
 
-		batch.Reset()
+		f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get(f.l))
-		for _, pkt := range pkts {
-			if batch.Len() >= batch.Cap() {
-				f.flushBatch(batch, i)
-				batch.Reset()
-			}
-			f.consumeInsidePacket(pkt, fwPacket, nb, batch, rejectBuf, i, conntrackCache.Get(f.l))
-		}
-		if batch.Len() > 0 {
-			f.flushBatch(batch, i)
-		}
-	}
-
-	f.l.Debugf("overlay reader %v is done", i)
-}
-
-func (f *Interface) flushBatch(batch *sendBatch, q int) {
-	if err := f.writers[q].WriteBatch(batch.bufs, batch.dsts); err != nil {
-		f.l.WithError(err).WithField("writer", q).Error("Failed to write outgoing batch")
 	}
 }
 
 func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) {
 	c.RegisterReloadCallback(f.reloadFirewall)
 	c.RegisterReloadCallback(f.reloadSendRecvError)
-	c.RegisterReloadCallback(f.reloadAcceptRecvError)
 	c.RegisterReloadCallback(f.reloadDisconnectInvalid)
 	c.RegisterReloadCallback(f.reloadMisc)
 
@@ -452,16 +365,16 @@ func (f *Interface) reloadSendRecvError(c *config.C) {
 
 		switch stringValue {
 		case "always":
-			f.sendRecvErrorConfig = recvErrorAlways
+			f.sendRecvErrorConfig = sendRecvErrorAlways
 		case "never":
-			f.sendRecvErrorConfig = recvErrorNever
+			f.sendRecvErrorConfig = sendRecvErrorNever
 		case "private":
-			f.sendRecvErrorConfig = recvErrorPrivate
+			f.sendRecvErrorConfig = sendRecvErrorPrivate
 		default:
 			if c.GetBool("listen.send_recv_error", true) {
-				f.sendRecvErrorConfig = recvErrorAlways
+				f.sendRecvErrorConfig = sendRecvErrorAlways
 			} else {
-				f.sendRecvErrorConfig = recvErrorNever
+				f.sendRecvErrorConfig = sendRecvErrorNever
 			}
 		}
 
@@ -470,30 +383,6 @@ func (f *Interface) reloadSendRecvError(c *config.C) {
 	}
 }
 
-func (f *Interface) reloadAcceptRecvError(c *config.C) {
-	if c.InitialLoad() || c.HasChanged("listen.accept_recv_error") {
-		stringValue := c.GetString("listen.accept_recv_error", "always")
-
-		switch stringValue {
-		case "always":
-			f.acceptRecvErrorConfig = recvErrorAlways
-		case "never":
-			f.acceptRecvErrorConfig = recvErrorNever
-		case "private":
-			f.acceptRecvErrorConfig = recvErrorPrivate
-		default:
-			if c.GetBool("listen.accept_recv_error", true) {
-				f.acceptRecvErrorConfig = recvErrorAlways
-			} else {
-				f.acceptRecvErrorConfig = recvErrorNever
-			}
-		}
-
-		f.l.WithField("acceptRecvError", f.acceptRecvErrorConfig.String()).
-			Info("Loaded accept_recv_error config")
-	}
-}
-
 func (f *Interface) reloadMisc(c *config.C) {
 	if c.HasChanged("counters.try_promote") {
 		n := c.GetUint32("counters.try_promote", defaultPromoteEvery)
@@ -521,7 +410,7 @@ func (f *Interface) emitStats(ctx context.Context, i time.Duration) {
 	udpStats := udp.NewUDPStatsEmitter(f.writers)
 
 	certExpirationGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil)
-	certInitiatingVersion := metrics.GetOrRegisterGauge("certificate.initiating_version", nil)
+	certDefaultVersion := metrics.GetOrRegisterGauge("certificate.default_version", nil)
 	certMaxVersion := metrics.GetOrRegisterGauge("certificate.max_version", nil)
 
 	for {
@@ -536,7 +425,7 @@ func (f *Interface) emitStats(ctx context.Context, i time.Duration) {
 			certState := f.pki.getCertState()
 			defaultCrt := certState.GetDefaultCertificate()
 			certExpirationGauge.Update(int64(defaultCrt.NotAfter().Sub(time.Now()) / time.Second))
-			certInitiatingVersion.Update(int64(defaultCrt.Version()))
+			certDefaultVersion.Update(int64(defaultCrt.Version()))
 
 			// Report the max certificate version we are capable of using
 			if certState.v2Cert != nil {
@@ -557,23 +446,15 @@ func (f *Interface) GetCertState() *CertState {
 }
 
 func (f *Interface) Close() error {
-	var errs []error
 	f.closed.Store(true)
 
-	// Release the udp readers
+	for _, u := range f.writers {
-	for i, u := range f.writers {
 		err := u.Close()
 		if err != nil {
-			f.l.WithError(err).WithField("writer", i).Error("Error while closing udp socket")
+			f.l.WithError(err).Error("Error while closing udp socket")
-			errs = append(errs, err)
 		}
 	}
 
-	// Release the tun device (closing the tun also closes all readers)
+	// Release the tun device
-	closeErr := f.inside.Close()
+	return f.inside.Close()
-	if closeErr != nil {
-		errs = append(errs, closeErr)
-	}
-	f.wg.Done()
-	return errors.Join(errs...)
 }

lighthouse.go

@@ -24,7 +24,6 @@ import (
 )
 
 var ErrHostNotKnown = errors.New("host not known")
-var ErrBadDetailsVpnAddr = errors.New("invalid packet, malformed detailsVpnAddr")
 
 type LightHouse struct {
 	//TODO: We need a timer wheel to kick out vpnAddrs that haven't reported in a long time
@@ -33,7 +32,7 @@ type LightHouse struct {
 	amLighthouse bool
 
 	myVpnNetworks      []netip.Prefix
-	myVpnNetworksTable *bart.Lite
+	myVpnNetworksTable *bart.Table[struct{}]
 	punchConn          udp.Conn
 	punchy             *Punchy
 
@@ -57,7 +56,7 @@ type LightHouse struct {
 	// staticList exists to avoid having a bool in each addrMap entry
 	// since static should be rare
 	staticList  atomic.Pointer[map[netip.Addr]struct{}]
-	lighthouses atomic.Pointer[[]netip.Addr]
+	lighthouses atomic.Pointer[map[netip.Addr]struct{}]
 
 	interval     atomic.Int64
 	updateCancel context.CancelFunc
@@ -108,7 +107,7 @@ func NewLightHouseFromConfig(ctx context.Context, l *logrus.Logger, c *config.C,
 		queryChan:          make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)),
 		l:                  l,
 	}
-	lighthouses := make([]netip.Addr, 0)
+	lighthouses := make(map[netip.Addr]struct{})
 	h.lighthouses.Store(&lighthouses)
 	staticList := make(map[netip.Addr]struct{})
 	h.staticList.Store(&staticList)
@@ -144,7 +143,7 @@ func (lh *LightHouse) GetStaticHostList() map[netip.Addr]struct{} {
 	return *lh.staticList.Load()
 }
 
-func (lh *LightHouse) GetLighthouses() []netip.Addr {
+func (lh *LightHouse) GetLighthouses() map[netip.Addr]struct{} {
 	return *lh.lighthouses.Load()
 }
 
@@ -202,7 +201,8 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
 
 			//TODO: we could technically insert all returned addrs instead of just the first one if a dns lookup was used
 			addr := addrs[0].Unmap()
-			if lh.myVpnNetworksTable.Contains(addr) {
+			_, found := lh.myVpnNetworksTable.Lookup(addr)
+			if found {
 				lh.l.WithField("addr", rawAddr).WithField("entry", i+1).
 					Warn("Ignoring lighthouse.advertise_addrs report because it is within the nebula network range")
 				continue
@@ -307,12 +307,13 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
 	}
 
 	if initial || c.HasChanged("lighthouse.hosts") {
-		lhList, err := lh.parseLighthouses(c)
+		lhMap := make(map[netip.Addr]struct{})
+		err := lh.parseLighthouses(c, lhMap)
 		if err != nil {
 			return err
 		}
 
-		lh.lighthouses.Store(&lhList)
+		lh.lighthouses.Store(&lhMap)
 		if !initial {
 			//NOTE: we are not tearing down existing lighthouse connections because they might be used for non lighthouse traffic
 			lh.l.Info("lighthouse.hosts has changed")
@@ -346,38 +347,37 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
 	return nil
 }
 
-func (lh *LightHouse) parseLighthouses(c *config.C) ([]netip.Addr, error) {
+func (lh *LightHouse) parseLighthouses(c *config.C, lhMap map[netip.Addr]struct{}) error {
 	lhs := c.GetStringSlice("lighthouse.hosts", []string{})
 	if lh.amLighthouse && len(lhs) != 0 {
 		lh.l.Warn("lighthouse.am_lighthouse enabled on node but upstream lighthouses exist in config")
 	}
-	out := make([]netip.Addr, len(lhs))
 
 	for i, host := range lhs {
 		addr, err := netip.ParseAddr(host)
 		if err != nil {
-			return nil, util.NewContextualError("Unable to parse lighthouse host entry", m{"host": host, "entry": i + 1}, err)
+			return util.NewContextualError("Unable to parse lighthouse host entry", m{"host": host, "entry": i + 1}, err)
 		}
 
-		if !lh.myVpnNetworksTable.Contains(addr) {
+		_, found := lh.myVpnNetworksTable.Lookup(addr)
-			lh.l.WithFields(m{"vpnAddr": addr, "networks": lh.myVpnNetworks}).
+		if !found {
-				Warn("lighthouse host is not within our networks, lighthouse functionality will work but layer 3 network traffic to the lighthouse will not")
+			return util.NewContextualError("lighthouse host is not in our networks, invalid", m{"vpnAddr": addr, "networks": lh.myVpnNetworks}, nil)
 		}
-		out[i] = addr
+		lhMap[addr] = struct{}{}
 	}
 
-	if !lh.amLighthouse && len(out) == 0 {
+	if !lh.amLighthouse && len(lhMap) == 0 {
 		lh.l.Warn("No lighthouse.hosts configured, this host will only be able to initiate tunnels with static_host_map entries")
 	}
 
 	staticList := lh.GetStaticHostList()
-	for i := range out {
+	for lhAddr, _ := range lhMap {
-		if _, ok := staticList[out[i]]; !ok {
+		if _, ok := staticList[lhAddr]; !ok {
-			return nil, fmt.Errorf("lighthouse %s does not have a static_host_map entry", out[i])
+			return fmt.Errorf("lighthouse %s does not have a static_host_map entry", lhAddr)
 		}
 	}
 
-	return out, nil
+	return nil
 }
 
 func getStaticMapCadence(c *config.C) (time.Duration, error) {
@@ -422,7 +422,7 @@ func (lh *LightHouse) loadStaticMap(c *config.C, staticList map[netip.Addr]struc
 		return err
 	}
 
-	shm := c.GetMap("static_host_map", map[string]any{})
+	shm := c.GetMap("static_host_map", map[interface{}]interface{}{})
 	i := 0
 
 	for k, v := range shm {
@@ -431,14 +431,14 @@ func (lh *LightHouse) loadStaticMap(c *config.C, staticList map[netip.Addr]struc
 			return util.NewContextualError("Unable to parse static_host_map entry", m{"host": k, "entry": i + 1}, err)
 		}
 
-		if !lh.myVpnNetworksTable.Contains(vpnAddr) {
+		_, found := lh.myVpnNetworksTable.Lookup(vpnAddr)
-			lh.l.WithFields(m{"vpnAddr": vpnAddr, "networks": lh.myVpnNetworks, "entry": i + 1}).
+		if !found {
-				Warn("static_host_map key is not within our networks, layer 3 network traffic to this host will not work")
+			return util.NewContextualError("static_host_map key is not in our network, invalid", m{"vpnAddr": vpnAddr, "networks": lh.myVpnNetworks, "entry": i + 1}, nil)
 		}
 
-		vals, ok := v.([]any)
+		vals, ok := v.([]interface{})
 		if !ok {
-			vals = []any{v}
+			vals = []interface{}{v}
 		}
 		remoteAddrs := []string{}
 		for _, v := range vals {
@@ -489,7 +489,7 @@ func (lh *LightHouse) QueryCache(vpnAddrs []netip.Addr) *RemoteList {
 	lh.Lock()
 	defer lh.Unlock()
 	// Add an entry if we don't already have one
-	return lh.unlockedGetRemoteList(vpnAddrs) //todo CERT-V2 this contains addrmap lookups we could potentially skip
+	return lh.unlockedGetRemoteList(vpnAddrs)
 }
 
 // queryAndPrepMessage is a lock helper on RemoteList, assisting the caller to build a lighthouse message containing
@@ -522,15 +522,11 @@ func (lh *LightHouse) queryAndPrepMessage(vpnAddr netip.Addr, f func(*cache) (in
 }
 
 func (lh *LightHouse) DeleteVpnAddrs(allVpnAddrs []netip.Addr) {
-	// First we check the static host map. If any of the VpnAddrs to be deleted are present, do nothing.
+	// First we check the static mapping
-	staticList := lh.GetStaticHostList()
+	// and do nothing if it is there
-	for _, addr := range allVpnAddrs {
+	if _, ok := lh.GetStaticHostList()[allVpnAddrs[0]]; ok {
-		if _, ok := staticList[addr]; ok {
+		return
-			return
-		}
 	}
-
-	// None of the VpnAddrs were present. Now we can do the deletes.
 	lh.Lock()
 	rm, ok := lh.addrMap[allVpnAddrs[0]]
 	if ok {
@@ -572,7 +568,7 @@ func (lh *LightHouse) addStaticRemotes(i int, d time.Duration, network string, t
 	am.unlockedSetHostnamesResults(hr)
 
 	for _, addrPort := range hr.GetAddrs() {
-		if !lh.shouldAdd([]netip.Addr{vpnAddr}, addrPort.Addr()) {
+		if !lh.shouldAdd(vpnAddr, addrPort.Addr()) {
 			continue
 		}
 		switch {
@@ -634,37 +630,31 @@ func (lh *LightHouse) addCalculatedRemotes(vpnAddr netip.Addr) bool {
 	return len(calculatedV4) > 0 || len(calculatedV6) > 0
 }
 
-// unlockedGetRemoteList assumes you have the lh lock
+// unlockedGetRemoteList
+// assumes you have the lh lock
 func (lh *LightHouse) unlockedGetRemoteList(allAddrs []netip.Addr) *RemoteList {
-	// before we go and make a new remotelist, we need to make sure we don't have one for any of this set of vpnaddrs yet
+	am, ok := lh.addrMap[allAddrs[0]]
-	for i, addr := range allAddrs {
+	if !ok {
-		am, ok := lh.addrMap[addr]
+		am = NewRemoteList(allAddrs, func(a netip.Addr) bool { return lh.shouldAdd(allAddrs[0], a) })
-		if ok {
+		for _, addr := range allAddrs {
-			if i != 0 {
+			lh.addrMap[addr] = am
-				lh.addrMap[allAddrs[0]] = am
-			}
-			return am
 		}
 	}
-
-	am := NewRemoteList(allAddrs, lh.shouldAdd)
-	for _, addr := range allAddrs {
-		lh.addrMap[addr] = am
-	}
 	return am
 }
 
-func (lh *LightHouse) shouldAdd(vpnAddrs []netip.Addr, to netip.Addr) bool {
+func (lh *LightHouse) shouldAdd(vpnAddr netip.Addr, to netip.Addr) bool {
-	allow := lh.GetRemoteAllowList().AllowAll(vpnAddrs, to)
+	allow := lh.GetRemoteAllowList().Allow(vpnAddr, to)
 	if lh.l.Level >= logrus.TraceLevel {
-		lh.l.WithField("vpnAddrs", vpnAddrs).WithField("udpAddr", to).WithField("allow", allow).
+		lh.l.WithField("vpnAddr", vpnAddr).WithField("udpAddr", to).WithField("allow", allow).
 			Trace("remoteAllowList.Allow")
 	}
 	if !allow {
 		return false
 	}
 
-	if lh.myVpnNetworksTable.Contains(to) {
+	_, found := lh.myVpnNetworksTable.Lookup(to)
+	if found {
 		return false
 	}
 
@@ -684,7 +674,8 @@ func (lh *LightHouse) unlockedShouldAddV4(vpnAddr netip.Addr, to *V4AddrPort) bo
 		return false
 	}
 
-	if lh.myVpnNetworksTable.Contains(udpAddr.Addr()) {
+	_, found := lh.myVpnNetworksTable.Lookup(udpAddr.Addr())
+	if found {
 		return false
 	}
 
@@ -704,7 +695,8 @@ func (lh *LightHouse) unlockedShouldAddV6(vpnAddr netip.Addr, to *V6AddrPort) bo
 		return false
 	}
 
-	if lh.myVpnNetworksTable.Contains(udpAddr.Addr()) {
+	_, found := lh.myVpnNetworksTable.Lookup(udpAddr.Addr())
+	if found {
 		return false
 	}
 
@@ -712,14 +704,18 @@ func (lh *LightHouse) unlockedShouldAddV6(vpnAddr netip.Addr, to *V6AddrPort) bo
 }
 
 func (lh *LightHouse) IsLighthouseAddr(vpnAddr netip.Addr) bool {
-	l := lh.GetLighthouses()
+	if _, ok := lh.GetLighthouses()[vpnAddr]; ok {
-	return slices.Contains(l, vpnAddr)
+		return true
+	}
+	return false
 }
 
-func (lh *LightHouse) IsAnyLighthouseAddr(vpnAddrs []netip.Addr) bool {
+// TODO: CERT-V2 IsLighthouseAddr should be sufficient, we just need to update the vpnAddrs for lighthouses after a handshake
+// so that we know all the lighthouse vpnAddrs, not just the ones we were configured to talk to initially
+func (lh *LightHouse) IsAnyLighthouseAddr(vpnAddr []netip.Addr) bool {
 	l := lh.GetLighthouses()
-	for i := range vpnAddrs {
+	for _, a := range vpnAddr {
-		if slices.Contains(l, vpnAddrs[i]) {
+		if _, ok := l[a]; ok {
 			return true
 		}
 	}
@@ -762,12 +758,12 @@ func (lh *LightHouse) innerQueryServer(addr netip.Addr, nb, out []byte) {
 	queried := 0
 	lighthouses := lh.GetLighthouses()
 
-	for _, lhVpnAddr := range lighthouses {
+	for lhVpnAddr := range lighthouses {
 		hi := lh.ifce.GetHostInfo(lhVpnAddr)
 		if hi != nil {
 			v = hi.ConnectionState.myCert.Version()
 		} else {
-			v = lh.ifce.GetCertState().initiatingVersion
+			v = lh.ifce.GetCertState().defaultVersion
 		}
 
 		if v == cert.Version1 {
@@ -860,7 +856,8 @@ func (lh *LightHouse) SendUpdate() {
 
 	lal := lh.GetLocalAllowList()
 	for _, e := range localAddrs(lh.l, lal) {
-		if lh.myVpnNetworksTable.Contains(e) {
+		_, found := lh.myVpnNetworksTable.Lookup(e)
+		if found {
 			continue
 		}
 
@@ -880,13 +877,13 @@ func (lh *LightHouse) SendUpdate() {
 	updated := 0
 	lighthouses := lh.GetLighthouses()
 
-	for _, lhVpnAddr := range lighthouses {
+	for lhVpnAddr := range lighthouses {
 		var v cert.Version
 		hi := lh.ifce.GetHostInfo(lhVpnAddr)
 		if hi != nil {
 			v = hi.ConnectionState.myCert.Version()
 		} else {
-			v = lh.ifce.GetCertState().initiatingVersion
+			v = lh.ifce.GetCertState().defaultVersion
 		}
 		if v == cert.Version1 {
 			if v1Update == nil {
@@ -938,6 +935,7 @@ func (lh *LightHouse) SendUpdate() {
 						V4AddrPorts:   v4,
 						V6AddrPorts:   v6,
 						RelayVpnAddrs: relays,
+						VpnAddr:       netAddrToProtoAddr(lh.myVpnNetworks[0].Addr()),
 					},
 				}
 
@@ -1057,19 +1055,19 @@ func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, fromVpnAddrs []neti
 		return
 	}
 
-	queryVpnAddr, useVersion, err := n.Details.GetVpnAddrAndVersion()
+	useVersion := cert.Version1
-	if err != nil {
+	var queryVpnAddr netip.Addr
+	if n.Details.OldVpnAddr != 0 {
+		b := [4]byte{}
+		binary.BigEndian.PutUint32(b[:], n.Details.OldVpnAddr)
+		queryVpnAddr = netip.AddrFrom4(b)
+		useVersion = 1
+	} else if n.Details.VpnAddr != nil {
+		queryVpnAddr = protoAddrToNetAddr(n.Details.VpnAddr)
+		useVersion = 2
+	} else {
 		if lhh.l.Level >= logrus.DebugLevel {
-			lhh.l.WithField("from", fromVpnAddrs).WithField("details", n.Details).
+			lhh.l.WithField("from", fromVpnAddrs).WithField("details", n.Details).Debugln("Dropping malformed HostQuery")
-				Debugln("Dropping malformed HostQuery")
-		}
-		return
-	}
-	if useVersion == cert.Version1 && queryVpnAddr.Is6() {
-		// this case really shouldn't be possible to represent, but reject it anyway.
-		if lhh.l.Level >= logrus.DebugLevel {
-			lhh.l.WithField("vpnAddrs", fromVpnAddrs).WithField("queryVpnAddr", queryVpnAddr).
-				Debugln("invalid vpn addr for v1 handleHostQuery")
 		}
 		return
 	}
@@ -1078,6 +1076,9 @@ func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, fromVpnAddrs []neti
 		n = lhh.resetMeta()
 		n.Type = NebulaMeta_HostQueryReply
 		if useVersion == cert.Version1 {
+			if !queryVpnAddr.Is4() {
+				return 0, fmt.Errorf("invalid vpn addr for v1 handleHostQuery")
+			}
 			b := queryVpnAddr.As4()
 			n.Details.OldVpnAddr = binary.BigEndian.Uint32(b[:])
 		} else {
@@ -1113,7 +1114,7 @@ func (lhh *LightHouseHandler) sendHostPunchNotification(n *NebulaMeta, fromVpnAd
 		targetHI := lhh.lh.ifce.GetHostInfo(punchNotifDest)
 		var useVersion cert.Version
 		if targetHI == nil {
-			useVersion = lhh.lh.ifce.GetCertState().initiatingVersion
+			useVersion = lhh.lh.ifce.GetCertState().defaultVersion
 		} else {
 			crt := targetHI.GetCert().Certificate
 			useVersion = crt.Version()
@@ -1122,9 +1123,8 @@ func (lhh *LightHouseHandler) sendHostPunchNotification(n *NebulaMeta, fromVpnAd
 			if ok {
 				whereToPunch = newDest
 			} else {
-				if lhh.l.Level >= logrus.DebugLevel {
+				//TODO: CERT-V2 this means the destination will have no addresses in common with the punch-ee
-					lhh.l.WithField("to", crt.Networks()).Debugln("unable to punch to host, no addresses in common")
+				//choosing to do nothing for now, but maybe we return an error?
-				}
 			}
 		}
 
@@ -1183,17 +1183,19 @@ func (lhh *LightHouseHandler) coalesceAnswers(v cert.Version, c *cache, n *Nebul
 				if !r.Is4() {
 					continue
 				}
+
 				b = r.As4()
 				n.Details.OldRelayVpnAddrs = append(n.Details.OldRelayVpnAddrs, binary.BigEndian.Uint32(b[:]))
 			}
+
 		} else if v == cert.Version2 {
 			for _, r := range c.relay.relay {
 				n.Details.RelayVpnAddrs = append(n.Details.RelayVpnAddrs, netAddrToProtoAddr(r))
 			}
+
 		} else {
-			if lhh.l.Level >= logrus.DebugLevel {
+			//TODO: CERT-V2 don't panic
-				lhh.l.WithField("version", v).Debug("unsupported protocol version")
+			panic("unsupported version")
-			}
 		}
 	}
 }
@@ -1203,16 +1205,18 @@ func (lhh *LightHouseHandler) handleHostQueryReply(n *NebulaMeta, fromVpnAddrs [
 		return
 	}
 
-	certVpnAddr, _, err := n.Details.GetVpnAddrAndVersion()
+	lhh.lh.Lock()
-	if err != nil {
+
-		if lhh.l.Level >= logrus.DebugLevel {
+	var certVpnAddr netip.Addr
-			lhh.l.WithError(err).WithField("vpnAddrs", fromVpnAddrs).Error("dropping malformed HostQueryReply")
+	if n.Details.OldVpnAddr != 0 {
-		}
+		b := [4]byte{}
-		return
+		binary.BigEndian.PutUint32(b[:], n.Details.OldVpnAddr)
+		certVpnAddr = netip.AddrFrom4(b)
+	} else if n.Details.VpnAddr != nil {
+		certVpnAddr = protoAddrToNetAddr(n.Details.VpnAddr)
 	}
 	relays := n.Details.GetRelays()
 
-	lhh.lh.Lock()
 	am := lhh.lh.unlockedGetRemoteList([]netip.Addr{certVpnAddr})
 	am.Lock()
 	lhh.lh.Unlock()
@@ -1237,24 +1241,27 @@ func (lhh *LightHouseHandler) handleHostUpdateNotification(n *NebulaMeta, fromVp
 		return
 	}
 
-	// not using GetVpnAddrAndVersion because we don't want to error on a blank detailsVpnAddr
 	var detailsVpnAddr netip.Addr
-	var useVersion cert.Version
+	useVersion := cert.Version1
-	if n.Details.OldVpnAddr != 0 { //v1 always sets this field
+	if n.Details.OldVpnAddr != 0 {
 		b := [4]byte{}
 		binary.BigEndian.PutUint32(b[:], n.Details.OldVpnAddr)
 		detailsVpnAddr = netip.AddrFrom4(b)
 		useVersion = cert.Version1
-	} else if n.Details.VpnAddr != nil { //this field is "optional" in v2, but if it's set, we should enforce it
+	} else if n.Details.VpnAddr != nil {
 		detailsVpnAddr = protoAddrToNetAddr(n.Details.VpnAddr)
 		useVersion = cert.Version2
 	} else {
-		detailsVpnAddr = netip.Addr{}
+		if lhh.l.Level >= logrus.DebugLevel {
-		useVersion = cert.Version2
+			lhh.l.WithField("details", n.Details).Debugf("dropping invalid HostUpdateNotification")
+		}
+		return
 	}
 
-	//Simple check that the host sent this not someone else, if detailsVpnAddr is filled
+	//TODO: CERT-V2 hosts with only v2 certs cannot provide their ipv6 addr when contacting the lighthouse via v4?
-	if detailsVpnAddr.IsValid() && !slices.Contains(fromVpnAddrs, detailsVpnAddr) {
+	//TODO: CERT-V2 why do we care about the vpnAddr in the packet? We know where it came from, right?
+	//Simple check that the host sent this not someone else
+	if !slices.Contains(fromVpnAddrs, detailsVpnAddr) {
 		if lhh.l.Level >= logrus.DebugLevel {
 			lhh.l.WithField("vpnAddrs", fromVpnAddrs).WithField("answer", detailsVpnAddr).Debugln("Host sent invalid update")
 		}
@@ -1268,24 +1275,24 @@ func (lhh *LightHouseHandler) handleHostUpdateNotification(n *NebulaMeta, fromVp
 	am.Lock()
 	lhh.lh.Unlock()
 
-	am.unlockedSetV4(fromVpnAddrs[0], fromVpnAddrs[0], n.Details.V4AddrPorts, lhh.lh.unlockedShouldAddV4)
+	am.unlockedSetV4(fromVpnAddrs[0], detailsVpnAddr, n.Details.V4AddrPorts, lhh.lh.unlockedShouldAddV4)
-	am.unlockedSetV6(fromVpnAddrs[0], fromVpnAddrs[0], n.Details.V6AddrPorts, lhh.lh.unlockedShouldAddV6)
+	am.unlockedSetV6(fromVpnAddrs[0], detailsVpnAddr, n.Details.V6AddrPorts, lhh.lh.unlockedShouldAddV6)
 	am.unlockedSetRelay(fromVpnAddrs[0], relays)
 	am.Unlock()
 
 	n = lhh.resetMeta()
 	n.Type = NebulaMeta_HostUpdateNotificationAck
-	switch useVersion {
+
-	case cert.Version1:
+	if useVersion == cert.Version1 {
 		if !fromVpnAddrs[0].Is4() {
 			lhh.l.WithField("vpnAddrs", fromVpnAddrs).Error("Can not send HostUpdateNotificationAck for a ipv6 vpn ip in a v1 message")
 			return
 		}
 		vpnAddrB := fromVpnAddrs[0].As4()
 		n.Details.OldVpnAddr = binary.BigEndian.Uint32(vpnAddrB[:])
-	case cert.Version2:
+	} else if useVersion == cert.Version2 {
-		// do nothing, we want to send a blank message
+		n.Details.VpnAddr = netAddrToProtoAddr(fromVpnAddrs[0])
-	default:
+	} else {
 		lhh.l.WithField("useVersion", useVersion).Error("invalid protocol version")
 		return
 	}
@@ -1303,20 +1310,13 @@ func (lhh *LightHouseHandler) handleHostUpdateNotification(n *NebulaMeta, fromVp
 func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, w EncWriter) {
 	//It's possible the lighthouse is communicating with us using a non primary vpn addr,
 	//which means we need to compare all fromVpnAddrs against all configured lighthouse vpn addrs.
+	//maybe one day we'll have a better idea, if it matters.
 	if !lhh.lh.IsAnyLighthouseAddr(fromVpnAddrs) {
 		return
 	}
 
-	detailsVpnAddr, _, err := n.Details.GetVpnAddrAndVersion()
-	if err != nil {
-		if lhh.l.Level >= logrus.DebugLevel {
-			lhh.l.WithField("details", n.Details).WithError(err).Debugln("dropping invalid HostPunchNotification")
-		}
-		return
-	}
-
 	empty := []byte{0}
-	punch := func(vpnPeer netip.AddrPort, logVpnAddr netip.Addr) {
+	punch := func(vpnPeer netip.AddrPort) {
 		if !vpnPeer.IsValid() {
 			return
 		}
@@ -1328,38 +1328,48 @@ func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, fromVpn
 		}()
 
 		if lhh.l.Level >= logrus.DebugLevel {
+			var logVpnAddr netip.Addr
+			if n.Details.OldVpnAddr != 0 {
+				b := [4]byte{}
+				binary.BigEndian.PutUint32(b[:], n.Details.OldVpnAddr)
+				logVpnAddr = netip.AddrFrom4(b)
+			} else if n.Details.VpnAddr != nil {
+				logVpnAddr = protoAddrToNetAddr(n.Details.VpnAddr)
+			}
 			lhh.l.Debugf("Punching on %v for %v", vpnPeer, logVpnAddr)
 		}
 	}
 
-	remoteAllowList := lhh.lh.GetRemoteAllowList()
 	for _, a := range n.Details.V4AddrPorts {
-		b := protoV4AddrPortToNetAddrPort(a)
+		punch(protoV4AddrPortToNetAddrPort(a))
-		if remoteAllowList.Allow(detailsVpnAddr, b.Addr()) {
-			punch(b, detailsVpnAddr)
-		}
 	}
 
 	for _, a := range n.Details.V6AddrPorts {
-		b := protoV6AddrPortToNetAddrPort(a)
+		punch(protoV6AddrPortToNetAddrPort(a))
-		if remoteAllowList.Allow(detailsVpnAddr, b.Addr()) {
-			punch(b, detailsVpnAddr)
-		}
 	}
 
 	// This sends a nebula test packet to the host trying to contact us. In the case
 	// of a double nat or other difficult scenario, this may help establish
 	// a tunnel.
 	if lhh.lh.punchy.GetRespond() {
+		var queryVpnAddr netip.Addr
+		if n.Details.OldVpnAddr != 0 {
+			b := [4]byte{}
+			binary.BigEndian.PutUint32(b[:], n.Details.OldVpnAddr)
+			queryVpnAddr = netip.AddrFrom4(b)
+		} else if n.Details.VpnAddr != nil {
+			queryVpnAddr = protoAddrToNetAddr(n.Details.VpnAddr)
+		}
+
 		go func() {
 			time.Sleep(lhh.lh.punchy.GetRespondDelay())
 			if lhh.l.Level >= logrus.DebugLevel {
-				lhh.l.Debugf("Sending a nebula test packet to vpn addr %s", detailsVpnAddr)
+				lhh.l.Debugf("Sending a nebula test packet to vpn addr %s", queryVpnAddr)
 			}
 			//NOTE: we have to allocate a new output buffer here since we are spawning a new goroutine
 			// for each punchBack packet. We should move this into a timerwheel or a single goroutine
 			// managed by a channel.
-			w.SendMessageToVpnAddr(header.Test, header.TestRequest, detailsVpnAddr, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
+			w.SendMessageToVpnAddr(header.Test, header.TestRequest, queryVpnAddr, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
 		}()
 	}
 }
@@ -1438,17 +1448,3 @@ func findNetworkUnion(prefixes []netip.Prefix, addrs []netip.Addr) (netip.Addr,
 	}
 	return netip.Addr{}, false
 }
-
-func (d *NebulaMetaDetails) GetVpnAddrAndVersion() (netip.Addr, cert.Version, error) {
-	if d.OldVpnAddr != 0 {
-		b := [4]byte{}
-		binary.BigEndian.PutUint32(b[:], d.OldVpnAddr)
-		detailsVpnAddr := netip.AddrFrom4(b)
-		return detailsVpnAddr, cert.Version1, nil
-	} else if d.VpnAddr != nil {
-		detailsVpnAddr := protoAddrToNetAddr(d.VpnAddr)
-		return detailsVpnAddr, cert.Version2, nil
-	} else {
-		return netip.Addr{}, cert.Version1, ErrBadDetailsVpnAddr
-	}
-}

lighthouse_test.go

@@ -1,6 +1,7 @@
 package nebula
 
 import (
+	"context"
 	"encoding/binary"
 	"fmt"
 	"net/netip"
@@ -13,7 +14,7 @@ import (
 	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.yaml.in/yaml/v3"
+	"gopkg.in/yaml.v2"
 )
 
 func TestOldIPv4Only(t *testing.T) {
@@ -30,8 +31,8 @@ func TestOldIPv4Only(t *testing.T) {
 func Test_lhStaticMapping(t *testing.T) {
 	l := test.NewLogger()
 	myVpnNet := netip.MustParsePrefix("10.128.0.1/16")
-	nt := new(bart.Lite)
+	nt := new(bart.Table[struct{}])
-	nt.Insert(myVpnNet)
+	nt.Insert(myVpnNet, struct{}{})
 	cs := &CertState{
 		myVpnNetworks:      []netip.Prefix{myVpnNet},
 		myVpnNetworksTable: nt,
@@ -39,24 +40,24 @@ func Test_lhStaticMapping(t *testing.T) {
 	lh1 := "10.128.0.2"
 
 	c := config.NewC(l)
-	c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1}}
+	c.Settings["lighthouse"] = map[interface{}]interface{}{"hosts": []interface{}{lh1}}
-	c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}}
+	c.Settings["static_host_map"] = map[interface{}]interface{}{lh1: []interface{}{"1.1.1.1:4242"}}
-	_, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
+	_, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
 	require.NoError(t, err)
 
 	lh2 := "10.128.0.3"
 	c = config.NewC(l)
-	c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1, lh2}}
+	c.Settings["lighthouse"] = map[interface{}]interface{}{"hosts": []interface{}{lh1, lh2}}
-	c.Settings["static_host_map"] = map[string]any{lh1: []any{"100.1.1.1:4242"}}
+	c.Settings["static_host_map"] = map[interface{}]interface{}{lh1: []interface{}{"100.1.1.1:4242"}}
-	_, err = NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
+	_, err = NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
 	require.EqualError(t, err, "lighthouse 10.128.0.3 does not have a static_host_map entry")
 }
 
 func TestReloadLighthouseInterval(t *testing.T) {
 	l := test.NewLogger()
 	myVpnNet := netip.MustParsePrefix("10.128.0.1/16")
-	nt := new(bart.Lite)
+	nt := new(bart.Table[struct{}])
-	nt.Insert(myVpnNet)
+	nt.Insert(myVpnNet, struct{}{})
 	cs := &CertState{
 		myVpnNetworks:      []netip.Prefix{myVpnNet},
 		myVpnNetworksTable: nt,
@@ -64,13 +65,13 @@ func TestReloadLighthouseInterval(t *testing.T) {
 	lh1 := "10.128.0.2"
 
 	c := config.NewC(l)
-	c.Settings["lighthouse"] = map[string]any{
+	c.Settings["lighthouse"] = map[interface{}]interface{}{
-		"hosts":    []any{lh1},
+		"hosts":    []interface{}{lh1},
 		"interval": "1s",
 	}
 
-	c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}}
+	c.Settings["static_host_map"] = map[interface{}]interface{}{lh1: []interface{}{"1.1.1.1:4242"}}
-	lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
+	lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
 	require.NoError(t, err)
 	lh.ifce = &mockEncWriter{}
 
@@ -90,15 +91,15 @@ func TestReloadLighthouseInterval(t *testing.T) {
 func BenchmarkLighthouseHandleRequest(b *testing.B) {
 	l := test.NewLogger()
 	myVpnNet := netip.MustParsePrefix("10.128.0.1/0")
-	nt := new(bart.Lite)
+	nt := new(bart.Table[struct{}])
-	nt.Insert(myVpnNet)
+	nt.Insert(myVpnNet, struct{}{})
 	cs := &CertState{
 		myVpnNetworks:      []netip.Prefix{myVpnNet},
 		myVpnNetworksTable: nt,
 	}
 
 	c := config.NewC(l)
-	lh, err := NewLightHouseFromConfig(b.Context(), l, c, cs, nil, nil)
+	lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
 	require.NoError(b, err)
 
 	hAddr := netip.MustParseAddrPort("4.5.6.7:12345")
@@ -191,17 +192,17 @@ func TestLighthouse_Memory(t *testing.T) {
 	theirVpnIp := netip.MustParseAddr("10.128.0.3")
 
 	c := config.NewC(l)
-	c.Settings["lighthouse"] = map[string]any{"am_lighthouse": true}
+	c.Settings["lighthouse"] = map[interface{}]interface{}{"am_lighthouse": true}
-	c.Settings["listen"] = map[string]any{"port": 4242}
+	c.Settings["listen"] = map[interface{}]interface{}{"port": 4242}
 
 	myVpnNet := netip.MustParsePrefix("10.128.0.1/24")
-	nt := new(bart.Lite)
+	nt := new(bart.Table[struct{}])
-	nt.Insert(myVpnNet)
+	nt.Insert(myVpnNet, struct{}{})
 	cs := &CertState{
 		myVpnNetworks:      []netip.Prefix{myVpnNet},
 		myVpnNetworksTable: nt,
 	}
-	lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
+	lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
 	lh.ifce = &mockEncWriter{}
 	require.NoError(t, err)
 	lhh := lh.NewRequestHandler()
@@ -276,23 +277,23 @@ func TestLighthouse_Memory(t *testing.T) {
 func TestLighthouse_reload(t *testing.T) {
 	l := test.NewLogger()
 	c := config.NewC(l)
-	c.Settings["lighthouse"] = map[string]any{"am_lighthouse": true}
+	c.Settings["lighthouse"] = map[interface{}]interface{}{"am_lighthouse": true}
-	c.Settings["listen"] = map[string]any{"port": 4242}
+	c.Settings["listen"] = map[interface{}]interface{}{"port": 4242}
 
 	myVpnNet := netip.MustParsePrefix("10.128.0.1/24")
-	nt := new(bart.Lite)
+	nt := new(bart.Table[struct{}])
-	nt.Insert(myVpnNet)
+	nt.Insert(myVpnNet, struct{}{})
 	cs := &CertState{
 		myVpnNetworks:      []netip.Prefix{myVpnNet},
 		myVpnNetworksTable: nt,
 	}
 
-	lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
+	lh, err := NewLightHouseFromConfig(context.Background(), l, c, cs, nil, nil)
 	require.NoError(t, err)
 
-	nc := map[string]any{
+	nc := map[interface{}]interface{}{
-		"static_host_map": map[string]any{
+		"static_host_map": map[interface{}]interface{}{
-			"10.128.0.2": []any{"1.1.1.1:4242"},
+			"10.128.0.2": []interface{}{"1.1.1.1:4242"},
 		},
 	}
 	rc, err := yaml.Marshal(nc)
@@ -416,7 +417,7 @@ func (tw *testEncWriter) GetHostInfo(vpnIp netip.Addr) *HostInfo {
 }
 
 func (tw *testEncWriter) GetCertState() *CertState {
-	return &CertState{initiatingVersion: tw.protocolVersion}
+	return &CertState{defaultVersion: tw.protocolVersion}
 }
 
 // assertIp4InArray asserts every address in want is at the same position in have and that the lengths match
@@ -492,123 +493,3 @@ func Test_findNetworkUnion(t *testing.T) {
 	out, ok = findNetworkUnion([]netip.Prefix{fc00}, []netip.Addr{a1, afe81})
 	assert.False(t, ok)
 }
-
-func TestLighthouse_Dont_Delete_Static_Hosts(t *testing.T) {
-	l := test.NewLogger()
-
-	myUdpAddr2 := netip.MustParseAddrPort("1.2.3.4:4242")
-
-	testSameHostNotStatic := netip.MustParseAddr("10.128.0.41")
-	testStaticHost := netip.MustParseAddr("10.128.0.42")
-	//myVpnIp := netip.MustParseAddr("10.128.0.2")
-
-	c := config.NewC(l)
-	lh1 := "10.128.0.2"
-	c.Settings["lighthouse"] = map[string]any{
-		"hosts":    []any{lh1},
-		"interval": "1s",
-	}
-
-	c.Settings["listen"] = map[string]any{"port": 4242}
-	c.Settings["static_host_map"] = map[string]any{
-		lh1:           []any{"1.1.1.1:4242"},
-		"10.128.0.42": []any{"1.2.3.4:4242"},
-	}
-
-	myVpnNet := netip.MustParsePrefix("10.128.0.1/24")
-	nt := new(bart.Lite)
-	nt.Insert(myVpnNet)
-	cs := &CertState{
-		myVpnNetworks:      []netip.Prefix{myVpnNet},
-		myVpnNetworksTable: nt,
-	}
-	lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
-	require.NoError(t, err)
-	lh.ifce = &mockEncWriter{}
-
-	//test that we actually have the static entry:
-	out := lh.Query(testStaticHost)
-	assert.NotNil(t, out)
-	assert.Equal(t, out.vpnAddrs[0], testStaticHost)
-	out.Rebuild([]netip.Prefix{}) //why tho
-	assert.Equal(t, out.addrs[0], myUdpAddr2)
-
-	//bolt on a lower numbered primary IP
-	am := lh.unlockedGetRemoteList([]netip.Addr{testStaticHost})
-	am.vpnAddrs = []netip.Addr{testSameHostNotStatic, testStaticHost}
-	lh.addrMap[testSameHostNotStatic] = am
-	out.Rebuild([]netip.Prefix{}) //???
-
-	//test that we actually have the static entry:
-	out = lh.Query(testStaticHost)
-	assert.NotNil(t, out)
-	assert.Equal(t, out.vpnAddrs[0], testSameHostNotStatic)
-	assert.Equal(t, out.vpnAddrs[1], testStaticHost)
-	assert.Equal(t, out.addrs[0], myUdpAddr2)
-
-	//test that we actually have the static entry for BOTH:
-	out2 := lh.Query(testSameHostNotStatic)
-	assert.Same(t, out2, out)
-
-	//now do the delete
-	lh.DeleteVpnAddrs([]netip.Addr{testSameHostNotStatic, testStaticHost})
-	//verify
-	out = lh.Query(testSameHostNotStatic)
-	assert.NotNil(t, out)
-	if out == nil {
-		t.Fatal("expected non-nil query for the static host")
-	}
-	assert.Equal(t, out.vpnAddrs[0], testSameHostNotStatic)
-	assert.Equal(t, out.vpnAddrs[1], testStaticHost)
-	assert.Equal(t, out.addrs[0], myUdpAddr2)
-}
-
-func TestLighthouse_DeletesWork(t *testing.T) {
-	l := test.NewLogger()
-
-	myUdpAddr2 := netip.MustParseAddrPort("1.2.3.4:4242")
-	testHost := netip.MustParseAddr("10.128.0.42")
-
-	c := config.NewC(l)
-	lh1 := "10.128.0.2"
-	c.Settings["lighthouse"] = map[string]any{
-		"hosts":    []any{lh1},
-		"interval": "1s",
-	}
-
-	c.Settings["listen"] = map[string]any{"port": 4242}
-	c.Settings["static_host_map"] = map[string]any{
-		lh1: []any{"1.1.1.1:4242"},
-	}
-
-	myVpnNet := netip.MustParsePrefix("10.128.0.1/24")
-	nt := new(bart.Lite)
-	nt.Insert(myVpnNet)
-	cs := &CertState{
-		myVpnNetworks:      []netip.Prefix{myVpnNet},
-		myVpnNetworksTable: nt,
-	}
-	lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
-	require.NoError(t, err)
-	lh.ifce = &mockEncWriter{}
-
-	//insert the host
-	am := lh.unlockedGetRemoteList([]netip.Addr{testHost})
-	am.vpnAddrs = []netip.Addr{testHost}
-	am.addrs = []netip.AddrPort{myUdpAddr2}
-	lh.addrMap[testHost] = am
-	am.Rebuild([]netip.Prefix{}) //???
-
-	//test that we actually have the entry:
-	out := lh.Query(testHost)
-	assert.NotNil(t, out)
-	assert.Equal(t, out.vpnAddrs[0], testHost)
-	out.Rebuild([]netip.Prefix{}) //why tho
-	assert.Equal(t, out.addrs[0], myUdpAddr2)
-
-	//now do the delete
-	lh.DeleteVpnAddrs([]netip.Addr{testHost})
-	//verify
-	out = lh.Query(testHost)
-	assert.Nil(t, out)
-}

main.go

@@ -3,13 +3,8 @@ package nebula
 import (
 	"context"
 	"fmt"
-	"log"
 	"net"
-	"net/http"
-	_ "net/http/pprof"
 	"net/netip"
-	"runtime/debug"
-	"strings"
 	"time"
 
 	"github.com/sirupsen/logrus"
@@ -18,10 +13,10 @@ import (
 	"github.com/slackhq/nebula/sshd"
 	"github.com/slackhq/nebula/udp"
 	"github.com/slackhq/nebula/util"
-	"go.yaml.in/yaml/v3"
+	"gopkg.in/yaml.v2"
 )
 
-type m = map[string]any
+type m map[string]interface{}
 
 func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) {
 	ctx, cancel := context.WithCancel(context.Background())
@@ -32,10 +27,6 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		}
 	}()
 
-	if buildVersion == "" {
-		buildVersion = moduleVersion()
-	}
-
 	l := logger
 	l.Formatter = &logrus.TextFormatter{
 		FullTimestamp: true,
@@ -52,11 +43,6 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		l.Println(string(b))
 	}
 
-	//todo!!!
-	go func() {
-		log.Println(http.ListenAndServe("0.0.0.0:6060", nil))
-	}()
-
 	err := configLogger(l, c)
 	if err != nil {
 		return nil, util.ContextualizeIfNeeded("Failed to configure the logger", err)
@@ -89,8 +75,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	if c.GetBool("sshd.enabled", false) {
 		sshStart, err = configSSH(l, ssh, c)
 		if err != nil {
-			l.WithError(err).Warn("Failed to configure sshd, ssh debugging will not be available")
+			return nil, util.ContextualizeIfNeeded("Error while configuring the sshd", err)
-			sshStart = nil
 		}
 	}
 
@@ -113,7 +98,11 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		// deprecated and undocumented
 		tunQueues := c.GetInt("tun.routines", 1)
 		udpQueues := c.GetInt("listen.routines", 1)
-		routines = max(tunQueues, udpQueues)
+		if tunQueues > udpQueues {
+			routines = tunQueues
+		} else {
+			routines = udpQueues
+		}
 		if routines != 1 {
 			l.WithField("routines", routines).Warn("Setting tun.routines and listen.routines is deprecated. Use `routines` instead")
 		}
@@ -196,7 +185,6 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 
 	hostMap := NewHostMapFromConfig(l, c)
 	punchy := NewPunchyFromConfig(l, c)
-	connManager := newConnectionManagerFromConfig(l, c, hostMap, punchy)
 	lightHouse, err := NewLightHouseFromConfig(ctx, l, c, pki.getCertState(), udpConns[0], punchy)
 	if err != nil {
 		return nil, util.ContextualizeIfNeeded("Failed to initialize lighthouse handler", err)
@@ -232,26 +220,31 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		}
 	}
 
+	checkInterval := c.GetInt("timers.connection_alive_interval", 5)
+	pendingDeletionInterval := c.GetInt("timers.pending_deletion_interval", 10)
+
 	ifConfig := &InterfaceConfig{
-		HostMap:               hostMap,
+		HostMap:                 hostMap,
-		Inside:                tun,
+		Inside:                  tun,
-		Outside:               udpConns[0],
+		Outside:                 udpConns[0],
-		pki:                   pki,
+		pki:                     pki,
-		Firewall:              fw,
+		Firewall:                fw,
-		ServeDns:              serveDns,
+		ServeDns:                serveDns,
-		HandshakeManager:      handshakeManager,
+		HandshakeManager:        handshakeManager,
-		connectionManager:     connManager,
+		lightHouse:              lightHouse,
-		lightHouse:            lightHouse,
+		checkInterval:           time.Second * time.Duration(checkInterval),
-		tryPromoteEvery:       c.GetUint32("counters.try_promote", defaultPromoteEvery),
+		pendingDeletionInterval: time.Second * time.Duration(pendingDeletionInterval),
-		reQueryEvery:          c.GetUint32("counters.requery_every_packets", defaultReQueryEvery),
+		tryPromoteEvery:         c.GetUint32("counters.try_promote", defaultPromoteEvery),
-		reQueryWait:           c.GetDuration("timers.requery_wait_duration", defaultReQueryWait),
+		reQueryEvery:            c.GetUint32("counters.requery_every_packets", defaultReQueryEvery),
-		DropLocalBroadcast:    c.GetBool("tun.drop_local_broadcast", false),
+		reQueryWait:             c.GetDuration("timers.requery_wait_duration", defaultReQueryWait),
-		DropMulticast:         c.GetBool("tun.drop_multicast", false),
+		DropLocalBroadcast:      c.GetBool("tun.drop_local_broadcast", false),
-		routines:              routines,
+		DropMulticast:           c.GetBool("tun.drop_multicast", false),
-		MessageMetrics:        messageMetrics,
+		routines:                routines,
-		version:               buildVersion,
+		MessageMetrics:          messageMetrics,
-		relayManager:          NewRelayManager(ctx, l, hostMap, c),
+		version:                 buildVersion,
-		punchy:                punchy,
+		relayManager:            NewRelayManager(ctx, l, hostMap, c),
+		punchy:                  punchy,
+
 		ConntrackCacheTimeout: conntrackCacheTimeout,
 		l:                     l,
 	}
@@ -269,7 +262,6 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		ifce.RegisterConfigChangeCallbacks(c)
 		ifce.reloadDisconnectInvalid(c)
 		ifce.reloadSendRecvError(c)
-		ifce.reloadAcceptRecvError(c)
 
 		handshakeManager.f = ifce
 		go handshakeManager.Run(ctx)
@@ -296,30 +288,13 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 	}
 
 	return &Control{
-		state:                  StateReady,
+		ifce,
-		f:                      ifce,
+		l,
-		l:                      l,
+		ctx,
-		ctx:                    ctx,
+		cancel,
-		cancel:                 cancel,
+		sshStart,
-		sshStart:               sshStart,
+		statsStart,
-		statsStart:             statsStart,
+		dnsStart,
-		dnsStart:               dnsStart,
+		lightHouse.StartUpdateWorker,
-		lighthouseStart:        lightHouse.StartUpdateWorker,
-		connectionManagerStart: connManager.Start,
 	}, nil
 }
-
-func moduleVersion() string {
-	info, ok := debug.ReadBuildInfo()
-	if !ok {
-		return ""
-	}
-
-	for _, dep := range info.Deps {
-		if dep.Path == "github.com/slackhq/nebula" {
-			return strings.TrimPrefix(dep.Version, "v")
-		}
-	}
-
-	return ""
-}

noise.go

@@ -15,14 +15,14 @@ type endianness interface {
 var noiseEndianness endianness = binary.BigEndian
 
 type NebulaCipherState struct {
-	c cipher.AEAD
+	c noise.Cipher
 	//k [32]byte
 	//n uint64
 }
 
 func NewNebulaCipherState(s *noise.CipherState) *NebulaCipherState {
-	x := s.Cipher()
+	return &NebulaCipherState{c: s.Cipher()}
-	return &NebulaCipherState{c: x.(cipher.AEAD)}
+
 }
 
 // EncryptDanger encrypts and authenticates a given payload.
@@ -46,7 +46,7 @@ func (s *NebulaCipherState) EncryptDanger(out, ad, plaintext []byte, n uint64, n
 		nb[2] = 0
 		nb[3] = 0
 		noiseEndianness.PutUint64(nb[4:], n)
-		out = s.c.Seal(out, nb, plaintext, ad)
+		out = s.c.(cipher.AEAD).Seal(out, nb, plaintext, ad)
 		//l.Debugf("Encryption: outlen: %d, nonce: %d, ad: %s, plainlen %d", len(out), n, ad, len(plaintext))
 		return out, nil
 	} else {
@@ -61,7 +61,7 @@ func (s *NebulaCipherState) DecryptDanger(out, ad, ciphertext []byte, n uint64,
 		nb[2] = 0
 		nb[3] = 0
 		noiseEndianness.PutUint64(nb[4:], n)
-		return s.c.Open(out, nb, ciphertext, ad)
+		return s.c.(cipher.AEAD).Open(out, nb, ciphertext, ad)
 	} else {
 		return []byte{}, nil
 	}
@@ -69,7 +69,7 @@ func (s *NebulaCipherState) DecryptDanger(out, ad, ciphertext []byte, n uint64,
 
 func (s *NebulaCipherState) Overhead() int {
 	if s != nil {
-		return s.c.Overhead()
+		return s.c.(cipher.AEAD).Overhead()
 	}
 	return 0
 }

noiseutil/boring.go

@@ -22,7 +22,7 @@ const EncryptLockNeeded = true
 // NewGCMTLS is no longer exposed in go1.19+, so we need to link it in
 // See: https://github.com/golang/go/issues/56326
 //
-// NewGCMTLS is the internal method used with boringcrypto that provides a
+// NewGCMTLS is the internal method used with boringcrypto that provices a
 // validated mode of AES-GCM which enforces the nonce is strictly
 // monotonically increasing.  This is the TLS 1.2 specification for nonce
 // generation (which also matches the method used by the Noise Protocol)

notboring.go

@@ -1,4 +1,5 @@
 //go:build !boringcrypto
+// +build !boringcrypto
 
 package nebula
 

outside.go

@@ -19,21 +19,22 @@ const (
 	minFwPacketLen = 4
 )
 
-func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
+func (f *Interface) readOutsidePackets(ip netip.AddrPort, via *ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
 	err := h.Parse(packet)
 	if err != nil {
 		// Hole punch packets are 0 or 1 byte big, so lets ignore printing those errors
 		if len(packet) > 1 {
-			f.l.WithField("packet", packet).Infof("Error while parsing inbound packet from %s: %s", via, err)
+			f.l.WithField("packet", packet).Infof("Error while parsing inbound packet from %s: %s", ip, err)
 		}
 		return
 	}
 
 	//l.Error("in packet ", header, packet[HeaderLen:])
-	if !via.IsRelayed {
+	if ip.IsValid() {
-		if f.myVpnNetworksTable.Contains(via.UdpAddr.Addr()) {
+		_, found := f.myVpnNetworksTable.Lookup(ip.Addr())
+		if found {
 			if f.l.Level >= logrus.DebugLevel {
-				f.l.WithField("from", via).Debug("Refusing to process double encrypted packet")
+				f.l.WithField("udpAddr", ip).Debug("Refusing to process double encrypted packet")
 			}
 			return
 		}
@@ -54,7 +55,8 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 
 	switch h.Type {
 	case header.Message:
-		if !f.handleEncrypted(ci, via, h) {
+		// TODO handleEncrypted sends directly to addr on error. Handle this in the tunneling case.
+		if !f.handleEncrypted(ci, ip, h) {
 			return
 		}
 
@@ -78,9 +80,9 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 			// Successfully validated the thing. Get rid of the Relay header.
 			signedPayload = signedPayload[header.Len:]
 			// Pull the Roaming parts up here, and return in all call paths.
-			f.handleHostRoaming(hostinfo, via)
+			f.handleHostRoaming(hostinfo, ip)
 			// Track usage of both the HostInfo and the Relay for the received & authenticated packet
-			f.connectionManager.In(hostinfo)
+			f.connectionManager.In(hostinfo.localIndexId)
 			f.connectionManager.RelayUsed(h.RemoteIndex)
 
 			relay, ok := hostinfo.relayState.QueryRelayForByIdx(h.RemoteIndex)
@@ -95,14 +97,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 			case TerminalType:
 				// If I am the target of this relay, process the unwrapped packet
 				// From this recursive point, all these variables are 'burned'. We shouldn't rely on them again.
-				via = ViaSender{
+				f.readOutsidePackets(netip.AddrPort{}, &ViaSender{relayHI: hostinfo, remoteIdx: relay.RemoteIndex, relay: relay}, out[:0], signedPayload, h, fwPacket, lhf, nb, q, localCache)
-					UdpAddr:   via.UdpAddr,
-					relayHI:   hostinfo,
-					remoteIdx: relay.RemoteIndex,
-					relay:     relay,
-					IsRelayed: true,
-				}
-				f.readOutsidePackets(via, out[:0], signedPayload, h, fwPacket, lhf, nb, q, localCache)
 				return
 			case ForwardingType:
 				// Find the target HostInfo relay object
@@ -132,32 +127,31 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 
 	case header.LightHouse:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		if !f.handleEncrypted(ci, via, h) {
+		if !f.handleEncrypted(ci, ip, h) {
 			return
 		}
 
 		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("from", via).
+			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", ip).
 				WithField("packet", packet).
 				Error("Failed to decrypt lighthouse packet")
 			return
 		}
 
-		//TODO: assert via is not relayed
+		lhf.HandleRequest(ip, hostinfo.vpnAddrs, d, f)
-		lhf.HandleRequest(via.UdpAddr, hostinfo.vpnAddrs, d, f)
 
 		// Fallthrough to the bottom to record incoming traffic
 
 	case header.Test:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		if !f.handleEncrypted(ci, via, h) {
+		if !f.handleEncrypted(ci, ip, h) {
 			return
 		}
 
 		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("from", via).
+			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", ip).
 				WithField("packet", packet).
 				Error("Failed to decrypt test packet")
 			return
@@ -166,7 +160,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 		if h.Subtype == header.TestRequest {
 			// This testRequest might be from TryPromoteBest, so we should roam
 			// to the new IP address before responding
-			f.handleHostRoaming(hostinfo, via)
+			f.handleHostRoaming(hostinfo, ip)
 			f.send(header.Test, header.TestReply, ci, hostinfo, d, nb, out)
 		}
 
@@ -177,41 +171,34 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 
 	case header.Handshake:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		f.handshakeManager.HandleIncoming(via, packet, h)
+		f.handshakeManager.HandleIncoming(ip, via, packet, h)
 		return
 
 	case header.RecvError:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		f.handleRecvError(via.UdpAddr, h)
+		f.handleRecvError(ip, h)
 		return
 
 	case header.CloseTunnel:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		if !f.handleEncrypted(ci, via, h) {
+		if !f.handleEncrypted(ci, ip, h) {
-			return
-		}
-		_, err = f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
-		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("from", via).
-				WithField("packet", packet).
-				Error("Failed to decrypt CloseTunnel packet")
 			return
 		}
 
-		hostinfo.logger(f.l).WithField("from", via).
+		hostinfo.logger(f.l).WithField("udpAddr", ip).
 			Info("Close tunnel received, tearing down.")
 
 		f.closeTunnel(hostinfo)
 		return
 
 	case header.Control:
-		if !f.handleEncrypted(ci, via, h) {
+		if !f.handleEncrypted(ci, ip, h) {
 			return
 		}
 
 		d, err := f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
 		if err != nil {
-			hostinfo.logger(f.l).WithError(err).WithField("from", via).
+			hostinfo.logger(f.l).WithError(err).WithField("udpAddr", ip).
 				WithField("packet", packet).
 				Error("Failed to decrypt Control packet")
 			return
@@ -221,13 +208,13 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 
 	default:
 		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		hostinfo.logger(f.l).Debugf("Unexpected packet received from %s", via)
+		hostinfo.logger(f.l).Debugf("Unexpected packet received from %s", ip)
 		return
 	}
 
-	f.handleHostRoaming(hostinfo, via)
+	f.handleHostRoaming(hostinfo, ip)
 
-	f.connectionManager.In(hostinfo)
+	f.connectionManager.In(hostinfo.localIndexId)
 }
 
 // closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote
@@ -244,42 +231,40 @@ func (f *Interface) sendCloseTunnel(h *HostInfo) {
 	f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, make([]byte, 12, 12), make([]byte, mtu))
 }
 
-func (f *Interface) handleHostRoaming(hostinfo *HostInfo, via ViaSender) {
+func (f *Interface) handleHostRoaming(hostinfo *HostInfo, udpAddr netip.AddrPort) {
-	if !via.IsRelayed && hostinfo.remote != via.UdpAddr {
+	if udpAddr.IsValid() && hostinfo.remote != udpAddr {
-		if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, via.UdpAddr.Addr()) {
+		if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, udpAddr.Addr()) {
-			hostinfo.logger(f.l).WithField("newAddr", via.UdpAddr).Debug("lighthouse.remote_allow_list denied roaming")
+			hostinfo.logger(f.l).WithField("newAddr", udpAddr).Debug("lighthouse.remote_allow_list denied roaming")
 			return
 		}
 
-		if !hostinfo.lastRoam.IsZero() && via.UdpAddr == hostinfo.lastRoamRemote && time.Since(hostinfo.lastRoam) < RoamingSuppressSeconds*time.Second {
+		if !hostinfo.lastRoam.IsZero() && udpAddr == hostinfo.lastRoamRemote && time.Since(hostinfo.lastRoam) < RoamingSuppressSeconds*time.Second {
 			if f.l.Level >= logrus.DebugLevel {
-				hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", via.UdpAddr).
+				hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", udpAddr).
 					Debugf("Suppressing roam back to previous remote for %d seconds", RoamingSuppressSeconds)
 			}
 			return
 		}
 
-		hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", via.UdpAddr).
+		hostinfo.logger(f.l).WithField("udpAddr", hostinfo.remote).WithField("newAddr", udpAddr).
 			Info("Host roamed to new udp ip/port.")
 		hostinfo.lastRoam = time.Now()
 		hostinfo.lastRoamRemote = hostinfo.remote
-		hostinfo.SetRemote(via.UdpAddr)
+		hostinfo.SetRemote(udpAddr)
 	}
 
 }
 
-// handleEncrypted returns true if a packet should be processed, false otherwise
+func (f *Interface) handleEncrypted(ci *ConnectionState, addr netip.AddrPort, h *header.H) bool {
-func (f *Interface) handleEncrypted(ci *ConnectionState, via ViaSender, h *header.H) bool {
+	// If connectionstate exists and the replay protector allows, process packet
-	// If connectionstate does not exist, send a recv error, if possible, to encourage a fast reconnect
+	// Else, send recv errors for 300 seconds after a restart to allow fast reconnection.
-	if ci == nil {
+	if ci == nil || !ci.window.Check(f.l, h.MessageCounter) {
-		if !via.IsRelayed {
+		if addr.IsValid() {
-			f.maybeSendRecvError(via.UdpAddr, h.RemoteIndex)
+			f.maybeSendRecvError(addr, h.RemoteIndex)
+			return false
+		} else {
+			return false
 		}
-		return false
-	}
-	// If the window check fails, refuse to process the packet, but don't send a recv error
-	if !ci.window.Check(f.l, h.MessageCounter) {
-		return false
 	}
 
 	return true
@@ -328,35 +313,20 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 	offset := ipv6.HeaderLen // Start at the end of the ipv6 header
 	next := 0
 	for {
-		if protoAt >= dataLen {
+		if dataLen < offset {
 			break
 		}
-		proto := layers.IPProtocol(data[protoAt])
 
+		proto := layers.IPProtocol(data[protoAt])
+		//fmt.Println(proto, protoAt)
 		switch proto {
-		case layers.IPProtocolESP, layers.IPProtocolNoNextHeader:
+		case layers.IPProtocolICMPv6, layers.IPProtocolESP, layers.IPProtocolNoNextHeader:
 			fp.Protocol = uint8(proto)
 			fp.RemotePort = 0
 			fp.LocalPort = 0
 			fp.Fragment = false
 			return nil
 
-		case layers.IPProtocolICMPv6:
-			if dataLen < offset+6 {
-				return ErrIPv6PacketTooShort
-			}
-			fp.Protocol = uint8(proto)
-			fp.LocalPort = 0 //incoming vs outgoing doesn't matter for icmpv6
-			icmptype := data[offset+1]
-			switch icmptype {
-			case layers.ICMPv6TypeEchoRequest, layers.ICMPv6TypeEchoReply:
-				fp.RemotePort = binary.BigEndian.Uint16(data[offset+4 : offset+6]) //identifier
-			default:
-				fp.RemotePort = 0
-			}
-			fp.Fragment = false
-			return nil
-
 		case layers.IPProtocolTCP, layers.IPProtocolUDP:
 			if dataLen < offset+4 {
 				return ErrIPv6PacketTooShort
@@ -396,7 +366,7 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 
 		case layers.IPProtocolAH:
 			// Auth headers, used by IPSec, have a different meaning for header length
-			if dataLen <= offset+1 {
+			if dataLen < offset+1 {
 				break
 			}
 
@@ -404,7 +374,7 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 
 		default:
 			// Normal ipv6 header length processing
-			if dataLen <= offset+1 {
+			if dataLen < offset+1 {
 				break
 			}
 
@@ -446,38 +416,34 @@ func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
 
 	// Accounting for a variable header length, do we have enough data for our src/dst tuples?
 	minLen := ihl
-	if !fp.Fragment {
+	if !fp.Fragment && fp.Protocol != firewall.ProtoICMP {
-		if fp.Protocol == firewall.ProtoICMP {
+		minLen += minFwPacketLen
-			minLen += minFwPacketLen + 2
-		} else {
-			minLen += minFwPacketLen
-		}
 	}
-
 	if len(data) < minLen {
 		return ErrIPv4InvalidHeaderLength
 	}
 
-	if incoming { // Firewall packets are locally oriented
+	// Firewall packets are locally oriented
+	if incoming {
 		fp.RemoteAddr, _ = netip.AddrFromSlice(data[12:16])
 		fp.LocalAddr, _ = netip.AddrFromSlice(data[16:20])
+		if fp.Fragment || fp.Protocol == firewall.ProtoICMP {
+			fp.RemotePort = 0
+			fp.LocalPort = 0
+		} else {
+			fp.RemotePort = binary.BigEndian.Uint16(data[ihl : ihl+2])
+			fp.LocalPort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4])
+		}
 	} else {
 		fp.LocalAddr, _ = netip.AddrFromSlice(data[12:16])
 		fp.RemoteAddr, _ = netip.AddrFromSlice(data[16:20])
-	}
+		if fp.Fragment || fp.Protocol == firewall.ProtoICMP {
-
+			fp.RemotePort = 0
-	if fp.Fragment {
+			fp.LocalPort = 0
-		fp.RemotePort = 0
+		} else {
-		fp.LocalPort = 0
+			fp.LocalPort = binary.BigEndian.Uint16(data[ihl : ihl+2])
-	} else if fp.Protocol == firewall.ProtoICMP { //note that orientation doesn't matter on ICMP
+			fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4])
-		fp.RemotePort = binary.BigEndian.Uint16(data[ihl+4 : ihl+6]) //identifier
+		}
-		fp.LocalPort = 0                                             //code would be uint16(data[ihl+1])
-	} else if incoming {
-		fp.RemotePort = binary.BigEndian.Uint16(data[ihl : ihl+2])  //src port
-		fp.LocalPort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
-	} else {
-		fp.LocalPort = binary.BigEndian.Uint16(data[ihl : ihl+2])    //src port
-		fp.RemotePort = binary.BigEndian.Uint16(data[ihl+2 : ihl+4]) //dst port
 	}
 
 	return nil
@@ -534,8 +500,8 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
 		return false
 	}
 
-	f.connectionManager.In(hostinfo)
+	f.connectionManager.In(hostinfo.localIndexId)
-	err = f.tunCoalescers[q].Add(out)
+	_, err = f.readers[q].Write(out)
 	if err != nil {
 		f.l.WithError(err).Error("Failed to write to tun")
 	}
@@ -543,7 +509,7 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
 }
 
 func (f *Interface) maybeSendRecvError(endpoint netip.AddrPort, index uint32) {
-	if f.sendRecvErrorConfig.ShouldRecvError(endpoint) {
+	if f.sendRecvErrorConfig.ShouldSendRecvError(endpoint) {
 		f.sendRecvError(endpoint, index)
 	}
 }
@@ -561,13 +527,6 @@ func (f *Interface) sendRecvError(endpoint netip.AddrPort, index uint32) {
 }
 
 func (f *Interface) handleRecvError(addr netip.AddrPort, h *header.H) {
-	if !f.acceptRecvErrorConfig.ShouldRecvError(addr) {
-		f.l.WithField("index", h.RemoteIndex).
-			WithField("udpAddr", addr).
-			Debug("Recv error received, ignoring")
-		return
-	}
-
 	if f.l.Level >= logrus.DebugLevel {
 		f.l.WithField("index", h.RemoteIndex).
 			WithField("udpAddr", addr).
@@ -580,6 +539,10 @@ func (f *Interface) handleRecvError(addr netip.AddrPort, h *header.H) {
 		return
 	}
 
+	if !hostinfo.RecvErrorExceeded() {
+		return
+	}
+
 	if hostinfo.remote.IsValid() && hostinfo.remote != addr {
 		f.l.Infoln("Someone spoofing recv_errors? ", addr, hostinfo.remote)
 		return

outside_test.go

@@ -117,46 +117,6 @@ func Test_newPacket_v6(t *testing.T) {
 	err = newPacket(buffer.Bytes(), true, p)
 	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
 
-	// A v6 packet with a hop-by-hop extension
-	// ICMPv6 Payload (Echo Request)
-	icmpLayer := layers.ICMPv6{
-		TypeCode: layers.ICMPv6TypeEchoRequest,
-	}
-	// Hop-by-Hop Extension Header
-	hopOption := layers.IPv6HopByHopOption{}
-	hopOption.OptionData = []byte{0, 0, 0, 0}
-	hopByHop := layers.IPv6HopByHop{}
-	hopByHop.Options = append(hopByHop.Options, &hopOption)
-
-	ip = layers.IPv6{
-		Version:    6,
-		HopLimit:   128,
-		NextHeader: layers.IPProtocolIPv6Destination,
-		SrcIP:      net.IPv6linklocalallrouters,
-		DstIP:      net.IPv6linklocalallnodes,
-	}
-
-	buffer.Clear()
-	err = gopacket.SerializeLayers(buffer, gopacket.SerializeOptions{
-		ComputeChecksums: false,
-		FixLengths:       true,
-	}, &ip, &hopByHop, &icmpLayer)
-	if err != nil {
-		panic(err)
-	}
-	// Ensure buffer length checks during parsing with the next 2 tests.
-
-	// A full IPv6 header and 1 byte in the first extension, but missing
-	// the length byte.
-	err = newPacket(buffer.Bytes()[:41], true, p)
-	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
-
-	// A full IPv6 header plus 1 full extension, but only 1 byte of the
-	// next layer, missing length byte
-	err = newPacket(buffer.Bytes()[:49], true, p)
-	require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
-	err = nil
-
 	// A good ICMP packet
 	ip = layers.IPv6{
 		Version:    6,
@@ -166,26 +126,20 @@ func Test_newPacket_v6(t *testing.T) {
 		DstIP:      net.IPv6linklocalallnodes,
 	}
 
-	icmp := layers.ICMPv6{
+	icmp := layers.ICMPv6{}
-		TypeCode: layers.ICMPv6TypeEchoRequest,
-		Checksum: 0x1234,
-	}
 
 	buffer.Clear()
-	require.NoError(t, gopacket.SerializeLayers(buffer, opt, &ip, &icmp))
+	err = gopacket.SerializeLayers(buffer, opt, &ip, &icmp)
-	require.Error(t, newPacket(buffer.Bytes(), true, p))
+	if err != nil {
-
+		panic(err)
-	buffer.Clear()
-	echo := layers.ICMPv6Echo{
-		Identifier: 0xabcd,
-		SeqNumber:  1234,
 	}
-	require.NoError(t, gopacket.SerializeLayers(buffer, opt, &ip, &icmp, &echo))
+
-	require.NoError(t, newPacket(buffer.Bytes(), true, p))
+	err = newPacket(buffer.Bytes(), true, p)
+	require.NoError(t, err)
 	assert.Equal(t, uint8(layers.IPProtocolICMPv6), p.Protocol)
 	assert.Equal(t, netip.MustParseAddr("ff02::2"), p.RemoteAddr)
 	assert.Equal(t, netip.MustParseAddr("ff02::1"), p.LocalAddr)
-	assert.Equal(t, uint16(0xabcd), p.RemotePort)
+	assert.Equal(t, uint16(0), p.RemotePort)
 	assert.Equal(t, uint16(0), p.LocalPort)
 	assert.False(t, p.Fragment)
 
@@ -334,10 +288,6 @@ func Test_newPacket_v6(t *testing.T) {
 	assert.Equal(t, uint16(22), p.LocalPort)
 	assert.False(t, p.Fragment)
 
-	// Ensure buffer bounds checking during processing
-	err = newPacket(b[:41], true, p)
-	require.ErrorIs(t, err, ErrIPv6PacketTooShort)
-
 	// Invalid AH header
 	b = buffer.Bytes()
 	err = newPacket(b, true, p)
@@ -581,7 +531,7 @@ func BenchmarkParseV6(b *testing.B) {
 	}
 
 	evilBytes := buffer.Bytes()
-	for range 200 {
+	for i := 0; i < 200; i++ {
 		evilBytes = append(evilBytes, hopHeader...)
 	}
 	evilBytes = append(evilBytes, lastHopHeader...)

overlay/coalesce/tcp_coalesce.go

@@ -1,484 +0,0 @@
-package coalesce
-
-import (
-	"bytes"
-	"encoding/binary"
-	"io"
-
-	"github.com/slackhq/nebula/overlay/tio"
-)
-
-// ipProtoTCP is the IANA protocol number for TCP. Hardcoded instead of
-// reaching for golang.org/x/sys/unix — that package doesn't define the
-// constant on Windows, which would break cross-compiles even though this
-// file runs unchanged on every platform.
-const ipProtoTCP = 6
-
-// tcpCoalesceBufSize caps total bytes per superpacket. Mirrors the kernel's
-// sk_gso_max_size of ~64KiB; anything beyond this would be rejected anyway.
-const tcpCoalesceBufSize = 65535
-
-// tcpCoalesceMaxSegs caps how many segments we'll coalesce into a single
-// superpacket. Keeping this well below the kernel's TSO ceiling bounds
-// latency.
-const tcpCoalesceMaxSegs = 64
-
-// tcpCoalesceHdrCap is the scratch space we copy a seed's IP+TCP header
-// into. IPv6 (40) + TCP with full options (60) = 100 bytes.
-const tcpCoalesceHdrCap = 100
-
-// initialSlots is the starting capacity of the slot pool. One flow per
-// packet is the worst case so this matches a typical UDP recvmmsg batch.
-const initialSlots = 64
-
-// flowKey identifies a TCP flow by {src, dst, sport, dport, family}.
-// Comparable, so linear scans over the slot list stay tight.
-type flowKey struct {
-	src, dst     [16]byte
-	sport, dport uint16
-	isV6         bool
-}
-
-// coalesceSlot is one entry in the coalescer's ordered event queue. When
-// passthrough is true the slot holds a single borrowed packet that must be
-// emitted verbatim (non-TCP, non-admissible TCP, or oversize seed). When
-// passthrough is false the slot is an in-progress coalesced superpacket:
-// hdrBuf is a mutable copy of the seed's IP+TCP header (we patch total
-// length and pseudo-header partial at flush), and payIovs are *borrowed*
-// slices from the caller's plaintext buffers — no payload is ever copied.
-// The caller (listenOut) must keep those buffers alive until Flush.
-type coalesceSlot struct {
-	passthrough bool
-	rawPkt      []byte // borrowed when passthrough
-
-	fk       flowKey
-	hdrBuf   [tcpCoalesceHdrCap]byte
-	hdrLen   int
-	ipHdrLen int
-	isV6     bool
-	gsoSize  int
-	numSeg   int
-	totalPay int
-	nextSeq  uint32
-	// psh closes the chain: set when the last-accepted segment had PSH or
-	// was sub-gsoSize. No further appends after that.
-	psh     bool
-	payIovs [][]byte
-}
-
-// TCPCoalescer accumulates adjacent in-flow TCP data segments across
-// multiple concurrent flows and emits each flow's run as a single TSO
-// superpacket via tio.GSOWriter. All output — coalesced or not — is
-// deferred until Flush so arrival order is preserved on the wire. Owns
-// no locks; one coalescer per TUN write queue.
-type TCPCoalescer struct {
-	plainW io.Writer
-	gsoW   tio.GSOWriter // nil when the queue doesn't support TSO
-
-	// slots is the ordered event queue. Flush walks it once and emits each
-	// entry as either a WriteGSO (coalesced) or a plainW.Write (passthrough).
-	slots []*coalesceSlot
-	// openSlots maps a flow key to its most recent non-sealed slot, so new
-	// segments can extend an in-progress superpacket in O(1). Slots are
-	// removed from this map when they close (PSH or short-last-segment),
-	// when a non-admissible packet for that flow arrives, or in Flush.
-	openSlots map[flowKey]*coalesceSlot
-	pool      []*coalesceSlot // free list for reuse
-}
-
-func NewTCPCoalescer(w io.Writer) *TCPCoalescer {
-	c := &TCPCoalescer{
-		plainW:    w,
-		slots:     make([]*coalesceSlot, 0, initialSlots),
-		openSlots: make(map[flowKey]*coalesceSlot, initialSlots),
-		pool:      make([]*coalesceSlot, 0, initialSlots),
-	}
-	if gw, ok := w.(tio.GSOWriter); ok && gw.GSOSupported() {
-		c.gsoW = gw
-	}
-	return c
-}
-
-// parsedTCP holds the fields extracted from a single parse so later steps
-// (admission, slot lookup, canAppend) don't re-walk the header.
-type parsedTCP struct {
-	fk        flowKey
-	ipHdrLen  int
-	tcpHdrLen int
-	hdrLen    int
-	payLen    int
-	seq       uint32
-	flags     byte
-}
-
-// parseTCPBase extracts the flow key and IP/TCP offsets for any TCP packet,
-// regardless of whether it's admissible for coalescing. Returns ok=false
-// for non-TCP or malformed input. Accepts IPv4 (no options, no fragmentation)
-// and IPv6 (no extension headers).
-func parseTCPBase(pkt []byte) (parsedTCP, bool) {
-	var p parsedTCP
-	if len(pkt) < 20 {
-		return p, false
-	}
-	v := pkt[0] >> 4
-	switch v {
-	case 4:
-		ihl := int(pkt[0]&0x0f) * 4
-		if ihl != 20 {
-			return p, false
-		}
-		if pkt[9] != ipProtoTCP {
-			return p, false
-		}
-		// Reject actual fragmentation (MF or non-zero frag offset).
-		if binary.BigEndian.Uint16(pkt[6:8])&0x3fff != 0 {
-			return p, false
-		}
-		totalLen := int(binary.BigEndian.Uint16(pkt[2:4]))
-		if totalLen > len(pkt) || totalLen < ihl {
-			return p, false
-		}
-		p.ipHdrLen = 20
-		p.fk.isV6 = false
-		copy(p.fk.src[:4], pkt[12:16])
-		copy(p.fk.dst[:4], pkt[16:20])
-		pkt = pkt[:totalLen]
-	case 6:
-		if len(pkt) < 40 {
-			return p, false
-		}
-		if pkt[6] != ipProtoTCP {
-			return p, false
-		}
-		payloadLen := int(binary.BigEndian.Uint16(pkt[4:6]))
-		if 40+payloadLen > len(pkt) {
-			return p, false
-		}
-		p.ipHdrLen = 40
-		p.fk.isV6 = true
-		copy(p.fk.src[:], pkt[8:24])
-		copy(p.fk.dst[:], pkt[24:40])
-		pkt = pkt[:40+payloadLen]
-	default:
-		return p, false
-	}
-
-	if len(pkt) < p.ipHdrLen+20 {
-		return p, false
-	}
-	tcpOff := int(pkt[p.ipHdrLen+12]>>4) * 4
-	if tcpOff < 20 || tcpOff > 60 {
-		return p, false
-	}
-	if len(pkt) < p.ipHdrLen+tcpOff {
-		return p, false
-	}
-	p.tcpHdrLen = tcpOff
-	p.hdrLen = p.ipHdrLen + tcpOff
-	p.payLen = len(pkt) - p.hdrLen
-	p.seq = binary.BigEndian.Uint32(pkt[p.ipHdrLen+4 : p.ipHdrLen+8])
-	p.flags = pkt[p.ipHdrLen+13]
-	p.fk.sport = binary.BigEndian.Uint16(pkt[p.ipHdrLen : p.ipHdrLen+2])
-	p.fk.dport = binary.BigEndian.Uint16(pkt[p.ipHdrLen+2 : p.ipHdrLen+4])
-	return p, true
-}
-
-// coalesceable reports whether a parsed TCP segment is eligible for
-// coalescing. Accepts only ACK or ACK|PSH with a non-empty payload.
-func (p parsedTCP) coalesceable() bool {
-	const ack = 0x10
-	const psh = 0x08
-	if p.flags&^(ack|psh) != 0 || p.flags&ack == 0 {
-		return false
-	}
-	return p.payLen > 0
-}
-
-// Add borrows pkt. The caller must keep pkt valid until the next Flush,
-// whether or not the packet was coalesced — passthrough (non-admissible)
-// packets are queued and written at Flush time, not synchronously.
-func (c *TCPCoalescer) Add(pkt []byte) error {
-	if c.gsoW == nil {
-		c.addPassthrough(pkt)
-		return nil
-	}
-
-	info, ok := parseTCPBase(pkt)
-	if !ok {
-		// Non-TCP or malformed — can't possibly collide with an open flow.
-		c.addPassthrough(pkt)
-		return nil
-	}
-	if !info.coalesceable() {
-		// TCP but not admissible (SYN/FIN/RST/URG/CWR/ECE or zero-payload).
-		// Seal this flow's open slot so later in-flow packets don't extend
-		// it and accidentally reorder past this passthrough.
-		delete(c.openSlots, info.fk)
-		c.addPassthrough(pkt)
-		return nil
-	}
-
-	if open := c.openSlots[info.fk]; open != nil {
-		if c.canAppend(open, pkt, info) {
-			c.appendPayload(open, pkt, info)
-			if open.psh {
-				delete(c.openSlots, info.fk)
-			}
-			return nil
-		}
-		// Can't extend — seal it and fall through to seed a fresh slot.
-		delete(c.openSlots, info.fk)
-	}
-	c.seed(pkt, info)
-	return nil
-}
-
-// Flush emits every queued event in arrival order. Coalesced slots go out
-// via WriteGSO; passthrough slots go out via plainW.Write. Returns the
-// first error observed; keeps draining so one bad packet doesn't hold up
-// the rest. After Flush returns, borrowed payload slices may be recycled.
-func (c *TCPCoalescer) Flush() error {
-	var first error
-	for _, s := range c.slots {
-		var err error
-		if s.passthrough {
-			_, err = c.plainW.Write(s.rawPkt)
-		} else {
-			err = c.flushSlot(s)
-		}
-		if err != nil && first == nil {
-			first = err
-		}
-		c.release(s)
-	}
-	for i := range c.slots {
-		c.slots[i] = nil
-	}
-	c.slots = c.slots[:0]
-	for k := range c.openSlots {
-		delete(c.openSlots, k)
-	}
-	return first
-}
-
-func (c *TCPCoalescer) addPassthrough(pkt []byte) {
-	s := c.take()
-	s.passthrough = true
-	s.rawPkt = pkt
-	c.slots = append(c.slots, s)
-}
-
-func (c *TCPCoalescer) seed(pkt []byte, info parsedTCP) {
-	if info.hdrLen > tcpCoalesceHdrCap || info.hdrLen+info.payLen > tcpCoalesceBufSize {
-		// Pathological shape — can't fit our scratch, emit as-is.
-		c.addPassthrough(pkt)
-		return
-	}
-	s := c.take()
-	s.passthrough = false
-	s.rawPkt = nil
-	copy(s.hdrBuf[:], pkt[:info.hdrLen])
-	s.hdrLen = info.hdrLen
-	s.ipHdrLen = info.ipHdrLen
-	s.isV6 = info.fk.isV6
-	s.fk = info.fk
-	s.gsoSize = info.payLen
-	s.numSeg = 1
-	s.totalPay = info.payLen
-	s.nextSeq = info.seq + uint32(info.payLen)
-	s.psh = info.flags&0x08 != 0
-	s.payIovs = append(s.payIovs[:0], pkt[info.hdrLen:info.hdrLen+info.payLen])
-	c.slots = append(c.slots, s)
-	if !s.psh {
-		c.openSlots[info.fk] = s
-	}
-}
-
-// canAppend reports whether info's packet extends the slot's seed: same
-// header shape and stable contents, adjacent seq, not oversized, chain not
-// closed.
-func (c *TCPCoalescer) canAppend(s *coalesceSlot, pkt []byte, info parsedTCP) bool {
-	if s.psh {
-		return false
-	}
-	if info.hdrLen != s.hdrLen {
-		return false
-	}
-	if info.seq != s.nextSeq {
-		return false
-	}
-	if s.numSeg >= tcpCoalesceMaxSegs {
-		return false
-	}
-	if info.payLen > s.gsoSize {
-		return false
-	}
-	if s.hdrLen+s.totalPay+info.payLen > tcpCoalesceBufSize {
-		return false
-	}
-	if !headersMatch(s.hdrBuf[:s.hdrLen], pkt[:info.hdrLen], s.isV6, s.ipHdrLen) {
-		return false
-	}
-	return true
-}
-
-func (c *TCPCoalescer) appendPayload(s *coalesceSlot, pkt []byte, info parsedTCP) {
-	s.payIovs = append(s.payIovs, pkt[info.hdrLen:info.hdrLen+info.payLen])
-	s.numSeg++
-	s.totalPay += info.payLen
-	s.nextSeq = info.seq + uint32(info.payLen)
-	if info.payLen < s.gsoSize || info.flags&0x08 != 0 {
-		s.psh = true
-	}
-}
-
-func (c *TCPCoalescer) take() *coalesceSlot {
-	if n := len(c.pool); n > 0 {
-		s := c.pool[n-1]
-		c.pool[n-1] = nil
-		c.pool = c.pool[:n-1]
-		return s
-	}
-	return &coalesceSlot{}
-}
-
-func (c *TCPCoalescer) release(s *coalesceSlot) {
-	s.passthrough = false
-	s.rawPkt = nil
-	for i := range s.payIovs {
-		s.payIovs[i] = nil
-	}
-	s.payIovs = s.payIovs[:0]
-	s.numSeg = 0
-	s.totalPay = 0
-	s.psh = false
-	c.pool = append(c.pool, s)
-}
-
-// flushSlot patches the header and calls WriteGSO. Does not remove the
-// slot from c.slots.
-func (c *TCPCoalescer) flushSlot(s *coalesceSlot) error {
-	total := s.hdrLen + s.totalPay
-	l4Len := total - s.ipHdrLen
-	hdr := s.hdrBuf[:s.hdrLen]
-
-	if s.isV6 {
-		binary.BigEndian.PutUint16(hdr[4:6], uint16(l4Len))
-	} else {
-		binary.BigEndian.PutUint16(hdr[2:4], uint16(total))
-		hdr[10] = 0
-		hdr[11] = 0
-		binary.BigEndian.PutUint16(hdr[10:12], ipv4HdrChecksum(hdr[:s.ipHdrLen]))
-	}
-
-	var psum uint32
-	if s.isV6 {
-		psum = pseudoSumIPv6(hdr[8:24], hdr[24:40], ipProtoTCP, l4Len)
-	} else {
-		psum = pseudoSumIPv4(hdr[12:16], hdr[16:20], ipProtoTCP, l4Len)
-	}
-	tcsum := s.ipHdrLen + 16
-	binary.BigEndian.PutUint16(hdr[tcsum:tcsum+2], foldOnceNoInvert(psum))
-
-	return c.gsoW.WriteGSO(hdr, s.payIovs, uint16(s.gsoSize), s.isV6, uint16(s.ipHdrLen))
-}
-
-// headersMatch compares two IP+TCP header prefixes for byte-for-byte
-// equality on every field that must be identical across coalesced
-// segments. Size/IPID/IPCsum/seq/flags/tcpCsum are masked out.
-func headersMatch(a, b []byte, isV6 bool, ipHdrLen int) bool {
-	if len(a) != len(b) {
-		return false
-	}
-	if isV6 {
-		// IPv6: bytes [0:4] = version/TC/flow-label, [6:8] = next_hdr/hop,
-		// [8:40] = src+dst. Skip [4:6] payload length.
-		if !bytes.Equal(a[0:4], b[0:4]) {
-			return false
-		}
-		if !bytes.Equal(a[6:40], b[6:40]) {
-			return false
-		}
-	} else {
-		// IPv4: [0:2] version/IHL/TOS, [6:10] flags/fragoff/TTL/proto,
-		// [12:20] src+dst. Skip [2:4] total len, [4:6] id, [10:12] csum.
-		if !bytes.Equal(a[0:2], b[0:2]) {
-			return false
-		}
-		if !bytes.Equal(a[6:10], b[6:10]) {
-			return false
-		}
-		if !bytes.Equal(a[12:20], b[12:20]) {
-			return false
-		}
-	}
-	// TCP: compare [0:4] ports, [8:13] ack+dataoff, [14:16] window,
-	// [18:tcpHdrLen] options (incl. urgent).
-	tcp := ipHdrLen
-	if !bytes.Equal(a[tcp:tcp+4], b[tcp:tcp+4]) {
-		return false
-	}
-	if !bytes.Equal(a[tcp+8:tcp+13], b[tcp+8:tcp+13]) {
-		return false
-	}
-	if !bytes.Equal(a[tcp+14:tcp+16], b[tcp+14:tcp+16]) {
-		return false
-	}
-	if !bytes.Equal(a[tcp+18:], b[tcp+18:]) {
-		return false
-	}
-	return true
-}
-
-// ipv4HdrChecksum computes the IPv4 header checksum over hdr (which must
-// already have its checksum field zeroed) and returns the folded/inverted
-// 16-bit value to store.
-func ipv4HdrChecksum(hdr []byte) uint16 {
-	var sum uint32
-	for i := 0; i+1 < len(hdr); i += 2 {
-		sum += uint32(binary.BigEndian.Uint16(hdr[i : i+2]))
-	}
-	if len(hdr)%2 == 1 {
-		sum += uint32(hdr[len(hdr)-1]) << 8
-	}
-	for sum>>16 != 0 {
-		sum = (sum & 0xffff) + (sum >> 16)
-	}
-	return ^uint16(sum)
-}
-
-// pseudoSumIPv4 / pseudoSumIPv6 build the TCP pseudo-header partial sum
-// expected by the virtio NEEDS_CSUM kernel path: the 32-bit accumulator
-// before folding.
-func pseudoSumIPv4(src, dst []byte, proto byte, l4Len int) uint32 {
-	var sum uint32
-	sum += uint32(binary.BigEndian.Uint16(src[0:2]))
-	sum += uint32(binary.BigEndian.Uint16(src[2:4]))
-	sum += uint32(binary.BigEndian.Uint16(dst[0:2]))
-	sum += uint32(binary.BigEndian.Uint16(dst[2:4]))
-	sum += uint32(proto)
-	sum += uint32(l4Len)
-	return sum
-}
-
-func pseudoSumIPv6(src, dst []byte, proto byte, l4Len int) uint32 {
-	var sum uint32
-	for i := 0; i < 16; i += 2 {
-		sum += uint32(binary.BigEndian.Uint16(src[i : i+2]))
-		sum += uint32(binary.BigEndian.Uint16(dst[i : i+2]))
-	}
-	sum += uint32(l4Len >> 16)
-	sum += uint32(l4Len & 0xffff)
-	sum += uint32(proto)
-	return sum
-}
-
-// foldOnceNoInvert folds the 32-bit accumulator to 16 bits and returns it
-// unchanged (no one's complement). This is what virtio NEEDS_CSUM wants in
-// the L4 checksum field — the kernel will add the payload sum and invert.
-func foldOnceNoInvert(sum uint32) uint16 {
-	for sum>>16 != 0 {
-		sum = (sum & 0xffff) + (sum >> 16)
-	}
-	return uint16(sum)
-}

overlay/coalesce/tcp_coalesce_test.go

@@ -1,576 +0,0 @@
-package coalesce
-
-import (
-	"encoding/binary"
-	"testing"
-)
-
-// fakeTunWriter records plain Writes and WriteGSO calls without touching a
-// real TUN fd. WriteGSO preserves the split between hdr and borrowed pays
-// so tests can inspect each independently.
-type fakeTunWriter struct {
-	gsoEnabled bool
-	writes     [][]byte
-	gsoWrites  []fakeGSOWrite
-}
-
-type fakeGSOWrite struct {
-	hdr       []byte
-	pays      [][]byte
-	gsoSize   uint16
-	isV6      bool
-	csumStart uint16
-}
-
-// total returns hdrLen + sum of pay lens.
-func (g fakeGSOWrite) total() int {
-	n := len(g.hdr)
-	for _, p := range g.pays {
-		n += len(p)
-	}
-	return n
-}
-
-// payLen sums the pays.
-func (g fakeGSOWrite) payLen() int {
-	var n int
-	for _, p := range g.pays {
-		n += len(p)
-	}
-	return n
-}
-
-func (w *fakeTunWriter) Write(p []byte) (int, error) {
-	buf := make([]byte, len(p))
-	copy(buf, p)
-	w.writes = append(w.writes, buf)
-	return len(p), nil
-}
-
-func (w *fakeTunWriter) WriteGSO(hdr []byte, pays [][]byte, gsoSize uint16, isV6 bool, csumStart uint16) error {
-	hcopy := make([]byte, len(hdr))
-	copy(hcopy, hdr)
-	paysCopy := make([][]byte, len(pays))
-	for i, p := range pays {
-		pc := make([]byte, len(p))
-		copy(pc, p)
-		paysCopy[i] = pc
-	}
-	w.gsoWrites = append(w.gsoWrites, fakeGSOWrite{
-		hdr:       hcopy,
-		pays:      paysCopy,
-		gsoSize:   gsoSize,
-		isV6:      isV6,
-		csumStart: csumStart,
-	})
-	return nil
-}
-
-func (w *fakeTunWriter) GSOSupported() bool { return w.gsoEnabled }
-
-// buildTCPv4 constructs a minimal IPv4+TCP packet with the given payload,
-// seq, and flags. Assumes no IP options and a 20-byte TCP header.
-func buildTCPv4(seq uint32, flags byte, payload []byte) []byte {
-	return buildTCPv4Ports(1000, 2000, seq, flags, payload)
-}
-
-// buildTCPv4Ports is buildTCPv4 with caller-specified ports so tests can
-// build distinct flows.
-func buildTCPv4Ports(sport, dport uint16, seq uint32, flags byte, payload []byte) []byte {
-	const ipHdrLen = 20
-	const tcpHdrLen = 20
-	total := ipHdrLen + tcpHdrLen + len(payload)
-	pkt := make([]byte, total)
-
-	pkt[0] = 0x45
-	pkt[1] = 0x00
-	binary.BigEndian.PutUint16(pkt[2:4], uint16(total))
-	binary.BigEndian.PutUint16(pkt[4:6], 0)
-	binary.BigEndian.PutUint16(pkt[6:8], 0x4000)
-	pkt[8] = 64
-	pkt[9] = ipProtoTCP
-	copy(pkt[12:16], []byte{10, 0, 0, 1})
-	copy(pkt[16:20], []byte{10, 0, 0, 2})
-
-	binary.BigEndian.PutUint16(pkt[20:22], sport)
-	binary.BigEndian.PutUint16(pkt[22:24], dport)
-	binary.BigEndian.PutUint32(pkt[24:28], seq)
-	binary.BigEndian.PutUint32(pkt[28:32], 12345)
-	pkt[32] = 0x50
-	pkt[33] = flags
-	binary.BigEndian.PutUint16(pkt[34:36], 0xffff)
-
-	copy(pkt[40:], payload)
-	return pkt
-}
-
-const (
-	tcpAck    = 0x10
-	tcpPsh    = 0x08
-	tcpSyn    = 0x02
-	tcpFin    = 0x01
-	tcpAckPsh = tcpAck | tcpPsh
-)
-
-func TestCoalescerPassthroughWhenGSOUnavailable(t *testing.T) {
-	w := &fakeTunWriter{gsoEnabled: false}
-	c := NewTCPCoalescer(w)
-	pkt := buildTCPv4(1000, tcpAck, []byte("hello"))
-	if err := c.Add(pkt); err != nil {
-		t.Fatal(err)
-	}
-	// No sync write — passthrough is deferred to Flush.
-	if len(w.writes) != 0 || len(w.gsoWrites) != 0 {
-		t.Fatalf("no Add-time writes: got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
-	}
-	if err := c.Flush(); err != nil {
-		t.Fatal(err)
-	}
-	if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
-		t.Fatalf("want single plain write, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
-	}
-}
-
-func TestCoalescerNonTCPPassthrough(t *testing.T) {
-	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w)
-	pkt := make([]byte, 28)
-	pkt[0] = 0x45
-	binary.BigEndian.PutUint16(pkt[2:4], 28)
-	pkt[9] = 1
-	copy(pkt[12:16], []byte{10, 0, 0, 1})
-	copy(pkt[16:20], []byte{10, 0, 0, 2})
-	if err := c.Add(pkt); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Flush(); err != nil {
-		t.Fatal(err)
-	}
-	if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
-		t.Fatalf("ICMP should pass through unchanged")
-	}
-}
-
-func TestCoalescerSeedThenFlushAlone(t *testing.T) {
-	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w)
-	pkt := buildTCPv4(1000, tcpAck, make([]byte, 1000))
-	if err := c.Add(pkt); err != nil {
-		t.Fatal(err)
-	}
-	if len(w.writes) != 0 || len(w.gsoWrites) != 0 {
-		t.Fatalf("unexpected output before flush")
-	}
-	if err := c.Flush(); err != nil {
-		t.Fatal(err)
-	}
-	// Single-segment flush now goes through WriteGSO with GSO_NONE
-	// (virtio NEEDS_CSUM lets the kernel fill in the L4 csum).
-	if len(w.gsoWrites) != 1 || len(w.writes) != 0 {
-		t.Fatalf("single-seg flush: writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
-	}
-	g := w.gsoWrites[0]
-	if g.total() != 40+1000 {
-		t.Errorf("super total=%d want %d", g.total(), 40+1000)
-	}
-	if g.payLen() != 1000 {
-		t.Errorf("payLen=%d want 1000", g.payLen())
-	}
-}
-
-func TestCoalescerCoalescesAdjacentACKs(t *testing.T) {
-	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w)
-	pay := make([]byte, 1200)
-	if err := c.Add(buildTCPv4(1000, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Add(buildTCPv4(2200, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Add(buildTCPv4(3400, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Flush(); err != nil {
-		t.Fatal(err)
-	}
-	if len(w.gsoWrites) != 1 {
-		t.Fatalf("want 1 gso write, got %d (plain=%d)", len(w.gsoWrites), len(w.writes))
-	}
-	g := w.gsoWrites[0]
-	if g.gsoSize != 1200 {
-		t.Errorf("gsoSize=%d want 1200", g.gsoSize)
-	}
-	if len(g.hdr) != 40 {
-		t.Errorf("hdrLen=%d want 40", len(g.hdr))
-	}
-	if g.csumStart != 20 {
-		t.Errorf("csumStart=%d want 20", g.csumStart)
-	}
-	if len(g.pays) != 3 {
-		t.Errorf("pay count=%d want 3", len(g.pays))
-	}
-	if g.total() != 40+3*1200 {
-		t.Errorf("superpacket len=%d want %d", g.total(), 40+3*1200)
-	}
-	if tot := binary.BigEndian.Uint16(g.hdr[2:4]); int(tot) != g.total() {
-		t.Errorf("ip total_length=%d want %d", tot, g.total())
-	}
-}
-
-func TestCoalescerRejectsSeqGap(t *testing.T) {
-	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w)
-	pay := make([]byte, 1200)
-	if err := c.Add(buildTCPv4(1000, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Add(buildTCPv4(3000, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Flush(); err != nil {
-		t.Fatal(err)
-	}
-	// Each packet flushes as its own single-segment WriteGSO now.
-	if len(w.gsoWrites) != 2 || len(w.writes) != 0 {
-		t.Fatalf("seq gap: want 2 gso writes got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
-	}
-}
-
-func TestCoalescerRejectsFlagMismatch(t *testing.T) {
-	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w)
-	pay := make([]byte, 1200)
-	if err := c.Add(buildTCPv4(1000, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	// SYN|ACK is non-admissible. Must flush matching flow's slot (gso)
-	// and then plain-write the SYN packet itself.
-	syn := buildTCPv4(2200, tcpSyn|tcpAck, pay)
-	if err := c.Add(syn); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Flush(); err != nil {
-		t.Fatal(err)
-	}
-	if len(w.writes) != 1 || len(w.gsoWrites) != 1 {
-		t.Fatalf("flag mismatch: want 1 plain + 1 gso, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
-	}
-}
-
-func TestCoalescerRejectsFIN(t *testing.T) {
-	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w)
-	fin := buildTCPv4(1000, tcpAck|tcpFin, []byte("x"))
-	if err := c.Add(fin); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Flush(); err != nil {
-		t.Fatal(err)
-	}
-	// FIN isn't admissible — passthrough as plain, no slot, no gso.
-	if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
-		t.Fatalf("FIN should be passthrough, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
-	}
-}
-
-func TestCoalescerShortLastSegmentClosesChain(t *testing.T) {
-	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w)
-	full := make([]byte, 1200)
-	half := make([]byte, 500)
-	if err := c.Add(buildTCPv4(1000, tcpAck, full)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Add(buildTCPv4(2200, tcpAck, half)); err != nil {
-		t.Fatal(err)
-	}
-	// Chain now closed; next packet seeds a new slot on the same flow
-	// after flushing the old one.
-	if err := c.Add(buildTCPv4(2700, tcpAck, full)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Flush(); err != nil {
-		t.Fatal(err)
-	}
-	// Expect two gso writes: the first two packets coalesced, then the
-	// third flushed alone (single-seg via GSO_NONE).
-	if len(w.gsoWrites) != 2 {
-		t.Fatalf("want 2 gso writes got %d", len(w.gsoWrites))
-	}
-	if len(w.writes) != 0 {
-		t.Fatalf("want 0 plain writes got %d", len(w.writes))
-	}
-	if w.gsoWrites[0].gsoSize != 1200 {
-		t.Errorf("gsoSize=%d want 1200", w.gsoWrites[0].gsoSize)
-	}
-	if got, want := w.gsoWrites[0].total(), 40+1200+500; got != want {
-		t.Errorf("super len=%d want %d", got, want)
-	}
-}
-
-func TestCoalescerPSHFinalizesChain(t *testing.T) {
-	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w)
-	pay := make([]byte, 1200)
-	if err := c.Add(buildTCPv4(1000, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Add(buildTCPv4(2200, tcpAckPsh, pay)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Add(buildTCPv4(3400, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Flush(); err != nil {
-		t.Fatal(err)
-	}
-	// First two coalesce; the third seeds a fresh slot that flushes alone.
-	if len(w.gsoWrites) != 2 {
-		t.Fatalf("want 2 gso writes got %d", len(w.gsoWrites))
-	}
-	if len(w.writes) != 0 {
-		t.Fatalf("want 0 plain writes got %d", len(w.writes))
-	}
-}
-
-func TestCoalescerRejectsDifferentFlow(t *testing.T) {
-	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w)
-	pay := make([]byte, 1200)
-	p1 := buildTCPv4(1000, tcpAck, pay)
-	p2 := buildTCPv4(2200, tcpAck, pay)
-	binary.BigEndian.PutUint16(p2[20:22], 9999)
-	if err := c.Add(p1); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Add(p2); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Flush(); err != nil {
-		t.Fatal(err)
-	}
-	// Two independent flows, each flushes its own single-segment WriteGSO.
-	if len(w.gsoWrites) != 2 || len(w.writes) != 0 {
-		t.Fatalf("diff flow: want 2 gso writes got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
-	}
-}
-
-func TestCoalescerRejectsIPOptions(t *testing.T) {
-	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w)
-	pay := make([]byte, 500)
-	pkt := buildTCPv4(1000, tcpAck, pay)
-	// Bump IHL to 6 to simulate 4 bytes of IP options. Don't actually add
-	// bytes — parser should bail before it matters.
-	pkt[0] = 0x46
-	if err := c.Add(pkt); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Flush(); err != nil {
-		t.Fatal(err)
-	}
-	// Non-admissible parse → passthrough as plain.
-	if len(w.writes) != 1 || len(w.gsoWrites) != 0 {
-		t.Fatalf("IP options should passthrough, got writes=%d gso=%d", len(w.writes), len(w.gsoWrites))
-	}
-}
-
-func TestCoalescerCapBySegments(t *testing.T) {
-	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w)
-	pay := make([]byte, 512)
-	seq := uint32(1000)
-	for i := 0; i < tcpCoalesceMaxSegs+5; i++ {
-		if err := c.Add(buildTCPv4(seq, tcpAck, pay)); err != nil {
-			t.Fatal(err)
-		}
-		seq += uint32(len(pay))
-	}
-	if err := c.Flush(); err != nil {
-		t.Fatal(err)
-	}
-	for _, g := range w.gsoWrites {
-		segs := len(g.pays)
-		if segs > tcpCoalesceMaxSegs {
-			t.Fatalf("super exceeded seg cap: %d > %d", segs, tcpCoalesceMaxSegs)
-		}
-	}
-}
-
-// TestCoalescerMultipleFlowsInSameBatch proves two interleaved bulk TCP
-// flows coalesce independently in a single Flush.
-func TestCoalescerMultipleFlowsInSameBatch(t *testing.T) {
-	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w)
-	pay := make([]byte, 1200)
-
-	// Flow A: sport 1000. Flow B: sport 3000.
-	if err := c.Add(buildTCPv4Ports(1000, 2000, 100, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Add(buildTCPv4Ports(3000, 2000, 500, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Add(buildTCPv4Ports(1000, 2000, 1300, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Add(buildTCPv4Ports(3000, 2000, 1700, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Add(buildTCPv4Ports(1000, 2000, 2500, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Add(buildTCPv4Ports(3000, 2000, 2900, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Flush(); err != nil {
-		t.Fatal(err)
-	}
-
-	if len(w.gsoWrites) != 2 {
-		t.Fatalf("want 2 gso writes (one per flow), got %d", len(w.gsoWrites))
-	}
-	if len(w.writes) != 0 {
-		t.Fatalf("want no plain writes, got %d", len(w.writes))
-	}
-	// Each superpacket should carry 3 segments.
-	for i, g := range w.gsoWrites {
-		if len(g.pays) != 3 {
-			t.Errorf("gso[%d]: segs=%d want 3", i, len(g.pays))
-		}
-		if g.gsoSize != 1200 {
-			t.Errorf("gso[%d]: gsoSize=%d want 1200", i, g.gsoSize)
-		}
-	}
-	// Verify each superpacket carries the source port it was seeded with.
-	seenSports := map[uint16]bool{}
-	for _, g := range w.gsoWrites {
-		sp := binary.BigEndian.Uint16(g.hdr[20:22])
-		seenSports[sp] = true
-	}
-	if !seenSports[1000] || !seenSports[3000] {
-		t.Errorf("expected superpackets for sports 1000 and 3000, got %v", seenSports)
-	}
-}
-
-// TestCoalescerPreservesArrivalOrder confirms that with passthrough and
-// coalesced events both queued, Flush emits them in Add order rather than
-// writing passthrough packets synchronously.
-func TestCoalescerPreservesArrivalOrder(t *testing.T) {
-	w := &orderedFakeWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w)
-	// Sequence: coalesceable TCP, ICMP (passthrough), coalesceable TCP on
-	// a different flow. Expected emit order: gso(X), plain(ICMP), gso(Y).
-	pay := make([]byte, 1200)
-	if err := c.Add(buildTCPv4Ports(1000, 2000, 100, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	icmp := make([]byte, 28)
-	icmp[0] = 0x45
-	binary.BigEndian.PutUint16(icmp[2:4], 28)
-	icmp[9] = 1
-	copy(icmp[12:16], []byte{10, 0, 0, 1})
-	copy(icmp[16:20], []byte{10, 0, 0, 3})
-	if err := c.Add(icmp); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Add(buildTCPv4Ports(3000, 2000, 500, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	// Nothing should have hit the writer synchronously.
-	if len(w.events) != 0 {
-		t.Fatalf("Add emitted events synchronously: %v", w.events)
-	}
-	if err := c.Flush(); err != nil {
-		t.Fatal(err)
-	}
-	if got, want := w.events, []string{"gso", "plain", "gso"}; !stringSliceEq(got, want) {
-		t.Fatalf("flush order=%v want %v", got, want)
-	}
-}
-
-// orderedFakeWriter records only the sequence of call types so tests can
-// assert arrival order without inspecting bytes.
-type orderedFakeWriter struct {
-	gsoEnabled bool
-	events     []string
-}
-
-func (w *orderedFakeWriter) Write(p []byte) (int, error) {
-	w.events = append(w.events, "plain")
-	return len(p), nil
-}
-
-func (w *orderedFakeWriter) WriteGSO(hdr []byte, pays [][]byte, gsoSize uint16, isV6 bool, csumStart uint16) error {
-	w.events = append(w.events, "gso")
-	return nil
-}
-
-func (w *orderedFakeWriter) GSOSupported() bool { return w.gsoEnabled }
-
-func stringSliceEq(a, b []string) bool {
-	if len(a) != len(b) {
-		return false
-	}
-	for i := range a {
-		if a[i] != b[i] {
-			return false
-		}
-	}
-	return true
-}
-
-// TestCoalescerInterleavedFlowsPreserveOrdering checks that a non-admissible
-// packet (SYN) mid-flow only flushes its own flow, not others.
-func TestCoalescerInterleavedFlowsPreserveOrdering(t *testing.T) {
-	w := &fakeTunWriter{gsoEnabled: true}
-	c := NewTCPCoalescer(w)
-	pay := make([]byte, 1200)
-
-	// Flow A two segments.
-	if err := c.Add(buildTCPv4Ports(1000, 2000, 100, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Add(buildTCPv4Ports(1000, 2000, 1300, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	// Flow B two segments.
-	if err := c.Add(buildTCPv4Ports(3000, 2000, 500, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Add(buildTCPv4Ports(3000, 2000, 1700, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	// Flow A SYN (non-admissible) — must flush only flow A's slot.
-	syn := buildTCPv4Ports(1000, 2000, 9999, tcpSyn|tcpAck, pay)
-	if err := c.Add(syn); err != nil {
-		t.Fatal(err)
-	}
-	// Flow B continues — should still be coalesced with its seed.
-	if err := c.Add(buildTCPv4Ports(3000, 2000, 2900, tcpAck, pay)); err != nil {
-		t.Fatal(err)
-	}
-	if err := c.Flush(); err != nil {
-		t.Fatal(err)
-	}
-
-	// Expected:
-	//   - 1 gso for flow A (first 2 segments)
-	//   - 1 plain for flow A SYN
-	//   - 1 gso for flow B (3 segments)
-	if len(w.gsoWrites) != 2 {
-		t.Fatalf("want 2 gso writes, got %d", len(w.gsoWrites))
-	}
-	if len(w.writes) != 1 {
-		t.Fatalf("want 1 plain write (SYN), got %d", len(w.writes))
-	}
-	// Find the 3-segment gso (flow B) and the 2-segment gso (flow A).
-	var segCounts []int
-	for _, g := range w.gsoWrites {
-		segCounts = append(segCounts, len(g.pays))
-	}
-	if !(segCounts[0] == 2 && segCounts[1] == 3) && !(segCounts[0] == 3 && segCounts[1] == 2) {
-		t.Errorf("unexpected segment counts: %v (want 2 and 3)", segCounts)
-	}
-}

overlay/device.go

@@ -3,22 +3,13 @@ package overlay
 import (
 	"io"
 	"net/netip"
-
-	"github.com/slackhq/nebula/overlay/tio"
-	"github.com/slackhq/nebula/routing"
 )
 
-// defaultBatchBufSize is the per-Queue scratch size for Read on backends
-// that don't do TSO segmentation. 65535 covers any single IP packet.
-const defaultBatchBufSize = 65535
-
 type Device interface {
-	io.Closer
+	io.ReadWriteCloser
 	Activate() error
 	Networks() []netip.Prefix
 	Name() string
-	RoutesFor(netip.Addr) routing.Gateways
+	RouteFor(netip.Addr) netip.Addr
-	SupportsMultiqueue() bool //todo remove?
+	NewMultiQueueReader() (io.ReadWriteCloser, error)
-	NewMultiQueueReader() error
-	Readers() []tio.Queue
 }

overlay/noop.go

@@ -1,55 +0,0 @@
-package overlay
-
-import (
-	"errors"
-	"net/netip"
-
-	"github.com/slackhq/nebula/overlay/tio"
-	"github.com/slackhq/nebula/routing"
-)
-
-type NoopTun struct{}
-
-func (NoopTun) RoutesFor(addr netip.Addr) routing.Gateways {
-	return routing.Gateways{}
-}
-
-func (NoopTun) Activate() error {
-	return nil
-}
-
-func (NoopTun) Networks() []netip.Prefix {
-	return []netip.Prefix{}
-}
-
-func (NoopTun) Name() string {
-	return "noop"
-}
-
-func (NoopTun) Read() ([][]byte, error) {
-	return nil, nil
-}
-
-func (NoopTun) Write([]byte) (int, error) {
-	return 0, nil
-}
-
-func (NoopTun) WriteReject(p []byte) (int, error) {
-	return 0, nil
-}
-
-func (NoopTun) SupportsMultiqueue() bool {
-	return false
-}
-
-func (NoopTun) NewMultiQueueReader() error {
-	return errors.New("unsupported")
-}
-
-func (NoopTun) Readers() []tio.Queue {
-	return []tio.Queue{NoopTun{}}
-}
-
-func (NoopTun) Close() error {
-	return nil
-}

overlay/route.go

@@ -11,14 +11,13 @@ import (
 	"github.com/gaissmai/bart"
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/routing"
 )
 
 type Route struct {
 	MTU     int
 	Metric  int
 	Cidr    netip.Prefix
-	Via     routing.Gateways
+	Via     netip.Addr
 	Install bool
 }
 
@@ -48,17 +47,15 @@ func (r Route) String() string {
 	return s
 }
 
-func makeRouteTree(l *logrus.Logger, routes []Route, allowMTU bool) (*bart.Table[routing.Gateways], error) {
+func makeRouteTree(l *logrus.Logger, routes []Route, allowMTU bool) (*bart.Table[netip.Addr], error) {
-	routeTree := new(bart.Table[routing.Gateways])
+	routeTree := new(bart.Table[netip.Addr])
 	for _, r := range routes {
 		if !allowMTU && r.MTU > 0 {
 			l.WithField("route", r).Warnf("route MTU is not supported in %s", runtime.GOOS)
 		}
 
-		gateways := r.Via
+		if r.Via.IsValid() {
-		if len(gateways) > 0 {
+			routeTree.Insert(r.Cidr, r.Via)
-			routing.CalculateBucketsForGateways(gateways)
-			routeTree.Insert(r.Cidr, gateways)
 		}
 	}
 	return routeTree, nil
@@ -72,7 +69,7 @@ func parseRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
 		return []Route{}, nil
 	}
 
-	rawRoutes, ok := r.([]any)
+	rawRoutes, ok := r.([]interface{})
 	if !ok {
 		return nil, fmt.Errorf("tun.routes is not an array")
 	}
@@ -83,7 +80,7 @@ func parseRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
 
 	routes := make([]Route, len(rawRoutes))
 	for i, r := range rawRoutes {
-		m, ok := r.(map[string]any)
+		m, ok := r.(map[interface{}]interface{})
 		if !ok {
 			return nil, fmt.Errorf("entry %v in tun.routes is invalid", i+1)
 		}
@@ -151,7 +148,7 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
 		return []Route{}, nil
 	}
 
-	rawRoutes, ok := r.([]any)
+	rawRoutes, ok := r.([]interface{})
 	if !ok {
 		return nil, fmt.Errorf("tun.unsafe_routes is not an array")
 	}
@@ -162,7 +159,7 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
 
 	routes := make([]Route, len(rawRoutes))
 	for i, r := range rawRoutes {
-		m, ok := r.(map[string]any)
+		m, ok := r.(map[interface{}]interface{})
 		if !ok {
 			return nil, fmt.Errorf("entry %v in tun.unsafe_routes is invalid", i+1)
 		}
@@ -204,63 +201,14 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
 			return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes is not present", i+1)
 		}
 
-		var gateways routing.Gateways
+		via, ok := rVia.(string)
+		if !ok {
+			return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes is not a string: found %T", i+1, rVia)
+		}
 
-		switch via := rVia.(type) {
+		viaVpnIp, err := netip.ParseAddr(via)
-		case string:
+		if err != nil {
-			viaIp, err := netip.ParseAddr(via)
+			return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes failed to parse address: %v", i+1, err)
-			if err != nil {
-				return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes failed to parse address: %v", i+1, err)
-			}
-
-			gateways = routing.Gateways{routing.NewGateway(viaIp, 1)}
-
-		case []any:
-			gateways = make(routing.Gateways, len(via))
-			for ig, v := range via {
-				gatewayMap, ok := v.(map[string]any)
-				if !ok {
-					return nil, fmt.Errorf("entry %v in tun.unsafe_routes[%v].via is invalid", i+1, ig+1)
-				}
-
-				rGateway, ok := gatewayMap["gateway"]
-				if !ok {
-					return nil, fmt.Errorf("entry .gateway in tun.unsafe_routes[%v].via[%v] is not present", i+1, ig+1)
-				}
-
-				parsedGateway, ok := rGateway.(string)
-				if !ok {
-					return nil, fmt.Errorf("entry .gateway in tun.unsafe_routes[%v].via[%v] is not a string", i+1, ig+1)
-				}
-
-				gatewayIp, err := netip.ParseAddr(parsedGateway)
-				if err != nil {
-					return nil, fmt.Errorf("entry .gateway in tun.unsafe_routes[%v].via[%v] failed to parse address: %v", i+1, ig+1, err)
-				}
-
-				rGatewayWeight, ok := gatewayMap["weight"]
-				if !ok {
-					rGatewayWeight = 1
-				}
-
-				gatewayWeight, ok := rGatewayWeight.(int)
-				if !ok {
-					_, err = strconv.ParseInt(rGatewayWeight.(string), 10, 32)
-					if err != nil {
-						return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1)
-					}
-				}
-
-				if gatewayWeight < 1 || gatewayWeight > math.MaxInt32 {
-					return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not in range (1-%d) : %v", i+1, ig+1, math.MaxInt32, gatewayWeight)
-				}
-
-				gateways[ig] = routing.NewGateway(gatewayIp, gatewayWeight)
-
-			}
-
-		default:
-			return nil, fmt.Errorf("entry %v.via in tun.unsafe_routes is not a string or list of gateways: found %T", i+1, rVia)
 		}
 
 		rRoute, ok := m["route"]
@@ -278,7 +226,7 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
 		}
 
 		r := Route{
-			Via:     gateways,
+			Via:     viaVpnIp,
 			MTU:     mtu,
 			Metric:  metric,
 			Install: install,