Experimenting

.github/actions/code-sign/action.yml

@@ -1,113 +0,0 @@
-name: Code-sign Windows binaries
-description: >
-  Sign every .exe under a given path in place via the DefinedNet code-signer
-  Lambda. If `role` or `bucket` is empty, logs a notice and skips signing so
-  forks and dev branches without AWS access still produce usable builds.
-
-inputs:
-  path:
-    description: "Directory whose .exe files should be signed in place"
-    required: true
-  role:
-    description: "IAM role ARN to assume via OIDC; empty disables signing"
-    required: false
-    default: ""
-  bucket:
-    description: "S3 staging bucket the code-signer Lambda reads from; empty disables signing"
-    required: false
-    default: ""
-  region:
-    description: "AWS region for the role and Lambda"
-    required: false
-    default: "us-east-2"
-  function-name:
-    description: "Code-signer Lambda function name"
-    required: false
-    default: "code-signer"
-  key-prefix:
-    description: "S3 key prefix the caller is authorized to write under"
-    required: false
-    default: "code-signing/slackhq/nebula"
-
-runs:
-  using: composite
-  steps:
-    - name: Skip notice
-      if: inputs.role == '' || inputs.bucket == ''
-      shell: sh
-      run: echo "::notice::code-signer role or bucket not set; skipping code signing."
-
-    - name: Configure AWS credentials
-      if: inputs.role != '' && inputs.bucket != ''
-      uses: aws-actions/configure-aws-credentials@v6
-      with:
-        role-to-assume: ${{ inputs.role }}
-        aws-region: ${{ inputs.region }}
-        # Default is 12 retries to ride out IAM trust-policy propagation; once
-        # the role is stable we want a real misconfiguration to fail fast.
-        retry-max-attempts: 5
-
-    - name: Sign .exe files
-      if: inputs.role != '' && inputs.bucket != ''
-      shell: sh
-      env:
-        SIGN_PATH: ${{ inputs.path }}
-        BUCKET: ${{ inputs.bucket }}
-        FUNCTION_NAME: ${{ inputs.function-name }}
-        KEY_PREFIX: ${{ inputs.key-prefix }}
-      run: |
-        set -eu
-        RUN="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
-
-        find "$SIGN_PATH" -name '*.exe' -print | while read -r path
-        do
-            rel=${path#"$SIGN_PATH"/}
-            file=$(basename "$path")
-            name=${file%.exe}
-            prefix="${KEY_PREFIX}/${RUN}"
-            src="${prefix}/unsigned/${rel}"
-            dst="${prefix}/signed/${rel}"
-
-            echo "::group::Sign ${rel}"
-            echo "Uploading unsigned to s3://${BUCKET}/${src}"
-            aws s3 cp --no-progress "$path" "s3://${BUCKET}/${src}" >/dev/null
-
-            echo "Invoking ${FUNCTION_NAME} Lambda"
-            payload=$(jq -nc \
-                --arg s "$src" \
-                --arg d "$dst" \
-                --arg p "$name" \
-                '{source_key: $s, dest_key: $d, program_name: $p}')
-            meta=$(aws lambda invoke \
-                --function-name "$FUNCTION_NAME" \
-                --cli-binary-format raw-in-base64-out \
-                --payload "$payload" \
-                --output json \
-                /tmp/sign-resp.json)
-            if echo "$meta" | jq -e '.FunctionError != null' >/dev/null
-            then
-                echo "::endgroup::"
-                echo "::error::code-signer Lambda failed for ${rel}"
-                cat /tmp/sign-resp.json >&2
-                exit 1
-            fi
-
-            echo "Downloading signed back to ${path}"
-            aws s3 cp --no-progress "s3://${BUCKET}/${dst}" "$path" >/dev/null
-
-            aws s3 rm "s3://${BUCKET}/${src}" >/dev/null 2>&1 || true
-            aws s3 rm "s3://${BUCKET}/${dst}" >/dev/null 2>&1 || true
-
-            # Sanity-check the bytes we got back actually carry an Authenticode
-            # signature that this machine can validate end to end.
-            status=$(powershell -NoProfile -Command "(Get-AuthenticodeSignature -FilePath '$path').Status" | tr -d '\r')
-            if [ "$status" != "Valid" ]
-            then
-                echo "::endgroup::"
-                echo "::error::${rel} signature status: ${status} (expected Valid)"
-                exit 1
-            fi
-
-            echo "Signed ${rel} (sha256=$(jq -r '.sha256' /tmp/sign-resp.json), status=${status})"
-            echo "::endgroup::"
-        done

.github/workflows/gofmt.yml

@@ -0,0 +1,34 @@
+name: gofmt
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    paths:
+      - '.github/workflows/gofmt.yml'
+      - '**.go'
+jobs:
+
+  gofmt:
+    name: Run gofmt
+    runs-on: ubuntu-latest
+    steps:
+
+    - uses: actions/checkout@v6
+
+    - uses: actions/setup-go@v6
+      with:
+        go-version: '1.25'
+        check-latest: true
+
+    - name: Install goimports
+      run: |
+        go install golang.org/x/tools/cmd/goimports@latest
+
+    - name: gofmt
+      run: |
+        if [ "$(find . -iname '*.go' | grep -v '\.pb\.go$' | xargs goimports -l)" ]
+        then
+          find . -iname '*.go' | grep -v '\.pb\.go$' | xargs goimports -d
+          exit 1
+        fi

.github/workflows/release.yml

@@ -10,7 +10,7 @@ jobs:
     name: Build Linux/BSD All
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v6
 
       - uses: actions/setup-go@v6
         with:
@@ -24,7 +24,7 @@ jobs:
           mv build/*.tar.gz release
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v6
         with:
           name: linux-latest
           path: release
@@ -32,11 +32,8 @@ jobs:
   build-windows:
     name: Build Windows
     runs-on: windows-latest
-    permissions:
-      id-token: write
-      contents: read
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v6
 
       - uses: actions/setup-go@v6
         with:
@@ -57,15 +54,8 @@ jobs:
           mkdir build\dist\windows
           mv dist\windows\wintun build\dist\windows\
 
-      - name: Code-sign
-        uses: ./.github/actions/code-sign
-        with:
-          path: build
-          role: ${{ secrets.DEFINED_CODE_SIGNER_ROLE }}
-          bucket: ${{ secrets.DEFINED_CODE_SIGNER_BUCKET }}
-
       - name: Upload artifacts
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v6
         with:
           name: windows-latest
           path: build
@@ -76,7 +66,7 @@ jobs:
       HAS_SIGNING_CREDS: ${{ secrets.AC_USERNAME != '' }}
     runs-on: macos-latest
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v6
 
       - uses: actions/setup-go@v6
         with:
@@ -85,7 +75,7 @@ jobs:
 
       - name: Import certificates
         if: env.HAS_SIGNING_CREDS == 'true'
-        uses: Apple-Actions/import-codesign-certs@v7
+        uses: Apple-Actions/import-codesign-certs@v6
         with:
           p12-file-base64: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_P12_BASE64 }}
           p12-password: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_PASSWORD }}
@@ -114,7 +104,7 @@ jobs:
           fi
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v7
+        uses: actions/upload-artifact@v6
         with:
           name: darwin-latest
           path: ./release/*
@@ -134,25 +124,25 @@ jobs:
       # be overwritten
       - name: Checkout code
         if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
-        uses: actions/checkout@v7
+        uses: actions/checkout@v6
 
       - name: Download artifacts
         if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
-        uses: actions/download-artifact@v8
+        uses: actions/download-artifact@v7
         with:
           name: linux-latest
           path: artifacts
 
       - name: Login to Docker Hub
         if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
-        uses: docker/login-action@v4
+        uses: docker/login-action@v3
         with:
           username: ${{ vars.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Set up Docker Buildx
         if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
-        uses: docker/setup-buildx-action@v4
+        uses: docker/setup-buildx-action@v3
 
       - name: Build and push images
         if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
@@ -163,20 +153,17 @@ jobs:
           mkdir -p build/linux-{amd64,arm64}
           tar -zxvf artifacts/nebula-linux-amd64.tar.gz -C build/linux-amd64/
           tar -zxvf artifacts/nebula-linux-arm64.tar.gz -C build/linux-arm64/
-          docker buildx build . --push -f docker/Dockerfile --platform linux/amd64,linux/arm64 \
+          docker buildx build . --push -f docker/Dockerfile --platform linux/amd64,linux/arm64 --tag "${DOCKER_IMAGE_REPO}:${DOCKER_IMAGE_TAG}" --tag "${DOCKER_IMAGE_REPO}:${GITHUB_REF#refs/tags/v}"
-            --build-arg VERSION="${GITHUB_REF#refs/tags/v}" \
-            --build-arg REVISION="${GITHUB_SHA}" \
-            --tag "${DOCKER_IMAGE_REPO}:${DOCKER_IMAGE_TAG}" --tag "${DOCKER_IMAGE_REPO}:${GITHUB_REF#refs/tags/v}"
 
   release:
     name: Create and Upload Release
     needs: [build-linux, build-darwin, build-windows]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v7
+      - uses: actions/checkout@v6
 
       - name: Download artifacts
-        uses: actions/download-artifact@v8
+        uses: actions/download-artifact@v7
         with:
           path: artifacts
 

.github/workflows/smoke-extra.yml

@@ -14,23 +14,15 @@ on:
       - 'go.sum'
 jobs:
 
-  smoke-extra-libvirt:
+  smoke-extra:
     if: github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'smoke-test-extra')
-    name: ${{ matrix.target }}
+    name: Run extra smoke tests
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        target:
-          - freebsd-amd64
-          - openbsd-amd64
-          - netbsd-amd64
-          - linux-amd64-ipv6disable
     env:
       VAGRANT_DEFAULT_PROVIDER: libvirt
     steps:
 
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v6
 
     - uses: actions/setup-go@v6
       with:
@@ -48,85 +40,28 @@ jobs:
         sudo chmod 666 /var/run/libvirt/libvirt-sock
         vagrant plugin install vagrant-libvirt
 
-    - name: ${{ matrix.target }}
+    - name: freebsd-amd64
-      run: make smoke-vagrant/${{ matrix.target }}
+      run: make smoke-vagrant/freebsd-amd64
 
-    timeout-minutes: 30
+    - name: openbsd-amd64
+      run: make smoke-vagrant/openbsd-amd64
 
-  # linux-386 needs VirtualBox, which conflicts with KVM/libvirt -- isolated job.
+    - name: netbsd-amd64
-  smoke-extra-virtualbox:
+      run: make smoke-vagrant/netbsd-amd64
-    if: github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'smoke-test-extra')
-    name: linux-386
-    runs-on: ubuntu-latest
-    env:
-      VAGRANT_DEFAULT_PROVIDER: virtualbox
-    steps:
 
-    - uses: actions/checkout@v7
+    - name: linux-amd64-ipv6disable
+      run: make smoke-vagrant/linux-amd64-ipv6disable
 
-    - uses: actions/setup-go@v6
+    # linux-386 runs last because it requires disabling KVM to use VirtualBox,
-      with:
+    # which prevents libvirt (used by the other tests) from working after this point.
-        go-version: '1.25'
+    - name: install virtualbox for i386 test
-        check-latest: true
-
-    - name: add hashicorp source
-      run: wget -O- https://apt.releases.hashicorp.com/gpg | gpg --dearmor | sudo tee /usr/share/keyrings/hashicorp-archive-keyring.gpg && echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
-
-    - name: install vagrant and virtualbox
       run: |
-        sudo apt-get update && sudo apt-get install -y vagrant virtualbox
+        sudo apt-get install -y virtualbox
         sudo rmmod kvm_amd kvm_intel kvm 2>/dev/null || true
 
     - name: linux-386
+      env:
+        VAGRANT_DEFAULT_PROVIDER: virtualbox
       run: make smoke-vagrant/linux-386
 
     timeout-minutes: 30
-
-  smoke-windows:
-    if: github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'smoke-test-extra')
-    name: Run windows smoke test
-    runs-on: windows-latest
-    steps:
-
-    - uses: actions/checkout@v7
-
-    - uses: actions/setup-go@v6
-      with:
-        go-version: '1.25'
-        check-latest: true
-
-    # WSL2 + Ubuntu so the smoke can run a real linux peer with its own
-    # netns. iputils-ping is needed for the in-WSL ping check. WSL1 has no
-    # real kernel and would lack /dev/net/tun, so we have to force WSL2.
-    - uses: Vampire/setup-wsl@v3
-      with:
-        distribution: Ubuntu-24.04
-        additional-packages: iputils-ping iproute2
-
-    # Vampire/setup-wsl provisions WSL1 even when the WSL2 platform is present.
-    # Convert the distro to WSL2 explicitly before we try to use /dev/net/tun.
-    - name: convert distro to WSL2
-      shell: pwsh
-      run: |
-        wsl --set-version Ubuntu-24.04 2
-        wsl --shutdown
-        wsl --list --verbose
-
-    - name: build windows nebula
-      run: make bin-windows
-
-    - name: build linux nebula for WSL
-      shell: bash
-      env:
-        GOOS: linux
-        GOARCH: amd64
-      run: |
-        mkdir -p build/linux-amd64
-        go build -o build/linux-amd64/nebula ./cmd/nebula
-
-    - name: run smoke-windows
-      shell: pwsh
-      working-directory: ./.github/workflows/smoke
-      run: ./smoke-windows.ps1
-
-    timeout-minutes: 15

.github/workflows/smoke.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v6
 
     - uses: actions/setup-go@v6
       with:
@@ -36,14 +36,6 @@ jobs:
       working-directory: ./.github/workflows/smoke
       run: ./smoke.sh
 
-    - name: setup docker image ipv6
-      working-directory: ./.github/workflows/smoke
-      run: SMOKE_OVERLAY_IPV6=1 ./build.sh
-
-    - name: run smoke ipv6
-      working-directory: ./.github/workflows/smoke
-      run: SMOKE_OVERLAY_IPV6=1 ./smoke.sh
-
     - name: setup relay docker image
       working-directory: ./.github/workflows/smoke
       run: ./build-relay.sh

.github/workflows/smoke/build.sh

@@ -5,19 +5,6 @@ set -e -x
 rm -rf ./build
 mkdir ./build
 
-if [ "$SMOKE_OVERLAY_IPV6" ]
-then
-    LIGHTHOUSE_NIP="fd00:4242:0:0:0:ffff:c0a8:6401"
-    HOST2_NIP="fd00:4242:0:0:0:ffff:c0a8:6402"
-    HOST3_NIP="fd00:4242:0:0:0:ffff:c0a8:6403"
-    HOST4_NIP="fd00:4242:0:0:0:ffff:c0a8:6404"
-else
-    LIGHTHOUSE_NIP="192.168.100.1"
-    HOST2_NIP="192.168.100.2"
-    HOST3_NIP="192.168.100.3"
-    HOST4_NIP="192.168.100.4"
-fi
-
 # Smoke containers run on a dedicated docker network whose subnet is allocated
 # at smoke time, not known at build time. Configs are written with TEST-NET-3
 # placeholder IPs (RFC 5737) and smoke.sh / smoke-vagrant.sh / smoke-relay.sh
@@ -44,24 +31,24 @@ LIGHTHOUSE_IP="203.0.113.2"
         ../genconfig.sh >lighthouse1.yml
 
     HOST="host2" \
-        LIGHTHOUSES="$LIGHTHOUSE_NIP $LIGHTHOUSE_IP:4242" \
+        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
         ../genconfig.sh >host2.yml
 
     HOST="host3" \
-        LIGHTHOUSES="$LIGHTHOUSE_NIP $LIGHTHOUSE_IP:4242" \
+        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
         INBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
         ../genconfig.sh >host3.yml
 
     HOST="host4" \
-        LIGHTHOUSES="$LIGHTHOUSE_NIP $LIGHTHOUSE_IP:4242" \
+        LIGHTHOUSES="192.168.100.1 $LIGHTHOUSE_IP:4242" \
         OUTBOUND='[{"port": "any", "proto": "icmp", "group": "lighthouse"}]' \
         ../genconfig.sh >host4.yml
 
     ../../../../nebula-cert ca -curve "${CURVE:-25519}" -name "Smoke Test"
-    ../../../../nebula-cert sign -name "lighthouse1" -groups "lighthouse,lighthouse1" -ip "$LIGHTHOUSE_NIP/24"
+    ../../../../nebula-cert sign -name "lighthouse1" -groups "lighthouse,lighthouse1" -ip "192.168.100.1/24"
-    ../../../../nebula-cert sign -name "host2" -groups "host,host2" -ip "$HOST2_NIP/24"
+    ../../../../nebula-cert sign -name "host2" -groups "host,host2" -ip "192.168.100.2/24"
-    ../../../../nebula-cert sign -name "host3" -groups "host,host3" -ip "$HOST3_NIP/24"
+    ../../../../nebula-cert sign -name "host3" -groups "host,host3" -ip "192.168.100.3/24"
-    ../../../../nebula-cert sign -name "host4" -groups "host,host4" -ip "$HOST4_NIP/24"
+    ../../../../nebula-cert sign -name "host4" -groups "host,host4" -ip "192.168.100.4/24"
 )
 
 docker build -t "nebula:${NAME:-smoke}" .

.github/workflows/smoke/smoke-windows.ps1

@@ -1,272 +0,0 @@
-#!/usr/bin/env pwsh
-# Windows smoke test for the nebula tun + UDP + NLM code paths.
-#
-# Topology:
-#   - lighthouse runs natively on the Windows host (wintun + windows UDP)
-#   - peer runs inside WSL2 (Linux build of nebula, /dev/net/tun)
-#
-# WSL2 gives us a real netns boundary so the loopback fast-path on Windows
-# does not short-circuit the overlay -- when WSL pings the lighthouse VPN IP,
-# Linux has no idea that IP is local to the Windows host, so the packet is
-# forced through nebula. Same in reverse.
-
-$ErrorActionPreference = 'Stop'
-
-# wsl.exe emits UTF-16 LE by default which PowerShell reads as bytes, mangling
-# every captured string. WSL_UTF8 makes wsl.exe emit UTF-8 instead.
-$env:WSL_UTF8 = '1'
-
-$RepoRoot = Resolve-Path "$PSScriptRoot\..\..\.."
-$Nebula = Join-Path $RepoRoot 'nebula.exe'
-$NebulaCert = Join-Path $RepoRoot 'nebula-cert.exe'
-$NebulaLinux = Join-Path $RepoRoot 'build\linux-amd64\nebula'
-
-if (-not (Test-Path $Nebula)) { throw "missing $Nebula; run 'make bin-windows' first" }
-if (-not (Test-Path $NebulaCert)) { throw "missing $NebulaCert; run 'make bin-windows' first" }
-if (-not (Test-Path $NebulaLinux)) { throw "missing $NebulaLinux; build the linux nebula first" }
-
-# Matches the distro installed by Vampire/setup-wsl in smoke-extra.yml.
-$Distro = 'Ubuntu-24.04'
-$listed = (wsl --list --quiet 2>$null) -join "`n"
-if ($listed -notmatch [regex]::Escape($Distro)) {
-    throw "WSL distro $Distro not registered. Got: $listed"
-}
-Write-Host "Using WSL distro: $Distro"
-
-# Windows host as seen from inside WSL: WSL's default-route gateway. We extract
-# it with a regex rather than awk fields so PowerShell does not eat any '$N'
-# tokens, and tabs/double-spaces in `ip route` output do not confuse a cut.
-$ipCmd = 'ip route show default | grep -oE "([0-9]+\.){3}[0-9]+" | head -1'
-$WindowsIp = (wsl -d $Distro -- bash -c $ipCmd).Trim()
-if (-not $WindowsIp) { throw "could not determine Windows host IP from WSL" }
-Write-Host "Windows host IP from WSL: $WindowsIp"
-
-$WorkDir = Join-Path $env:TEMP 'nebula-smoke-windows'
-if (Test-Path $WorkDir) { Remove-Item -Recurse -Force $WorkDir }
-New-Item -ItemType Directory -Path $WorkDir | Out-Null
-
-$WslDir = '/tmp/nebula-smoke'
-wsl -d $Distro -- bash -c "rm -rf $WslDir && mkdir -p $WslDir" | Out-Null
-
-$DevName = 'nebula-smoke'
-$Ip1 = '192.168.241.1'
-$Ip2 = '192.168.241.2'
-$Port = 4242
-
-& $NebulaCert ca -name 'smoke-ca' -out-crt "$WorkDir\ca.crt" -out-key "$WorkDir\ca.key"
-if ($LASTEXITCODE -ne 0) { throw "nebula-cert ca failed (exit $LASTEXITCODE)" }
-
-& $NebulaCert sign -name 'lighthouse' -networks "$Ip1/24" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\lighthouse.crt" -out-key "$WorkDir\lighthouse.key"
-if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign lighthouse failed (exit $LASTEXITCODE)" }
-
-& $NebulaCert sign -name 'peer' -networks "$Ip2/24" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\peer.crt" -out-key "$WorkDir\peer.key"
-if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign peer failed (exit $LASTEXITCODE)" }
-
-# Windows lighthouse config.
-@"
-pki:
-  ca: $WorkDir\ca.crt
-  cert: $WorkDir\lighthouse.crt
-  key: $WorkDir\lighthouse.key
-static_host_map: {}
-lighthouse:
-  am_lighthouse: true
-  interval: 60
-  hosts: []
-listen:
-  host: 0.0.0.0
-  port: $Port
-tun:
-  disabled: false
-  dev: $DevName
-  drop_local_broadcast: false
-  drop_multicast: false
-  tx_queue: 500
-  mtu: 1300
-  network_category: private
-logging:
-  level: info
-  format: text
-firewall:
-  outbound_action: drop
-  inbound_action: drop
-  conntrack:
-    tcp_timeout: 12m
-    udp_timeout: 3m
-    default_timeout: 10m
-  outbound:
-    - port: any
-      proto: any
-      host: any
-  inbound:
-    - port: any
-      proto: any
-      host: any
-"@ | Out-File -FilePath "$WorkDir\lighthouse.yml" -Encoding utf8
-
-# WSL peer config (paths are POSIX, deliberately).
-@"
-pki:
-  ca: $WslDir/ca.crt
-  cert: $WslDir/peer.crt
-  key: $WslDir/peer.key
-static_host_map:
-  "${Ip1}": ["${WindowsIp}:$Port"]
-lighthouse:
-  am_lighthouse: false
-  interval: 60
-  hosts:
-    - "${Ip1}"
-listen:
-  host: 0.0.0.0
-  port: 0
-tun:
-  disabled: false
-  dev: nebula1
-  drop_local_broadcast: false
-  drop_multicast: false
-  tx_queue: 500
-  mtu: 1300
-logging:
-  level: info
-  format: text
-firewall:
-  outbound_action: drop
-  inbound_action: drop
-  conntrack:
-    tcp_timeout: 12m
-    udp_timeout: 3m
-    default_timeout: 10m
-  outbound:
-    - port: any
-      proto: any
-      host: any
-  inbound:
-    - port: any
-      proto: any
-      host: any
-"@ | Out-File -FilePath "$WorkDir\peer.yml" -Encoding utf8
-
-# Stage WSL artifacts. Convert Windows paths to WSL paths ourselves rather than
-# calling `wslpath`, because PowerShell's argument-passing to external EXEs
-# strips backslashes from path arguments in ways that are hard to escape around.
-function ConvertTo-WslPath {
-    param([string]$WindowsPath)
-    if ($WindowsPath -notmatch '^([A-Za-z]):\\(.*)$') {
-        throw "cannot convert path to WSL: $WindowsPath"
-    }
-    return "/mnt/$($matches[1].ToLower())/$($matches[2].Replace('\','/'))"
-}
-
-$WslWorkDir = ConvertTo-WslPath $WorkDir
-$WslNebulaPath = ConvertTo-WslPath $NebulaLinux
-wsl -d $Distro -- bash -c "cp '$WslWorkDir/ca.crt' '$WslWorkDir/peer.crt' '$WslWorkDir/peer.key' '$WslWorkDir/peer.yml' $WslDir/ && cp '$WslNebulaPath' $WslDir/nebula && chmod +x $WslDir/nebula"
-
-# Make sure WSL has tun support and /dev/net/tun is usable before starting
-# nebula. Diagnostics first so a fail here points at the real problem (e.g.
-# WSL1 distros do not have a real kernel and will not have tun).
-Write-Host '=== WSL diagnostic ==='
-wsl --version 2>&1 | Out-Host
-wsl --list --verbose 2>&1 | Out-Host
-wsl -d $Distro -u root -- uname -a | Out-Host
-wsl -d $Distro -u root -- bash -c "modprobe tun 2>&1 || true; mkdir -p /dev/net; [ -c /dev/net/tun ] || mknod /dev/net/tun c 10 200; chmod 600 /dev/net/tun; ls -l /dev/net/tun"
-if ($LASTEXITCODE -ne 0) { throw "failed to prepare /dev/net/tun in WSL (TUN support missing?)" }
-
-# Deliberately no New-NetFirewallRule calls here -- nebula's windows_bypass_wdf
-# feature is supposed to install WFP permit filters that let inbound traffic
-# through Windows Defender Firewall on its own. If this smoke regresses, that
-# feature regressed.
-
-$lhOut = Join-Path $WorkDir 'lighthouse.out.log'
-$lhErr = Join-Path $WorkDir 'lighthouse.err.log'
-$lhProc = Start-Process -FilePath $Nebula -ArgumentList @('-config', "$WorkDir\lighthouse.yml") `
-    -PassThru -NoNewWindow `
-    -RedirectStandardOutput $lhOut `
-    -RedirectStandardError $lhErr
-
-# Run nebula in WSL as root with no sudo + no shell wrapper. PowerShell's
-# Start-Process arg quoting mangles `bash -c "..."` strings that contain
-# spaces/redirections, so we skip bash entirely and let Start-Process do the
-# stdout/stderr capture itself.
-$peerOut = Join-Path $WorkDir 'peer.out.log'
-$peerErr = Join-Path $WorkDir 'peer.err.log'
-$peerProc = Start-Process -FilePath 'wsl' `
-    -ArgumentList @('-d', $Distro, '-u', 'root', '--', "$WslDir/nebula", '-config', "$WslDir/peer.yml") `
-    -PassThru -NoNewWindow `
-    -RedirectStandardOutput $peerOut `
-    -RedirectStandardError $peerErr
-
-function Wait-Until {
-    param([scriptblock]$Predicate, [int]$TimeoutSec, [string]$What)
-    $deadline = (Get-Date).AddSeconds($TimeoutSec)
-    while ((Get-Date) -lt $deadline) {
-        if (& $Predicate) { return }
-        Start-Sleep -Milliseconds 500
-    }
-    throw "timed out waiting for: $What"
-}
-
-try {
-    Wait-Until -TimeoutSec 30 -What "windows wintun adapter $DevName with NetworkCategory=Private" -Predicate {
-        if ($lhProc.HasExited) { throw "lighthouse exited (code $($lhProc.ExitCode)) before tun was ready" }
-        $p = Get-NetConnectionProfile -InterfaceAlias $DevName -ErrorAction SilentlyContinue
-        $p -and ("$($p.NetworkCategory)" -ieq 'Private')
-    }
-    Write-Host "OK: $DevName NetworkCategory=Private"
-
-    Wait-Until -TimeoutSec 30 -What "WSL nebula1 with $Ip2" -Predicate {
-        if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before tun was ready" }
-        $r = wsl -d $Distro -u root -- bash -c "ip -o addr show nebula1 2>/dev/null | grep -q 'inet $Ip2' && echo yes"
-        ("$r").Trim() -eq 'yes'
-    }
-    Write-Host "OK: WSL nebula1 has $Ip2"
-
-    Wait-Until -TimeoutSec 30 -What "ping from WSL peer to windows lighthouse ($Ip1)" -Predicate {
-        if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before ping succeeded" }
-        $r = wsl -d $Distro -u root -- bash -c "ping -c1 -W1 $Ip1 >/dev/null 2>&1 && echo OK"
-        ("$r").Trim() -eq 'OK'
-    }
-    Write-Host "OK: WSL peer -> windows lighthouse"
-
-    Wait-Until -TimeoutSec 30 -What "ping from windows lighthouse to WSL peer ($Ip2)" -Predicate {
-        $null = & ping.exe -n 1 -w 1000 $Ip2
-        $LASTEXITCODE -eq 0
-    }
-    Write-Host "OK: windows lighthouse -> WSL peer"
-
-    Write-Host ''
-    Write-Host 'All smoke checks passed.'
-}
-catch {
-    Write-Host ''
-    Write-Host '=== lighthouse stdout ==='
-    Get-Content $lhOut -ErrorAction SilentlyContinue | Out-Host
-    Write-Host '=== lighthouse stderr ==='
-    Get-Content $lhErr -ErrorAction SilentlyContinue | Out-Host
-    Write-Host '=== peer stdout ==='
-    Get-Content $peerOut -ErrorAction SilentlyContinue | Out-Host
-    Write-Host '=== peer stderr ==='
-    Get-Content $peerErr -ErrorAction SilentlyContinue | Out-Host
-    Write-Host '=== nebula WFP filters ==='
-    # Dump nebula-installed filters so we can verify they got registered with
-    # the conditions we expect.
-    $wfpDump = Join-Path $WorkDir 'wfp.xml'
-    netsh wfp show filters file=$wfpDump 2>&1 | Out-Null
-    if (Test-Path $wfpDump) {
-        Select-String -Path $wfpDump -Pattern 'Nebula' -Context 0,80 -ErrorAction SilentlyContinue | Out-Host
-    }
-    throw
-}
-finally {
-    if (-not $lhProc.HasExited) {
-        Stop-Process -Id $lhProc.Id -Force -ErrorAction SilentlyContinue
-        $lhProc.WaitForExit(5000) | Out-Null
-    }
-    wsl -d $Distro -u root -- bash -c "pkill -f $WslDir/nebula 2>/dev/null; true" | Out-Null
-    # pkill returns 1 when no match and wsl propagates that; the smoke is done
-    # so we don't want it to leak into the script's exit code.
-    $global:LASTEXITCODE = 0
-    if ($peerProc -and -not $peerProc.HasExited) {
-        Stop-Process -Id $peerProc.Id -Force -ErrorAction SilentlyContinue
-    }
-}

.github/workflows/smoke/smoke.sh

@@ -47,19 +47,6 @@ HOST2_IP="$PREFIX.3"
 HOST3_IP="$PREFIX.4"
 HOST4_IP="$PREFIX.5"
 
-if [ "$SMOKE_OVERLAY_IPV6" ]
-then
-    LIGHTHOUSE_NIP="fd00:4242:0:0:0:ffff:c0a8:6401"
-    HOST2_NIP="fd00:4242:0:0:0:ffff:c0a8:6402"
-    HOST3_NIP="fd00:4242:0:0:0:ffff:c0a8:6403"
-    HOST4_NIP="fd00:4242:0:0:0:ffff:c0a8:6404"
-else
-    LIGHTHOUSE_NIP="192.168.100.1"
-    HOST2_NIP="192.168.100.2"
-    HOST3_NIP="192.168.100.3"
-    HOST4_NIP="192.168.100.4"
-fi
-
 # Sed the placeholder TEST-NET-3 IPs in the host configs to the real ones.
 # build/lighthouse1.yml has no IPs to rewrite so it's skipped.
 for f in build/host2.yml build/host3.yml build/host4.yml; do
@@ -93,28 +80,28 @@ docker exec host3 tcpdump -i eth0 -q -w - -U 2>logs/host3.outside.log >logs/host
 docker exec host4 tcpdump -i tun0 -q -w - -U 2>logs/host4.inside.log >logs/host4.inside.pcap &
 docker exec host4 tcpdump -i eth0 -q -w - -U 2>logs/host4.outside.log >logs/host4.outside.pcap &
 
-docker exec host2 ncat -nklv 2000 &
+docker exec host2 ncat -nklv 0.0.0.0 2000 &
-docker exec host3 ncat -nklv 2000 &
+docker exec host3 ncat -nklv 0.0.0.0 2000 &
-docker exec host4 ncat -e '/usr/bin/echo helloagainfromhost4' -nkluv 4000 &
+docker exec host4 ncat -e '/usr/bin/echo helloagainfromhost4' -nkluv 0.0.0.0 4000 &
-docker exec host2 ncat -e '/usr/bin/echo host2' -nkluv 3000 &
+docker exec host2 ncat -e '/usr/bin/echo host2' -nkluv 0.0.0.0 3000 &
-docker exec host3 ncat -e '/usr/bin/echo host3' -nkluv 3000 &
+docker exec host3 ncat -e '/usr/bin/echo host3' -nkluv 0.0.0.0 3000 &
 
 set +x
 echo
 echo " *** Testing ping from lighthouse1"
 echo
 set -x
-docker exec lighthouse1 ping -c1 $HOST2_NIP
+docker exec lighthouse1 ping -c1 192.168.100.2
-docker exec lighthouse1 ping -c1 $HOST3_NIP
+docker exec lighthouse1 ping -c1 192.168.100.3
 
 set +x
 echo
 echo " *** Testing ping from host2"
 echo
 set -x
-docker exec host2 ping -c1 $LIGHTHOUSE_NIP
+docker exec host2 ping -c1 192.168.100.1
 # Should fail because not allowed by host3 inbound firewall
-! docker exec host2 ping -c1 $HOST3_NIP -w5 || exit 1
+! docker exec host2 ping -c1 192.168.100.3 -w5 || exit 1
 
 set +x
 echo
@@ -122,34 +109,34 @@ echo " *** Testing ncat from host2"
 echo
 set -x
 # Should fail because not allowed by host3 inbound firewall
-! docker exec host2 ncat -nzv -w5 $HOST3_NIP 2000 || exit 1
+! docker exec host2 ncat -nzv -w5 192.168.100.3 2000 || exit 1
-! docker exec host2 ncat -nzuv -w5 $HOST3_NIP 3000 | grep -q host3 || exit 1
+! docker exec host2 ncat -nzuv -w5 192.168.100.3 3000 | grep -q host3 || exit 1
 
 set +x
 echo
 echo " *** Testing ping from host3"
 echo
 set -x
-docker exec host3 ping -c1 $LIGHTHOUSE_NIP
+docker exec host3 ping -c1 192.168.100.1
-docker exec host3 ping -c1 $HOST2_NIP
+docker exec host3 ping -c1 192.168.100.2
 
 set +x
 echo
 echo " *** Testing ncat from host3"
 echo
 set -x
-docker exec host3 ncat -nzv -w5 $HOST2_NIP 2000
+docker exec host3 ncat -nzv -w5 192.168.100.2 2000
-docker exec host3 ncat -nzuv -w5 $HOST2_NIP 3000 | grep -q host2
+docker exec host3 ncat -nzuv -w5 192.168.100.2 3000 | grep -q host2
 
 set +x
 echo
 echo " *** Testing ping from host4"
 echo
 set -x
-docker exec host4 ping -c1 $LIGHTHOUSE_NIP
+docker exec host4 ping -c1 192.168.100.1
 # Should fail because not allowed by host4 outbound firewall
-! docker exec host4 ping -c1 $HOST2_NIP -w5 || exit 1
+! docker exec host4 ping -c1 192.168.100.2 -w5 || exit 1
-! docker exec host4 ping -c1 $HOST3_NIP -w5 || exit 1
+! docker exec host4 ping -c1 192.168.100.3 -w5 || exit 1
 
 set +x
 echo
@@ -157,10 +144,10 @@ echo " *** Testing ncat from host4"
 echo
 set -x
 # Should fail because not allowed by host4 outbound firewall
-! docker exec host4 ncat -nzv -w5 $HOST2_NIP 2000 || exit 1
+! docker exec host4 ncat -nzv -w5 192.168.100.2 2000 || exit 1
-! docker exec host4 ncat -nzv -w5 $HOST3_NIP 2000 || exit 1
+! docker exec host4 ncat -nzv -w5 192.168.100.3 2000 || exit 1
-! docker exec host4 ncat -nzuv -w5 $HOST2_NIP 3000 | grep -q host2 || exit 1
+! docker exec host4 ncat -nzuv -w5 192.168.100.2 3000 | grep -q host2 || exit 1
-! docker exec host4 ncat -nzuv -w5 $HOST3_NIP 3000 | grep -q host3 || exit 1
+! docker exec host4 ncat -nzuv -w5 192.168.100.3 3000 | grep -q host3 || exit 1
 
 set +x
 echo
@@ -172,7 +159,7 @@ set -x
 # cannot initiate UDP to host2. Once host2 initiates a flow to host4:4000,
 # conntrack must let host4's listener reply on that flow. If it doesn't,
 # the echo back from host4 never reaches host2.
-docker exec host2 sh -c "(/usr/bin/echo host2; sleep 2) | ncat -nuv $HOST4_NIP 4000" | grep -q helloagainfromhost4
+docker exec host2 sh -c "(/usr/bin/echo host2; sleep 2) | ncat -nuv 192.168.100.4 4000" | grep -q helloagainfromhost4
 
 docker exec host4 sh -c 'kill 1'
 docker exec host3 sh -c 'kill 1'

.github/workflows/smoke/vagrant-netbsd-amd64/Vagrantfile

@@ -1,7 +1,7 @@
 # -*- mode: ruby -*-
 # vi: set ft=ruby :
 Vagrant.configure("2") do |config|
-  config.vm.box = "DefinedNet/netbsd10"
+  config.vm.box = "generic/netbsd9"
 
   config.vm.synced_folder "../build", "/nebula", type: "rsync"
 end

.github/workflows/test.yml

@@ -13,28 +13,20 @@ on:
       - 'go.sum'
 jobs:
 
-  static:
+  test-linux:
-    name: Static checks
+    name: Build all and test on ubuntu-linux
     runs-on: ubuntu-latest
     steps:
 
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v6
 
     - uses: actions/setup-go@v6
       with:
         go-version: '1.25'
         check-latest: true
 
-    - name: Install goimports
+    - name: Build
-      run: go install golang.org/x/tools/cmd/goimports@latest
+      run: make all
-
-    - name: gofmt
-      run: |
-        if [ "$(find . -iname '*.go' | grep -v '\.pb\.go$' | xargs goimports -l)" ]
-        then
-          find . -iname '*.go' | grep -v '\.pb\.go$' | xargs goimports -d
-          exit 1
-        fi
 
     - name: Vet
       run: make vet
@@ -44,41 +36,27 @@ jobs:
       with:
         version: v2.5
 
-  test:
+    - name: Test
-    name: Test ${{ matrix.name }}
+      run: make test
-    runs-on: ${{ matrix.os }}
+
-    strategy:
+    - name: End 2 end
-      fail-fast: false
+      run: make e2evv
-      matrix:
+
-        include:
+    - name: Build test mobile
-        - name: linux
+      run: make build-test-mobile
-          os: ubuntu-latest
+
-          build-cmd: go build ./cmd/nebula ./cmd/nebula-cert
+    - uses: actions/upload-artifact@v6
-          test-cmd: make test
+      with:
-          e2e-cmd: make e2evv
+        name: e2e packet flow linux-latest
-        - name: linux-boringcrypto
+        path: e2e/mermaid/linux-latest
-          os: ubuntu-latest
+        if-no-files-found: warn
-          build-cmd: make bin-boringcrypto
+
-          test-cmd: make test-boringcrypto
+  test-linux-boringcrypto:
-          e2e-cmd: make e2e GOEXPERIMENT=boringcrypto CGO_ENABLED=1 TEST_ENV="TEST_LOGS=1" TEST_FLAGS="-v -ldflags -checklinkname=0"
+    name: Build and test on linux with boringcrypto
-        - name: linux-pkcs11
+    runs-on: ubuntu-latest
-          os: ubuntu-latest
-          build-cmd: make bin-pkcs11
-          test-cmd: make test-pkcs11
-          e2e-cmd: ''
-        - name: macos
-          os: macos-latest
-          build-cmd: go build ./cmd/nebula ./cmd/nebula-cert
-          test-cmd: make test
-          e2e-cmd: make e2evv
-        - name: windows
-          os: windows-latest
-          build-cmd: go build ./cmd/nebula ./cmd/nebula-cert
-          test-cmd: make test
-          e2e-cmd: make e2evv
     steps:
 
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v6
 
     - uses: actions/setup-go@v6
       with:
@@ -86,65 +64,69 @@ jobs:
         check-latest: true
 
     - name: Build
-      run: ${{ matrix.build-cmd }}
+      run: make bin-boringcrypto
-
-    - name: Cross-build darwin-amd64
-      if: matrix.name == 'macos'
-      run: GOARCH=amd64 go build -o /tmp/nebula-amd64 ./cmd/nebula && GOARCH=amd64 go build -o /tmp/nebula-cert-amd64 ./cmd/nebula-cert
 
     - name: Test
-      run: ${{ matrix.test-cmd }}
+      run: make test-boringcrypto
 
     - name: End 2 end
-      if: matrix.e2e-cmd != ''
+      run: make e2e GOEXPERIMENT=boringcrypto CGO_ENABLED=1 TEST_ENV="TEST_LOGS=1" TEST_FLAGS="-v -ldflags -checklinkname=0"
-      run: ${{ matrix.e2e-cmd }}
 
-    - uses: actions/upload-artifact@v7
+  test-linux-pkcs11:
-      if: matrix.e2e-cmd != '' && always()
+    name: Build and test on linux with pkcs11
-      with:
-        name: e2e packet flow ${{ matrix.name }}
-        path: e2e/mermaid/
-        if-no-files-found: warn
-
-  cross-build:
-    name: Cross-build ${{ matrix.name }}
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-        - {name: linux-arm,   make-target: all-cross-linux-arm}
-        - {name: linux-mips,  make-target: all-cross-linux-mips}
-        - {name: linux-other, make-target: all-cross-linux-other}
-        - {name: freebsd,     make-target: all-freebsd}
-        - {name: openbsd,     make-target: all-openbsd}
-        - {name: netbsd,      make-target: all-netbsd}
-        - {name: windows,     make-target: all-cross-windows}
-        - {name: mobile,      make-target: build-test-mobile}
     steps:
 
-    - uses: actions/checkout@v7
+    - uses: actions/checkout@v6
 
     - uses: actions/setup-go@v6
       with:
         go-version: '1.25'
         check-latest: true
 
-    - name: Build ${{ matrix.name }}
+    - name: Build
-      run: make -j"$(nproc)" ${{ matrix.make-target }}
+      run: make bin-pkcs11
 
-  finish:
+    - name: Test
-    name: CI status
+      run: make test-pkcs11
-    if: always()
+
-    needs: [static, test, cross-build]
+  test:
-    runs-on: ubuntu-latest
+    name: Build and test on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [windows-latest, macos-latest]
     steps:
 
-    - name: Fail if any upstream job failed
+    - uses: actions/checkout@v6
-      if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-      run: |
-        echo "upstream results: ${{ toJSON(needs) }}"
-        exit 1
 
-    - name: All upstream jobs passed
+    - uses: actions/setup-go@v6
-      run: echo "ok"
+      with:
+        go-version: '1.25'
+        check-latest: true
+
+    - name: Build nebula
+      run: go build ./cmd/nebula
+
+    - name: Build nebula-cert
+      run: go build ./cmd/nebula-cert
+
+    - name: Vet
+      run: make vet
+
+    - name: golangci-lint
+      uses: golangci/golangci-lint-action@v9
+      with:
+        version: v2.5
+
+    - name: Test
+      run: make test
+
+    - name: End 2 end
+      run: make e2evv
+
+    - uses: actions/upload-artifact@v6
+      with:
+        name: e2e packet flow ${{ matrix.os }}
+        path: e2e/mermaid/${{ matrix.os }}
+        if-no-files-found: warn

Makefile

@@ -60,18 +60,6 @@ ALL = $(ALL_LINUX) \
 	windows-amd64 \
 	windows-arm64
 
-# Cross-build shards used by .github/workflows/test.yml — same as ALL_*
-# but with the arch that has a native CI runner removed, so the cross-build
-# job is not duplicating coverage the native test jobs already give.
-ALL_CROSS_LINUX = $(filter-out linux-amd64,$(ALL_LINUX))
-
-# ALL_CROSS_LINUX further split into family sub-shards so each can run on
-# its own CI runner in parallel. Union of the three must equal
-# ALL_CROSS_LINUX; adding a new linux arch goes into the matching family.
-ALL_CROSS_LINUX_ARM   = linux-arm-5 linux-arm-6 linux-arm-7 linux-arm64
-ALL_CROSS_LINUX_MIPS  = linux-mips linux-mipsle linux-mips64 linux-mips64le linux-mips-softfloat
-ALL_CROSS_LINUX_OTHER = linux-386 linux-ppc64le linux-riscv64 linux-loong64
-
 e2e:
 	$(TEST_ENV) go test -tags=e2e_testing -count=1 $(TEST_FLAGS) ./e2e
 
@@ -94,35 +82,6 @@ DOCKER_BIN = build/linux-amd64/nebula build/linux-amd64/nebula-cert
 
 all: $(ALL:%=build/%/nebula) $(ALL:%=build/%/nebula-cert)
 
-all-linux: $(ALL_LINUX:%=build/%/nebula) $(ALL_LINUX:%=build/%/nebula-cert)
-
-all-freebsd: $(ALL_FREEBSD:%=build/%/nebula) $(ALL_FREEBSD:%=build/%/nebula-cert)
-
-all-openbsd: $(ALL_OPENBSD:%=build/%/nebula) $(ALL_OPENBSD:%=build/%/nebula-cert)
-
-all-netbsd: $(ALL_NETBSD:%=build/%/nebula) $(ALL_NETBSD:%=build/%/nebula-cert)
-
-all-darwin: build/darwin-amd64/nebula build/darwin-amd64/nebula-cert build/darwin-arm64/nebula build/darwin-arm64/nebula-cert
-
-all-windows: build/windows-amd64/nebula.exe build/windows-amd64/nebula-cert.exe build/windows-arm64/nebula.exe build/windows-arm64/nebula-cert.exe
-
-# CI cross-build shards. darwin-arm64 is covered by the native macos-latest
-# job; windows-amd64 is covered by the native windows-latest job; both are
-# omitted here to avoid building them a second time. darwin-amd64 stays in
-# all-cross-darwin because intel mac is only a labeled/master-time native
-# job, so PRs still need cross-build coverage for it.
-all-cross-linux: $(ALL_CROSS_LINUX:%=build/%/nebula) $(ALL_CROSS_LINUX:%=build/%/nebula-cert)
-
-all-cross-linux-arm:   $(ALL_CROSS_LINUX_ARM:%=build/%/nebula)   $(ALL_CROSS_LINUX_ARM:%=build/%/nebula-cert)
-
-all-cross-linux-mips:  $(ALL_CROSS_LINUX_MIPS:%=build/%/nebula)  $(ALL_CROSS_LINUX_MIPS:%=build/%/nebula-cert)
-
-all-cross-linux-other: $(ALL_CROSS_LINUX_OTHER:%=build/%/nebula) $(ALL_CROSS_LINUX_OTHER:%=build/%/nebula-cert)
-
-all-cross-darwin: build/darwin-amd64/nebula build/darwin-amd64/nebula-cert
-
-all-cross-windows: build/windows-arm64/nebula.exe build/windows-arm64/nebula-cert.exe
-
 docker: docker/linux-$(shell go env GOARCH)
 
 release: $(ALL:%=build/nebula-%.tar.gz)
@@ -268,9 +227,6 @@ smoke-relay-docker: bin-docker
 	cd .github/workflows/smoke/ && ./build-relay.sh
 	cd .github/workflows/smoke/ && ./smoke-relay.sh
 
-smoke-docker-ipv6: export SMOKE_OVERLAY_IPV6 = 1
-smoke-docker-ipv6: smoke-docker
-
 smoke-docker-race: BUILD_ARGS = -race
 smoke-docker-race: CGO_ENABLED = 1
 smoke-docker-race: smoke-docker
@@ -280,5 +236,5 @@ smoke-vagrant/%: bin-docker build/%/nebula
 	cd .github/workflows/smoke/ && ./smoke-vagrant.sh $*
 
 .FORCE:
-.PHONY: all all-linux all-freebsd all-openbsd all-netbsd all-darwin all-windows all-cross-linux all-cross-linux-arm all-cross-linux-mips all-cross-linux-other all-cross-darwin all-cross-windows bench bench-cpu bench-cpu-long bin build-test-mobile e2e e2ev e2evv e2evvv e2evvvv proto release service smoke-docker smoke-docker-race test test-cov-html smoke-vagrant/%
+.PHONY: bench bench-cpu bench-cpu-long bin build-test-mobile e2e e2ev e2evv e2evvv e2evvvv proto release service smoke-docker smoke-docker-race test test-cov-html smoke-vagrant/%
 .DEFAULT_GOAL := bin

bits.go

@@ -2,42 +2,24 @@ package nebula
 
 import (
 	"context"
-	"fmt"
 	"log/slog"
-	"math"
-	mathbits "math/bits"
 
 	"github.com/rcrowley/go-metrics"
 )
 
-const bitsPerWord = 64
-
-// Bits is a sliding-window anti-replay tracker. The window is stored as a
-// circular bitmap packed into uint64 words (8x denser than a []bool), so a
-// length-N window costs N/8 bytes. length must be a power of two.
 type Bits struct {
 	length             uint64
-	lengthMask         uint64
 	current            uint64
-	bits               []uint64
+	bits               []bool
 	lostCounter        metrics.Counter
 	dupeCounter        metrics.Counter
 	outOfWindowCounter metrics.Counter
 }
 
-func NewBits(length uint64) *Bits {
+func NewBits(bits uint64) *Bits {
-	if length == 0 || length&(length-1) != 0 {
-		panic(fmt.Sprintf("Bits length must be a power of two, got %d", length))
-	}
-
-	nWords := length / bitsPerWord
-	if nWords == 0 {
-		nWords = 1
-	}
 	b := &Bits{
-		length:             length,
+		length:             bits,
-		lengthMask:         length - 1,
+		bits:               make([]bool, bits, bits),
-		bits:               make([]uint64, nWords),
 		current:            0,
 		lostCounter:        metrics.GetOrRegisterCounter("network.packets.lost", nil),
 		dupeCounter:        metrics.GetOrRegisterCounter("network.packets.duplicate", nil),
@@ -45,194 +27,71 @@ func NewBits(length uint64) *Bits {
 	}
 
 	// There is no counter value 0, mark it to avoid counting a lost packet later.
-	b.bits[0] = 1
+	b.bits[0] = true
+	b.current = 0
 	return b
 }
 
-func (b *Bits) get(i uint64) bool {
-	pos := i & b.lengthMask
-	//bit-shifting by 6 because i is a bit index, not a u64 index, and we need to find the u64 without bit in it
-	return b.bits[pos>>6]&(uint64(1)<<(pos&63)) != 0
-}
-
-func (b *Bits) set(i uint64) {
-	pos := i & b.lengthMask
-	b.bits[pos>>6] |= uint64(1) << (pos & 63)
-}
-
-// clearRange clears `count` bits starting at circular position `startPos`
-// (already masked to [0, length)) and returns how many of them were set
-// before the clear. count must be in [1, length].
-func (b *Bits) clearRange(startPos, count uint64) uint64 {
-	wasSet := uint64(0)
-	if count >= b.length {
-		for _, w := range b.bits {
-			wasSet += uint64(mathbits.OnesCount64(w))
-		}
-		clear(b.bits)
-		return wasSet
-	}
-
-	pos := startPos
-	remaining := count
-
-	// handle the potential partial word before pos becomes u64 aligned
-	word := pos >> 6
-	bit := pos & 63
-	take := uint64(64) - bit
-	if take > remaining {
-		take = remaining
-	}
-	if take > b.length-pos {
-		take = b.length - pos
-	}
-	var mask uint64
-	if take == 64 {
-		mask = math.MaxUint64
-	} else {
-		mask = ((uint64(1) << take) - 1) << bit
-	}
-	wasSet += uint64(mathbits.OnesCount64(b.bits[word] & mask))
-	b.bits[word] &^= mask
-	remaining -= take
-	pos = (pos + take) & b.lengthMask
-
-	// Clear whole words, keeping track of the number of set bits
-	for remaining >= 64 {
-		word = pos >> 6
-		wasSet += uint64(mathbits.OnesCount64(b.bits[word]))
-		b.bits[word] = 0
-		remaining -= 64
-		pos = (pos + 64) & b.lengthMask
-	}
-
-	// Clear the remaining partial word
-	if remaining > 0 {
-		word = pos >> 6
-		mask = (uint64(1) << remaining) - 1
-		wasSet += uint64(mathbits.OnesCount64(b.bits[word] & mask))
-		b.bits[word] &^= mask
-	}
-
-	return wasSet
-}
-
-func (b *Bits) strictlyWithinWindow(i uint64) bool {
-	// Handle the case where the window hasn't slid yet. This avoids u64 underflow.
-	inWarmup := b.current < b.length
-	if i < b.length && inWarmup {
-		return true
-	}
-
-	// Next, if the packet is in-window, see if we've seen it before
-	if i > b.current-b.length {
-		return true
-	}
-	return false //not within window!
-}
-
-// Check returns true if i is within (or way out in front of) the window, and not a replay
 func (b *Bits) Check(l *slog.Logger, i uint64) bool {
 	// If i is the next number, return true.
 	if i > b.current {
 		return true
 	}
 
-	if b.strictlyWithinWindow(i) {
+	// If i is within the window, check if it's been set already.
-		return !b.get(i)
+	if i > b.current-b.length || i < b.length && b.current < b.length {
+		return !b.bits[i%b.length]
 	}
 
 	// Not within the window
 	if l.Enabled(context.Background(), slog.LevelDebug) {
-		l.Debug("rejected a packet (top)", "current", b.current, "incoming", i)
+		l.Debug("rejected a packet (top)",
+			"current", b.current,
+			"incoming", i,
+		)
 	}
 	return false
 }
 
-// Update has three branches:
-//   - i == b.current+1: fast path; advance the cursor by one and lose-count
-//     the slot we just stomped (only past warmup; see the i > b.length guard
-//     below).
-//   - i  >  b.current+1: jump path; clear all slots between current and i
-//     (or up to a full window's worth, whichever is smaller) via clearRange,
-//     then mark i. Two arms here: a warmup arm that handles the very first
-//     window before the cursor has slid, and a steady-state arm that treats
-//     every cleared empty slot as a lost packet.
-//   - i  <= b.current: in-window check for duplicates; out-of-window otherwise.
-//
-// NewBits seeds bits[0]=1 so counter 0 looks "received" — Update never
-// clears that marker during warmup (clearRange skips position 0 when
-// startPos=1), and once b.current >= b.length the marker is no longer
-// consulted. The marker prevents a fictitious "lost" hit on the first real
-// counter.
 func (b *Bits) Update(l *slog.Logger, i uint64) bool {
-	// Fast path: i is the next expected counter. Split out so the function
+	// If i is the next number, return true and update current.
-	// stays small and avoids paying for the slow paths' slog argument-build
-	// stack frame on every call. The bit read/test/write is inlined to
-	// touch the backing word once.
 	if i == b.current+1 {
-		pos := i & b.lengthMask
+		// Check if the oldest bit was lost since we are shifting the window by 1 and occupying it with this counter
-		word := pos >> 6
+		// The very first window can only be tracked as lost once we are on the 2nd window or greater
-		mask := uint64(1) << (pos & 63)
+		if b.bits[i%b.length] == false && i > b.length {
-		w := b.bits[word]
-		if i > b.length && w&mask == 0 {
 			b.lostCounter.Inc(1)
 		}
-		b.bits[word] = w | mask
+		b.bits[i%b.length] = true
 		b.current = i
 		return true
 	}
-	return b.updateSlow(l, i)
-}
 
-// updateSlow handles jumps, in-window backfill, dupes, and out-of-window.
-func (b *Bits) updateSlow(l *slog.Logger, i uint64) bool {
 	// If i is a jump, adjust the window, record lost, update current, and return true
 	if i > b.current {
-		end := i
+		lost := int64(0)
-		if end > b.current+b.length {
+		// Zero out the bits between the current and the new counter value, limited by the window size,
-			end = b.current + b.length
+		// since the window is shifting
-		}
+		for n := b.current + 1; n <= min(i, b.current+b.length); n++ {
-		count := end - b.current
+			if b.bits[n%b.length] == false && n > b.length {
-		startPos := (b.current + 1) & b.lengthMask
+				lost++
-
-		var lost int64
-		if b.current >= b.length {
-			// Steady state: every cleared slot is past warmup, so any unset
-			// bit we evict is a lost packet from the previous cycle.
-			wasSet := b.clearRange(startPos, count)
-			lost = int64(count) - int64(wasSet)
-		} else {
-			// Warmup (the very first window). Some cleared slots represent
-			// packets <= length where eviction is not "lost" in the usual
-			// sense. This branch is taken at most once per connection so we
-			// don't bother optimizing it.
-			for n := b.current + 1; n <= end; n++ {
-				if !b.get(n) && n > b.length {
-					lost++
-				}
 			}
-			b.clearRange(startPos, count)
+			b.bits[n%b.length] = false
 		}
 
-		// Anything past the new window can never be backfilled, so it's lost.
+		// Only record any skipped packets as a result of the window moving further than the window length
-		if i > b.current+b.length {
+		// Any loss within the new window will be accounted for in future calls
-			lost += int64(i - b.current - b.length)
+		lost += max(0, int64(i-b.current-b.length))
-		}
 		b.lostCounter.Inc(lost)
 
-		b.set(i)
+		b.bits[i%b.length] = true
 		b.current = i
 		return true
 	}
 
-	// If i is within the current window but below the current counter, check to see if it's a duplicate
+	// If i is within the current window but below the current counter,
-	if b.strictlyWithinWindow(i) {
+	// Check to see if it's a duplicate
-		pos := i & b.lengthMask
+	if i > b.current-b.length || i < b.length && b.current < b.length {
-		word := pos >> 6
+		if b.current == i || b.bits[i%b.length] == true {
-		mask := uint64(1) << (pos & 63)
-		w := b.bits[word]
-		if b.current == i || w&mask != 0 {
 			if l.Enabled(context.Background(), slog.LevelDebug) {
 				l.Debug("Receive window",
 					"accepted", false,
@@ -245,7 +104,7 @@ func (b *Bits) updateSlow(l *slog.Logger, i uint64) bool {
 			return false
 		}
 
-		b.bits[word] = w | mask
+		b.bits[i%b.length] = true
 		return true
 	}
 

bits_test.go

@@ -7,79 +7,61 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
-// snapshot returns the bitmap as a []bool of length b.length, for readable
-// test assertions against the now-packed []uint64 storage.
-func (b *Bits) snapshot() []bool {
-	out := make([]bool, b.length)
-	for i := uint64(0); i < b.length; i++ {
-		out[i] = b.get(i)
-	}
-	return out
-}
-
-func TestBitsRequiresPowerOfTwo(t *testing.T) {
-	assert.Panics(t, func() { NewBits(10) })
-	assert.Panics(t, func() { NewBits(0) })
-	assert.NotPanics(t, func() { NewBits(1) })
-	assert.NotPanics(t, func() { NewBits(16) })
-	assert.NotPanics(t, func() { NewBits(1024) })
-	assert.NotPanics(t, func() { NewBits(16384) })
-}
-
 func TestBits(t *testing.T) {
 	l := test.NewLogger()
-	b := NewBits(16)
+	b := NewBits(10)
-	assert.EqualValues(t, 16, b.length)
+
+	// make sure it is the right size
+	assert.Len(t, b.bits, 10)
 
 	// This is initialized to zero - receive one. This should work.
 	assert.True(t, b.Check(l, 1))
 	assert.True(t, b.Update(l, 1))
 	assert.EqualValues(t, 1, b.current)
-	g := []bool{true, true, false, false, false, false, false, false, false, false, false, false, false, false, false, false}
+	g := []bool{true, true, false, false, false, false, false, false, false, false}
-	assert.Equal(t, g, b.snapshot())
+	assert.Equal(t, g, b.bits)
 
 	// Receive two
 	assert.True(t, b.Check(l, 2))
 	assert.True(t, b.Update(l, 2))
 	assert.EqualValues(t, 2, b.current)
-	g = []bool{true, true, true, false, false, false, false, false, false, false, false, false, false, false, false, false}
+	g = []bool{true, true, true, false, false, false, false, false, false, false}
-	assert.Equal(t, g, b.snapshot())
+	assert.Equal(t, g, b.bits)
 
 	// Receive two again - it will fail
 	assert.False(t, b.Check(l, 2))
 	assert.False(t, b.Update(l, 2))
 	assert.EqualValues(t, 2, b.current)
 
-	// Jump ahead to 25, which clears the window and sets slot 25%16 = 9.
+	// Jump ahead to 15, which should clear everything and set the 6th element
-	assert.True(t, b.Check(l, 25))
+	assert.True(t, b.Check(l, 15))
-	assert.True(t, b.Update(l, 25))
+	assert.True(t, b.Update(l, 15))
-	assert.EqualValues(t, 25, b.current)
+	assert.EqualValues(t, 15, b.current)
-	g = []bool{false, false, false, false, false, false, false, false, false, true, false, false, false, false, false, false}
+	g = []bool{false, false, false, false, false, true, false, false, false, false}
-	assert.Equal(t, g, b.snapshot())
+	assert.Equal(t, g, b.bits)
 
-	// Mark 24, which is in window (current 25, length 16, window covers [10,25]).
+	// Mark 14, which is allowed because it is in the window
-	assert.True(t, b.Check(l, 24))
+	assert.True(t, b.Check(l, 14))
-	assert.True(t, b.Update(l, 24))
+	assert.True(t, b.Update(l, 14))
-	assert.EqualValues(t, 25, b.current)
+	assert.EqualValues(t, 15, b.current)
-	g = []bool{false, false, false, false, false, false, false, false, true, true, false, false, false, false, false, false}
+	g = []bool{false, false, false, false, true, true, false, false, false, false}
-	assert.Equal(t, g, b.snapshot())
+	assert.Equal(t, g, b.bits)
 
-	// Mark 5, not allowed because 5 <= current-length (25-16=9).
+	// Mark 5, which is not allowed because it is not in the window
 	assert.False(t, b.Check(l, 5))
 	assert.False(t, b.Update(l, 5))
-	assert.EqualValues(t, 25, b.current)
+	assert.EqualValues(t, 15, b.current)
-	g = []bool{false, false, false, false, false, false, false, false, true, true, false, false, false, false, false, false}
+	g = []bool{false, false, false, false, true, true, false, false, false, false}
-	assert.Equal(t, g, b.snapshot())
+	assert.Equal(t, g, b.bits)
 
-	// Make sure we handle wrapping around once to the same slot. With
+	// make sure we handle wrapping around once to the current position
-	// length=16, packets 1 and 17 share slot 1.
+	b = NewBits(10)
-	b = NewBits(16)
 	assert.True(t, b.Update(l, 1))
-	assert.True(t, b.Update(l, 17))
+	assert.True(t, b.Update(l, 11))
-	assert.Equal(t, []bool{false, true, false, false, false, false, false, false, false, false, false, false, false, false, false, false}, b.snapshot())
+	assert.Equal(t, []bool{false, true, false, false, false, false, false, false, false, false}, b.bits)
 
 	// Walk through a few windows in order
-	b = NewBits(16)
+	b = NewBits(10)
 	for i := uint64(1); i <= 100; i++ {
 		assert.True(t, b.Check(l, i), "Error while checking %v", i)
 		assert.True(t, b.Update(l, i), "Error while updating %v", i)
@@ -90,31 +72,24 @@ func TestBits(t *testing.T) {
 
 func TestBitsLargeJumps(t *testing.T) {
 	l := test.NewLogger()
-
+	b := NewBits(10)
-	// length=16. Update(55) from current=0:
-	//   warmup, per-bit loop sees no n>16 with unset bits (slot 0 was set by
-	//   NewBits and gets re-evaluated when n=16; n=16 is not strictly > 16),
-	//   so the loop contributes 0. The jump exceeds the window so we record
-	//   55 - 0 - 16 = 39 packets fell out the back.
-	b := NewBits(16)
 	b.lostCounter.Clear()
-	assert.True(t, b.Update(l, 55))
-	assert.Equal(t, int64(39), b.lostCounter.Count())
 
-	// Update(100): clears 16 slots starting at slot 56%16=8. Only slot 7 (for
+	b = NewBits(10)
-	// packet 55) was set, so 16 - 1 = 15 evicted slots had unset bits.
+	b.lostCounter.Clear()
-	// Plus 100 - 55 - 16 = 29 packets fell past the window. Total 44.
+	assert.True(t, b.Update(l, 55)) // We saw packet 55 and can still track 45,46,47,48,49,50,51,52,53,54
-	assert.True(t, b.Update(l, 100))
+	assert.Equal(t, int64(45), b.lostCounter.Count())
-	assert.Equal(t, int64(39+44), b.lostCounter.Count())
 
-	// Update(200): same shape: 16 - 1 = 15 evicted unset, plus 200 - 100 - 16 = 84 past window. Total 99.
+	assert.True(t, b.Update(l, 100)) // We saw packet 55 and 100 and can still track 90,91,92,93,94,95,96,97,98,99
-	assert.True(t, b.Update(l, 200))
+	assert.Equal(t, int64(89), b.lostCounter.Count())
-	assert.Equal(t, int64(39+44+99), b.lostCounter.Count())
+
+	assert.True(t, b.Update(l, 200)) // We saw packet 55, 100, and 200 and can still track 190,191,192,193,194,195,196,197,198,199
+	assert.Equal(t, int64(188), b.lostCounter.Count())
 }
 
 func TestBitsDupeCounter(t *testing.T) {
 	l := test.NewLogger()
-	b := NewBits(16)
+	b := NewBits(10)
 	b.lostCounter.Clear()
 	b.dupeCounter.Clear()
 	b.outOfWindowCounter.Clear()
@@ -139,117 +114,120 @@ func TestBitsDupeCounter(t *testing.T) {
 
 func TestBitsOutOfWindowCounter(t *testing.T) {
 	l := test.NewLogger()
-	b := NewBits(16)
+	b := NewBits(10)
 	b.lostCounter.Clear()
 	b.dupeCounter.Clear()
 	b.outOfWindowCounter.Clear()
 
-	// Jump to 20 (warmup branch + 4 past-window packets).
 	assert.True(t, b.Update(l, 20))
 	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())
 
-	// 9 single-step advances, each evicts a slot whose bit was cleared during
+	assert.True(t, b.Update(l, 21))
-	// the jump above and whose value was never seen, so each contributes 1
+	assert.True(t, b.Update(l, 22))
-	// to lostCounter.
+	assert.True(t, b.Update(l, 23))
-	for n := uint64(21); n <= 29; n++ {
+	assert.True(t, b.Update(l, 24))
-		assert.True(t, b.Update(l, n))
+	assert.True(t, b.Update(l, 25))
-	}
+	assert.True(t, b.Update(l, 26))
+	assert.True(t, b.Update(l, 27))
+	assert.True(t, b.Update(l, 28))
+	assert.True(t, b.Update(l, 29))
 	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())
 
-	// 0 is below current-length (29-16=13) so it falls outside the window.
 	assert.False(t, b.Update(l, 0))
 	assert.Equal(t, int64(1), b.outOfWindowCounter.Count())
 
-	// 4 from the Update(20) jump + 9 from 21..29.
+	assert.Equal(t, int64(19), b.lostCounter.Count()) // packet 0 wasn't lost
-	assert.Equal(t, int64(13), b.lostCounter.Count())
 	assert.Equal(t, int64(0), b.dupeCounter.Count())
 	assert.Equal(t, int64(1), b.outOfWindowCounter.Count())
 }
 
 func TestBitsLostCounter(t *testing.T) {
 	l := test.NewLogger()
-	b := NewBits(16)
+	b := NewBits(10)
 	b.lostCounter.Clear()
 	b.dupeCounter.Clear()
 	b.outOfWindowCounter.Clear()
 
-	// Walk 20..29 like the original, just with a bigger window. Same
+	assert.True(t, b.Update(l, 20))
-	// reasoning as TestBitsOutOfWindowCounter: 4 past-window from Update(20),
+	assert.True(t, b.Update(l, 21))
-	// then 9 more from the unit advances.
+	assert.True(t, b.Update(l, 22))
-	for n := uint64(20); n <= 29; n++ {
+	assert.True(t, b.Update(l, 23))
-		assert.True(t, b.Update(l, n))
+	assert.True(t, b.Update(l, 24))
-	}
+	assert.True(t, b.Update(l, 25))
-	assert.Equal(t, int64(13), b.lostCounter.Count())
+	assert.True(t, b.Update(l, 26))
+	assert.True(t, b.Update(l, 27))
+	assert.True(t, b.Update(l, 28))
+	assert.True(t, b.Update(l, 29))
+	assert.Equal(t, int64(19), b.lostCounter.Count()) // packet 0 wasn't lost
 	assert.Equal(t, int64(0), b.dupeCounter.Count())
 	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())
 
-	b = NewBits(16)
+	b = NewBits(10)
 	b.lostCounter.Clear()
 	b.dupeCounter.Clear()
 	b.outOfWindowCounter.Clear()
 
-	// Update(15) clears the warmup window (no lost), sets slot 15.
+	assert.True(t, b.Update(l, 9))
-	assert.True(t, b.Update(l, 15))
 	assert.Equal(t, int64(0), b.lostCounter.Count())
-
+	// 10 will set 0 index, 0 was already set, no lost packets
-	// Update(16): slot 0 was already set (NewBits seeded it), and 16 is not
+	assert.True(t, b.Update(l, 10))
-	// strictly > length, so nothing is recorded as lost.
-	assert.True(t, b.Update(l, 16))
 	assert.Equal(t, int64(0), b.lostCounter.Count())
-
+	// 11 will set 1 index, 1 was missed, we should see 1 packet lost
-	// Update(17): we jumped straight from 0 to 15, so slot 1 was cleared
+	assert.True(t, b.Update(l, 11))
-	// (and never re-set). 17 > 16 is past warmup, so packet 1 is recorded lost.
-	assert.True(t, b.Update(l, 17))
 	assert.Equal(t, int64(1), b.lostCounter.Count())
+	// Now let's fill in the window, should end up with 8 lost packets
+	assert.True(t, b.Update(l, 12))
+	assert.True(t, b.Update(l, 13))
+	assert.True(t, b.Update(l, 14))
+	assert.True(t, b.Update(l, 15))
+	assert.True(t, b.Update(l, 16))
+	assert.True(t, b.Update(l, 17))
+	assert.True(t, b.Update(l, 18))
+	assert.True(t, b.Update(l, 19))
+	assert.Equal(t, int64(8), b.lostCounter.Count())
 
-	// Fill in 18..30 in single steps. Each i evicts slot i%16. Slots 2..14
+	// Jump ahead by a window size
-	// were all cleared during Update(15), and we never re-set any of them,
+	assert.True(t, b.Update(l, 29))
-	// so each i in 18..30 is a fresh lost packet — 13 more.
+	assert.Equal(t, int64(8), b.lostCounter.Count())
-	for n := uint64(18); n <= 30; n++ {
+	// Now lets walk ahead normally through the window, the missed packets should fill in
-		assert.True(t, b.Update(l, n))
+	assert.True(t, b.Update(l, 30))
-	}
+	assert.True(t, b.Update(l, 31))
-	assert.Equal(t, int64(14), b.lostCounter.Count())
+	assert.True(t, b.Update(l, 32))
+	assert.True(t, b.Update(l, 33))
+	assert.True(t, b.Update(l, 34))
+	assert.True(t, b.Update(l, 35))
+	assert.True(t, b.Update(l, 36))
+	assert.True(t, b.Update(l, 37))
+	assert.True(t, b.Update(l, 38))
+	// 39 packets tracked, 22 seen, 17 lost
+	assert.Equal(t, int64(17), b.lostCounter.Count())
 
-	// Jump ahead by exactly one window size.
+	// Jump ahead by 2 windows, should have recording 1 full window missing
-	assert.True(t, b.Update(l, 46))
+	assert.True(t, b.Update(l, 58))
-	// end = min(46, 30+16) = 46, count = 16, all slots cleared. Before the
+	assert.Equal(t, int64(27), b.lostCounter.Count())
-	// jump every slot 0..15 had been set (Update(15), (16), (17), 18..30),
+	// Now lets walk ahead normally through the window, the missed packets should fill in from this window
-	// so wasSet=16 and 46 == current+length means no past-window slack:
+	assert.True(t, b.Update(l, 59))
-	// lost contribution = 0.
+	assert.True(t, b.Update(l, 60))
-	assert.Equal(t, int64(14), b.lostCounter.Count())
+	assert.True(t, b.Update(l, 61))
-
+	assert.True(t, b.Update(l, 62))
-	// Walk 47..55. The Update(46) jump cleared every slot, so only slot 14
+	assert.True(t, b.Update(l, 63))
-	// (for packet 46) is set when we start. Each subsequent unit step lands
+	assert.True(t, b.Update(l, 64))
-	// on a slot that was cleared and is past warmup, so it counts as lost.
+	assert.True(t, b.Update(l, 65))
-	// 9 more = 23.
+	assert.True(t, b.Update(l, 66))
-	for n := uint64(47); n <= 55; n++ {
+	assert.True(t, b.Update(l, 67))
-		assert.True(t, b.Update(l, n))
+	// 68 packets tracked, 32 seen, 36 missed
-	}
+	assert.Equal(t, int64(36), b.lostCounter.Count())
-	assert.Equal(t, int64(23), b.lostCounter.Count())
-
-	// Jump ahead by two windows: clears the window plus past-window loss.
-	assert.True(t, b.Update(l, 87))
-	// current=55, length=16. end = min(87, 71) = 71. count=16, all slots
-	// cleared. Slots set before the clear are slots 14,15,0..7 (10 total).
-	// Lost from clear = 16 - 10 = 6. Past window: 87 - 55 - 16 = 16. +22.
-	assert.Equal(t, int64(45), b.lostCounter.Count())
 	assert.Equal(t, int64(0), b.dupeCounter.Count())
 	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())
 }
 
 func TestBitsLostCounterIssue1(t *testing.T) {
 	l := test.NewLogger()
-	b := NewBits(16)
+	b := NewBits(10)
 	b.lostCounter.Clear()
 	b.dupeCounter.Clear()
 	b.outOfWindowCounter.Clear()
 
-	// Receive 4, backfill 1, then 9, 2, 3, 5, 6, 7 (skip 8), 10, 11, 14.
-	// Then jump to 25 — slot 25%16=9 is being evicted, but it had been set
-	// (we received packet 9), so no spurious lost increment. The original
-	// regression was about double-counting a missing packet when its slot
-	// got cleared on a jump. With the jump path now using clearRange's
-	// word-level wasSet count, the same semantics hold.
 	assert.True(t, b.Update(l, 4))
 	assert.Equal(t, int64(0), b.lostCounter.Count())
 	assert.True(t, b.Update(l, 1))
@@ -266,7 +244,7 @@ func TestBitsLostCounterIssue1(t *testing.T) {
 	assert.Equal(t, int64(0), b.lostCounter.Count())
 	assert.True(t, b.Update(l, 7))
 	assert.Equal(t, int64(0), b.lostCounter.Count())
-	// Skip packet 8.
+	// assert.True(t, b.Update(l, 8))
 	assert.True(t, b.Update(l, 10))
 	assert.Equal(t, int64(0), b.lostCounter.Count())
 	assert.True(t, b.Update(l, 11))
@@ -274,23 +252,9 @@ func TestBitsLostCounterIssue1(t *testing.T) {
 
 	assert.True(t, b.Update(l, 14))
 	assert.Equal(t, int64(0), b.lostCounter.Count())
-
+	// Issue seems to be here, we reset missing packet 8 to false here and don't increment the lost counter
-	// Jump to 25. With length=16, slot 25%16=9 corresponds to packet 9
+	assert.True(t, b.Update(l, 19))
-	// (which we DID receive), so its bit is set and no lost++ from that
-	// eviction. The trace below shows the only loss is packet 8.
-	assert.True(t, b.Update(l, 25))
-	// current was 14, i=25. end=min(25,30)=25. count=11. startPos=15.
-	// steady? current=14<16, so warmup branch: per-bit n=15..25, count those
-	// with !get(n) AND n>16. n=17..25 are >16. Among slots 17%16=1..25%16=9
-	// did we set slots 1..9 (packets 1..9)? Yes for all but slot 8 (packet 8
-	// was skipped). n=24 maps to slot 8 which is FALSE → lost++. All other
-	// n in 17..25 map to slots that are set. n=16 is not strictly > 16. So
-	// lost = 1.
 	assert.Equal(t, int64(1), b.lostCounter.Count())
-
-	// Fill in 12, 13, 15, 16. Each is below current=25 (in-window). 16 must
-	// recheck slot 0 — it was set by NewBits and then cleared by the
-	// Update(25) jump, so 16 backfills cleanly.
 	assert.True(t, b.Update(l, 12))
 	assert.Equal(t, int64(1), b.lostCounter.Count())
 	assert.True(t, b.Update(l, 13))
@@ -299,140 +263,29 @@ func TestBitsLostCounterIssue1(t *testing.T) {
 	assert.Equal(t, int64(1), b.lostCounter.Count())
 	assert.True(t, b.Update(l, 16))
 	assert.Equal(t, int64(1), b.lostCounter.Count())
+	assert.True(t, b.Update(l, 17))
+	assert.Equal(t, int64(1), b.lostCounter.Count())
+	assert.True(t, b.Update(l, 18))
+	assert.Equal(t, int64(1), b.lostCounter.Count())
+	assert.True(t, b.Update(l, 20))
+	assert.Equal(t, int64(1), b.lostCounter.Count())
+	assert.True(t, b.Update(l, 21))
 
-	// We missed packet 8 above and that loss is still recorded once, never
+	// We missed packet 8 above
-	// double-counted, never zeroed.
 	assert.Equal(t, int64(1), b.lostCounter.Count())
 	assert.Equal(t, int64(0), b.dupeCounter.Count())
 	assert.Equal(t, int64(0), b.outOfWindowCounter.Count())
 }
 
-// TestBitsWarmupOvershoot exercises the jump path's warmup arm with an
+func BenchmarkBits(b *testing.B) {
-// overshoot past one full window. NewBits leaves current=0 with only slot 0
+	z := NewBits(10)
-// "set" by the marker. Jumping straight to length+k must (a) clear every
-// slot the jump straddles, (b) count only past-window slack (not the
-// in-window slots, which never had a "lost" tenant during warmup), and
-// (c) leave the cursor at the new counter so subsequent unit advances
-// count from steady state. The marker bit at slot 0 is irrelevant once
-// current >= length.
-func TestBitsWarmupOvershoot(t *testing.T) {
-	l := test.NewLogger()
-	b := NewBits(16)
-	b.lostCounter.Clear()
-
-	// Jump from current=0 to i=20 (length=16, overshoot=4).
-	// Warmup arm: counts slots in [1..16] where bit unset and n>length.
-	// Only n=16 was unset and >length: but slot 16%16=0 is the marker,
-	// so b.get(16) reads bits[0]=1 and skips. Result: 0 lost from the loop.
-	// Past-window: i - current - length = 20 - 0 - 16 = 4 lost.
-	assert.True(t, b.Update(l, 20))
-	assert.Equal(t, int64(4), b.lostCounter.Count())
-	assert.Equal(t, uint64(20), b.current)
-
-	// Steady state now (current=20 >= length=16). Unit advance to 21
-	// stomps slot 21%16=5, which was cleared by the jump and not reset,
-	// so this is +1 lost.
-	assert.True(t, b.Update(l, 21))
-	assert.Equal(t, int64(5), b.lostCounter.Count())
-}
-
-// TestBitsCheckAcrossWarmupBoundary pins the underflow trick in Check's
-// in-window clause. While in warmup, b.current-b.length underflows uint64
-// to a huge value so the first OR-clause is always false; the second
-// clause (i < length && current < length) carries the in-window check.
-// Once current >= length the regimes flip cleanly.
-func TestBitsCheckAcrossWarmupBoundary(t *testing.T) {
-	l := test.NewLogger()
-	b := NewBits(16)
-
-	// Warmup: current=0. Check(0) must read the marker (set) and return false.
-	assert.False(t, b.Check(l, 0), "marker slot should look already-received")
-	// Warmup: any 0 < i < length is in-window and unset → accepted.
-	for i := uint64(1); i < 16; i++ {
-		assert.True(t, b.Check(l, i), "warmup in-window i=%d should be accepted", i)
-	}
-	// Warmup: i >= length but > current is "next number" so accepted.
-	assert.True(t, b.Check(l, 16))
-	assert.True(t, b.Check(l, 1_000_000))
-
-	// Cross into steady state.
-	assert.True(t, b.Update(l, 100))
-	// Now current=100, length=16. In-window range is [85..100].
-	// 84 is just outside: the underflow clause activates; 84 > 100-16=84 is false.
-	// And the warmup clause is false (current >= length). So out of window.
-	assert.False(t, b.Check(l, 84))
-	// 85 sits at the boundary. 85 > 84 is true → in window, unset → accept.
-	assert.True(t, b.Check(l, 85))
-	// 100 is current itself; not strictly greater, in-window, but already set.
-	assert.False(t, b.Check(l, 100))
-	// Way out: clearly out of window.
-	assert.False(t, b.Check(l, 50))
-}
-
-// TestBitsMarkerInvariant verifies the seeded bits[0]=1 marker behaves
-// correctly across warmup and beyond. Update should never clear the marker
-// during warmup (clearRange skips position 0 when startPos=1), and once
-// current >= length the marker is no longer consulted by Check/Update on
-// the live path — but it must still report counter 0 as a duplicate while
-// we are in warmup.
-func TestBitsMarkerInvariant(t *testing.T) {
-	l := test.NewLogger()
-	b := NewBits(8)
-
-	// Counter 0 is the seeded marker; Check sees it as already received.
-	assert.False(t, b.Check(l, 0))
-	// Update(0) at current=0 hits the duplicate branch.
-	b.dupeCounter.Clear()
-	assert.False(t, b.Update(l, 0))
-	assert.Equal(t, int64(1), b.dupeCounter.Count())
-
-	// Walk forward through warmup; the marker must remain set.
-	for n := uint64(1); n <= 7; n++ {
-		assert.True(t, b.Update(l, n))
-	}
-	// Position 0 (the marker) should still read as set because we never
-	// cleared it; Update(0) still looks like a duplicate.
-	assert.False(t, b.Check(l, 0))
-
-	// Cross into steady state with a unit advance to 8: pos=0, evicts the
-	// marker bit. The lost-counter guard (i > b.length) is false (8 == 8),
-	// so this advance does NOT charge a lost packet — exactly what the
-	// marker is there to prevent.
-	b.lostCounter.Clear()
-	assert.True(t, b.Update(l, 8))
-	assert.Equal(t, int64(0), b.lostCounter.Count())
-	// The slot at pos 0 is now occupied by counter 8.
-	assert.False(t, b.Check(l, 8))
-}
-
-// BenchmarkBitsUpdateInOrder is the steady-state hot path: each call is
-// i == current+1.
-func BenchmarkBitsUpdateInOrder(b *testing.B) {
-	l := test.NewLogger()
-	z := NewBits(16384)
 	for n := 0; n < b.N; n++ {
-		z.Update(l, uint64(n)+1)
+		for i := range z.bits {
-	}
+			z.bits[i] = true
-}
+		}
+		for i := range z.bits {
+			z.bits[i] = false
+		}
 
-// BenchmarkBitsUpdateReorder simulates light reorder within the window:
-// every other packet arrives one slot behind its predecessor (forces the
-// in-window backfill branch).
-func BenchmarkBitsUpdateReorder(b *testing.B) {
-	l := test.NewLogger()
-	z := NewBits(16384)
-	for n := 0; n < b.N; n++ {
-		base := uint64(n) * 2
-		z.Update(l, base+2)
-		z.Update(l, base+1)
-	}
-}
-
-// BenchmarkBitsUpdateLargeJumps stresses the clearRange word-level path.
-func BenchmarkBitsUpdateLargeJumps(b *testing.B) {
-	l := test.NewLogger()
-	z := NewBits(16384)
-	for n := 0; n < b.N; n++ {
-		z.Update(l, uint64(n+1)*1000)
 	}
 }

cert/ca_pool.go

@@ -217,10 +217,6 @@ func (ncp *CAPool) verify(c Certificate, now time.Time, certFp string, signerFp
 		return nil, err
 	}
 
-	if signer.Certificate.Curve() != c.Curve() {
-		return nil, ErrCurveMismatch
-	}
-
 	if signer.Certificate.Expired(now) {
 		return nil, ErrRootExpired
 	}

cert/ca_pool_test.go

@@ -654,31 +654,3 @@ func TestCertificateV2_Verify_Subnets(t *testing.T) {
 	_, err = caPool.VerifyCertificate(time.Now(), c)
 	require.NoError(t, err)
 }
-
-func TestCertificateV2_CurveMismatch(t *testing.T) {
-	caIp1 := mustParsePrefixUnmapped("10.0.0.0/16")
-	caIp2 := mustParsePrefixUnmapped("192.168.0.0/24")
-	ca, _, caKey, _ := NewTestCaCert(Version2, Curve_P256, time.Now(), time.Now().Add(10*time.Minute), []netip.Prefix{caIp1, caIp2}, nil, []string{"test"})
-
-	caPem, err := ca.MarshalPEM()
-	require.NoError(t, err)
-
-	caPool := NewCAPool()
-	b, err := caPool.AddCAFromPEM(caPem)
-	require.NoError(t, err)
-	assert.Empty(t, b)
-
-	// ip is outside the network
-	cIp1 := mustParsePrefixUnmapped("10.0.0.1/24")
-	c, _, _, _ := NewTestCert(Version2, Curve_P256, ca, caKey, "test", time.Now(), time.Now().Add(5*time.Minute), []netip.Prefix{cIp1}, nil, []string{"test"})
-
-	fp, _ := c.Fingerprint()
-	_, err = caPool.verify(c, time.Now(), fp, c.Issuer())
-	require.NoError(t, err)
-	//
-	c2 := c.(*certificateV2)
-	c2.curve = Curve_CURVE25519
-	fp, _ = c.Fingerprint()
-	_, err = caPool.verify(c, time.Now(), fp, c.Issuer())
-	require.Error(t, err)
-}

cert/cert_v1.go

@@ -112,9 +112,6 @@ func (c *certificateV1) CheckSignature(key []byte) bool {
 	}
 	switch c.details.curve {
 	case Curve_CURVE25519:
-		if len(key) != ed25519.PublicKeySize {
-			return false //avoids a panic internal to ed25519
-		}
 		return ed25519.Verify(key, b, c.signature)
 	case Curve_P256:
 		pubKey, err := ecdsa.ParseUncompressedPublicKey(elliptic.P256(), key)

cert/cert_v2.go

@@ -151,9 +151,6 @@ func (c *certificateV2) CheckSignature(key []byte) bool {
 
 	switch c.curve {
 	case Curve_CURVE25519:
-		if len(key) != ed25519.PublicKeySize {
-			return false //avoids a panic internal to ed25519
-		}
 		return ed25519.Verify(key, b, c.signature)
 	case Curve_P256:
 		pubKey, err := ecdsa.ParseUncompressedPublicKey(elliptic.P256(), key)

cert/errors.go

@@ -22,7 +22,6 @@ var (
 	ErrCaNotFound                 = errors.New("could not find ca for the certificate")
 	ErrUnknownVersion             = errors.New("certificate version unrecognized")
 	ErrCertPubkeyPresent          = errors.New("certificate has unexpected pubkey present")
-	ErrCurveMismatch              = errors.New("certificate curve does not match CA")
 
 	ErrInvalidPEMBlock                   = errors.New("input did not contain a valid PEM encoded block")
 	ErrInvalidPEMCertificateBanner       = errors.New("bytes did not contain a proper certificate banner")

cert/helper_test.go

@@ -13,12 +13,6 @@ import (
 	"golang.org/x/crypto/ed25519"
 )
 
-// testCertNow is the reference "now" used to derive default before/after times
-// in NewTestCaCert and NewTestCert. Holding it fixed for the lifetime of the
-// test binary keeps CA and leaf defaults aligned at the same second, so a leaf
-// signed with default times can never expire after its CA on a rounding race.
-var testCertNow = time.Now().Round(time.Second)
-
 // NewTestCaCert will create a new ca certificate
 func NewTestCaCert(version Version, curve Curve, before, after time.Time, networks, unsafeNetworks []netip.Prefix, groups []string) (Certificate, []byte, []byte, []byte) {
 	var err error
@@ -40,10 +34,10 @@ func NewTestCaCert(version Version, curve Curve, before, after time.Time, networ
 	}
 
 	if before.IsZero() {
-		before = testCertNow.Add(time.Second * -60)
+		before = time.Now().Add(time.Second * -60).Round(time.Second)
 	}
 	if after.IsZero() {
-		after = testCertNow.Add(time.Second * 60)
+		after = time.Now().Add(time.Second * 60).Round(time.Second)
 	}
 
 	t := &TBSCertificate{
@@ -76,11 +70,11 @@ func NewTestCaCert(version Version, curve Curve, before, after time.Time, networ
 // Expiry times are defaulted if you do not pass them in
 func NewTestCert(v Version, curve Curve, ca Certificate, key []byte, name string, before, after time.Time, networks, unsafeNetworks []netip.Prefix, groups []string) (Certificate, []byte, []byte, []byte) {
 	if before.IsZero() {
-		before = testCertNow.Add(time.Second * -60)
+		before = time.Now().Add(time.Second * -60).Round(time.Second)
 	}
 
 	if after.IsZero() {
-		after = testCertNow.Add(time.Second * 60)
+		after = time.Now().Add(time.Second * 60).Round(time.Second)
 	}
 
 	if len(networks) == 0 {

cert/pem.go

@@ -148,9 +148,6 @@ func MarshalSigningPublicKeyToPEM(curve Curve, b []byte) []byte {
 	}
 }
 
-// UnmarshalPublicKeyFromPEM will try to unmarshal the first pem block in a byte array, returning any non
-// consumed data or an error on failure. Only key-agreement (ECDH) public key banners are accepted.
-// Use UnmarshalSigningPublicKeyFromPEM for Ed25519/ECDSA banners.
 func UnmarshalPublicKeyFromPEM(b []byte) ([]byte, []byte, Curve, error) {
 	k, r := pem.Decode(b)
 	if k == nil {
@@ -159,10 +156,10 @@ func UnmarshalPublicKeyFromPEM(b []byte) ([]byte, []byte, Curve, error) {
 	var expectedLen int
 	var curve Curve
 	switch k.Type {
-	case X25519PublicKeyBanner:
+	case X25519PublicKeyBanner, Ed25519PublicKeyBanner:
 		expectedLen = 32
 		curve = Curve_CURVE25519
-	case P256PublicKeyBanner:
+	case P256PublicKeyBanner, ECDSAP256PublicKeyBanner:
 		// Uncompressed
 		expectedLen = 65
 		curve = Curve_P256
@@ -175,33 +172,6 @@ func UnmarshalPublicKeyFromPEM(b []byte) ([]byte, []byte, Curve, error) {
 	return k.Bytes, r, curve, nil
 }
 
-// UnmarshalSigningPublicKeyFromPEM will try to unmarshal the first pem block in a byte array, returning any non
-// consumed data or an error on failure. Only Ed25519/ECDSA public key banners are accepted.
-// Use UnmarshalPublicKeyFromPEM for X25519/P256 (ECDH) banners.
-func UnmarshalSigningPublicKeyFromPEM(b []byte) ([]byte, []byte, Curve, error) {
-	k, r := pem.Decode(b)
-	if k == nil {
-		return nil, r, 0, fmt.Errorf("input did not contain a valid PEM encoded block")
-	}
-	var expectedLen int
-	var curve Curve
-	switch k.Type {
-	case Ed25519PublicKeyBanner:
-		expectedLen = 32
-		curve = Curve_CURVE25519
-	case ECDSAP256PublicKeyBanner:
-		// Uncompressed
-		expectedLen = 65
-		curve = Curve_P256
-	default:
-		return nil, r, 0, fmt.Errorf("bytes did not contain a proper Ed25519/ECDSA public key banner")
-	}
-	if len(k.Bytes) != expectedLen {
-		return nil, r, 0, fmt.Errorf("key was not %d bytes, is invalid %s public key", expectedLen, curve)
-	}
-	return k.Bytes, r, curve, nil
-}
-
 func MarshalPrivateKeyToPEM(curve Curve, b []byte) []byte {
 	switch curve {
 	case Curve_CURVE25519:

cert/pem_test.go

@@ -255,6 +255,60 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 func TestUnmarshalPublicKeyFromPEM(t *testing.T) {
 	t.Parallel()
 	pubKey := []byte(`# A good key
+-----BEGIN NEBULA ED25519 PUBLIC KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+-----END NEBULA ED25519 PUBLIC KEY-----
+`)
+	shortKey := []byte(`# A short key
+-----BEGIN NEBULA ED25519 PUBLIC KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==
+-----END NEBULA ED25519 PUBLIC KEY-----
+`)
+	invalidBanner := []byte(`# Invalid banner
+-----BEGIN NOT A NEBULA PUBLIC KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+-----END NOT A NEBULA PUBLIC KEY-----
+`)
+	invalidPem := []byte(`# Not a valid PEM format
+-BEGIN NEBULA ED25519 PUBLIC KEY-----
+AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+-END NEBULA ED25519 PUBLIC KEY-----`)
+
+	keyBundle := appendByteSlices(pubKey, shortKey, invalidBanner, invalidPem)
+
+	// Success test case
+	k, rest, curve, err := UnmarshalPublicKeyFromPEM(keyBundle)
+	assert.Len(t, k, 32)
+	assert.Equal(t, Curve_CURVE25519, curve)
+	require.NoError(t, err)
+	assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem))
+
+	// Fail due to short key
+	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
+	assert.Nil(t, k)
+	assert.Equal(t, Curve_CURVE25519, curve)
+	assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem))
+	require.EqualError(t, err, "key was not 32 bytes, is invalid CURVE25519 public key")
+
+	// Fail due to invalid banner
+	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
+	assert.Nil(t, k)
+	assert.Equal(t, Curve_CURVE25519, curve)
+	require.EqualError(t, err, "bytes did not contain a proper public key banner")
+	assert.Equal(t, rest, invalidPem)
+
+	// Fail due to invalid PEM format, because
+	// it's missing the requisite pre-encapsulation boundary.
+	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
+	assert.Nil(t, k)
+	assert.Equal(t, Curve_CURVE25519, curve)
+	assert.Equal(t, rest, invalidPem)
+	require.EqualError(t, err, "input did not contain a valid PEM encoded block")
+}
+
+func TestUnmarshalX25519PublicKey(t *testing.T) {
+	t.Parallel()
+	pubKey := []byte(`# A good key
 -----BEGIN NEBULA X25519 PUBLIC KEY-----
 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 -----END NEBULA X25519 PUBLIC KEY-----
@@ -265,7 +319,7 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
 AAAAAAAAAAAAAAAAAAAAAAA=
 -----END NEBULA P256 PUBLIC KEY-----
 `)
-	signingKey := []byte(`# A signing key has the wrong scope for this function
+	oldPubP256Key := []byte(`# A good key
 -----BEGIN NEBULA ECDSA P256 PUBLIC KEY-----
 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
 AAAAAAAAAAAAAAAAAAAAAAA=
@@ -286,118 +340,44 @@ AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
 -END NEBULA X25519 PUBLIC KEY-----`)
 
-	keyBundle := appendByteSlices(pubKey, pubP256Key, signingKey, shortKey, invalidBanner, invalidPem)
+	keyBundle := appendByteSlices(pubKey, pubP256Key, oldPubP256Key, shortKey, invalidBanner, invalidPem)
 
-	// X25519 key
+	// Success test case
 	k, rest, curve, err := UnmarshalPublicKeyFromPEM(keyBundle)
 	assert.Len(t, k, 32)
 	require.NoError(t, err)
-	assert.Equal(t, rest, appendByteSlices(pubP256Key, signingKey, shortKey, invalidBanner, invalidPem))
+	assert.Equal(t, rest, appendByteSlices(pubP256Key, oldPubP256Key, shortKey, invalidBanner, invalidPem))
 	assert.Equal(t, Curve_CURVE25519, curve)
 
-	// P256 key
+	// Success test case
 	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
 	assert.Len(t, k, 65)
 	require.NoError(t, err)
-	assert.Equal(t, rest, appendByteSlices(signingKey, shortKey, invalidBanner, invalidPem))
+	assert.Equal(t, rest, appendByteSlices(oldPubP256Key, shortKey, invalidBanner, invalidPem))
 	assert.Equal(t, Curve_P256, curve)
 
-	// Reject a signing public key (Ed25519/ECDSA banner)
+	// Success test case
-	k, rest, _, err = UnmarshalPublicKeyFromPEM(rest)
+	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
-	assert.Nil(t, k)
-	assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem))
-	require.EqualError(t, err, "bytes did not contain a proper public key banner")
-
-	// Fail due to short key
-	k, rest, _, err = UnmarshalPublicKeyFromPEM(rest)
-	assert.Nil(t, k)
-	assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem))
-	require.EqualError(t, err, "key was not 32 bytes, is invalid CURVE25519 public key")
-
-	// Fail due to invalid banner
-	k, rest, _, err = UnmarshalPublicKeyFromPEM(rest)
-	assert.Nil(t, k)
-	require.EqualError(t, err, "bytes did not contain a proper public key banner")
-	assert.Equal(t, rest, invalidPem)
-
-	// Fail due to invalid PEM format, because
-	// it's missing the requisite pre-encapsulation boundary.
-	k, rest, _, err = UnmarshalPublicKeyFromPEM(rest)
-	assert.Nil(t, k)
-	assert.Equal(t, rest, invalidPem)
-	require.EqualError(t, err, "input did not contain a valid PEM encoded block")
-}
-
-func TestUnmarshalSigningPublicKeyFromPEM(t *testing.T) {
-	t.Parallel()
-	pubKey := []byte(`# A good key
------BEGIN NEBULA ED25519 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA ED25519 PUBLIC KEY-----
-`)
-	pubP256Key := []byte(`# A good key
------BEGIN NEBULA ECDSA P256 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-AAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA ECDSA P256 PUBLIC KEY-----
-`)
-	ecdhKey := []byte(`# A key-agreement key has the wrong scope for this function
------BEGIN NEBULA X25519 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
------END NEBULA X25519 PUBLIC KEY-----
-`)
-	shortKey := []byte(`# A short key
------BEGIN NEBULA ED25519 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==
------END NEBULA ED25519 PUBLIC KEY-----
-`)
-	invalidBanner := []byte(`# Invalid banner
------BEGIN NOT A NEBULA PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
------END NOT A NEBULA PUBLIC KEY-----
-`)
-	invalidPem := []byte(`# Not a valid PEM format
--BEGIN NEBULA ED25519 PUBLIC KEY-----
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
--END NEBULA ED25519 PUBLIC KEY-----`)
-
-	keyBundle := appendByteSlices(pubKey, pubP256Key, ecdhKey, shortKey, invalidBanner, invalidPem)
-
-	// Ed25519 key
-	k, rest, curve, err := UnmarshalSigningPublicKeyFromPEM(keyBundle)
-	assert.Len(t, k, 32)
-	require.NoError(t, err)
-	assert.Equal(t, rest, appendByteSlices(pubP256Key, ecdhKey, shortKey, invalidBanner, invalidPem))
-	assert.Equal(t, Curve_CURVE25519, curve)
-
-	// ECDSA P256 key
-	k, rest, curve, err = UnmarshalSigningPublicKeyFromPEM(rest)
 	assert.Len(t, k, 65)
 	require.NoError(t, err)
-	assert.Equal(t, rest, appendByteSlices(ecdhKey, shortKey, invalidBanner, invalidPem))
+	assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem))
 	assert.Equal(t, Curve_P256, curve)
 
-	// Reject a key-agreement public key (X25519/P256 banner)
-	k, rest, _, err = UnmarshalSigningPublicKeyFromPEM(rest)
-	assert.Nil(t, k)
-	assert.Equal(t, rest, appendByteSlices(shortKey, invalidBanner, invalidPem))
-	require.EqualError(t, err, "bytes did not contain a proper Ed25519/ECDSA public key banner")
-
 	// Fail due to short key
-	k, rest, _, err = UnmarshalSigningPublicKeyFromPEM(rest)
+	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
 	assert.Nil(t, k)
 	assert.Equal(t, rest, appendByteSlices(invalidBanner, invalidPem))
 	require.EqualError(t, err, "key was not 32 bytes, is invalid CURVE25519 public key")
 
 	// Fail due to invalid banner
-	k, rest, _, err = UnmarshalSigningPublicKeyFromPEM(rest)
+	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
 	assert.Nil(t, k)
-	require.EqualError(t, err, "bytes did not contain a proper Ed25519/ECDSA public key banner")
+	require.EqualError(t, err, "bytes did not contain a proper public key banner")
 	assert.Equal(t, rest, invalidPem)
 
 	// Fail due to invalid PEM format, because
 	// it's missing the requisite pre-encapsulation boundary.
-	k, rest, _, err = UnmarshalSigningPublicKeyFromPEM(rest)
+	k, rest, curve, err = UnmarshalPublicKeyFromPEM(rest)
 	assert.Nil(t, k)
 	assert.Equal(t, rest, invalidPem)
 	require.EqualError(t, err, "input did not contain a valid PEM encoded block")

cert_test/cert.go

@@ -14,12 +14,6 @@ import (
 	"golang.org/x/crypto/ed25519"
 )
 
-// testCertNow is the reference "now" used to derive default before/after times
-// in NewTestCaCert and NewTestCert. Holding it fixed for the lifetime of the
-// test binary keeps CA and leaf defaults aligned at the same second, so a leaf
-// signed with default times can never expire after its CA on a rounding race.
-var testCertNow = time.Now().Round(time.Second)
-
 // NewTestCaCert will create a new ca certificate
 func NewTestCaCert(version cert.Version, curve cert.Curve, before, after time.Time, networks, unsafeNetworks []netip.Prefix, groups []string) (cert.Certificate, []byte, []byte, []byte) {
 	var err error
@@ -41,10 +35,10 @@ func NewTestCaCert(version cert.Version, curve cert.Curve, before, after time.Ti
 	}
 
 	if before.IsZero() {
-		before = testCertNow.Add(time.Second * -60)
+		before = time.Now().Add(time.Second * -60).Round(time.Second)
 	}
 	if after.IsZero() {
-		after = testCertNow.Add(time.Second * 60)
+		after = time.Now().Add(time.Second * 60).Round(time.Second)
 	}
 
 	t := &cert.TBSCertificate{
@@ -77,11 +71,11 @@ func NewTestCaCert(version cert.Version, curve cert.Curve, before, after time.Ti
 // Expiry times are defaulted if you do not pass them in
 func NewTestCert(v cert.Version, curve cert.Curve, ca cert.Certificate, key []byte, name string, before, after time.Time, networks, unsafeNetworks []netip.Prefix, groups []string) (cert.Certificate, []byte, []byte, []byte) {
 	if before.IsZero() {
-		before = testCertNow.Add(time.Second * -60)
+		before = time.Now().Add(time.Second * -60).Round(time.Second)
 	}
 
 	if after.IsZero() {
-		after = testCertNow.Add(time.Second * 60)
+		after = time.Now().Add(time.Second * 60).Round(time.Second)
 	}
 
 	var pub, priv []byte

cmd/nebula-cert/ca.go

@@ -97,19 +97,6 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error
 		if err = mustFlagString("out-key", cf.outKeyPath); err != nil {
 			return err
 		}
-	} else {
-		// out-key is meaningless under PKCS#11 because the private key never
-		// leaves the HSM; reject it so we never silently accept or claim a
-		// stdout slot for it.
-		outKeySet := false
-		cf.set.Visit(func(f *flag.Flag) {
-			if f.Name == "out-key" {
-				outKeySet = true
-			}
-		})
-		if outKeySet {
-			return newHelpErrorf("cannot set -out-key with -pkcs11")
-		}
 	}
 	if err := mustFlagString("out-crt", cf.outCertPath); err != nil {
 		return err
@@ -184,21 +171,12 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error
 		}
 	}
 
-	var claims ioClaims
-	if err := reserveOutputs(&claims,
-		"out-key", *cf.outKeyPath,
-		"out-crt", *cf.outCertPath,
-		"out-qr", *cf.outQRPath,
-	); err != nil {
-		return err
-	}
-
 	var passphrase []byte
 	if !isP11 && *cf.encryption {
 		passphrase = []byte(os.Getenv("NEBULA_CA_PASSPHRASE"))
 		if len(passphrase) == 0 {
 			for i := 0; i < 5; i++ {
-				errOut.Write([]byte("Enter passphrase: "))
+				out.Write([]byte("Enter passphrase: "))
 				passphrase, err = pr.ReadPassword()
 
 				if err == ErrNoTerminal {
@@ -283,16 +261,14 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error
 		Curve:          curve,
 	}
 
-	if !isP11 && !isStdio(*cf.outKeyPath) {
+	if !isP11 {
 		if _, err := os.Stat(*cf.outKeyPath); err == nil {
 			return fmt.Errorf("refusing to overwrite existing CA key: %s", *cf.outKeyPath)
 		}
 	}
 
-	if !isStdio(*cf.outCertPath) {
+	if _, err := os.Stat(*cf.outCertPath); err == nil {
-		if _, err := os.Stat(*cf.outCertPath); err == nil {
+		return fmt.Errorf("refusing to overwrite existing CA cert: %s", *cf.outCertPath)
-			return fmt.Errorf("refusing to overwrite existing CA cert: %s", *cf.outCertPath)
-		}
 	}
 
 	var c cert.Certificate
@@ -318,7 +294,7 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error
 			b = cert.MarshalSigningPrivateKeyToPEM(curve, rawPriv)
 		}
 
-		err = writeOutput(*cf.outKeyPath, b, 0600, out)
+		err = os.WriteFile(*cf.outKeyPath, b, 0600)
 		if err != nil {
 			return fmt.Errorf("error while writing out-key: %s", err)
 		}
@@ -329,7 +305,7 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error
 		return fmt.Errorf("error while marshalling certificate: %s", err)
 	}
 
-	err = writeOutput(*cf.outCertPath, b, 0600, out)
+	err = os.WriteFile(*cf.outCertPath, b, 0600)
 	if err != nil {
 		return fmt.Errorf("error while writing out-crt: %s", err)
 	}
@@ -340,7 +316,7 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error
 			return fmt.Errorf("error while generating qr code: %s", err)
 		}
 
-		err = writeOutput(*cf.outQRPath, b, 0600, out)
+		err = os.WriteFile(*cf.outQRPath, b, 0600)
 		if err != nil {
 			return fmt.Errorf("error while writing out-qr: %s", err)
 		}
@@ -356,7 +332,6 @@ func caSummary() string {
 func caHelp(out io.Writer) {
 	cf := newCaFlags()
 	out.Write([]byte("Usage of " + os.Args[0] + " " + caSummary() + "\n"))
-	out.Write([]byte(stdioHelpText))
 	cf.set.SetOutput(out)
 	cf.set.PrintDefaults()
 }

cmd/nebula-cert/ca_test.go

@@ -27,7 +27,6 @@ func Test_caHelp(t *testing.T) {
 	assert.Equal(
 		t,
 		"Usage of "+os.Args[0]+" ca <flags>: create a self signed certificate authority\n"+
-			"  Pass \"-\" to any path flag to read from stdin or write to stdout.\n"+
 			"  -argon-iterations uint\n"+
 			"    \tOptional: Argon2 iterations parameter used for encrypted private key passphrase (default 1)\n"+
 			"  -argon-memory uint\n"+
@@ -85,7 +84,7 @@ func Test_ca(t *testing.T) {
 		err:      nil,
 	}
 
-	pwPromptEB := "Enter passphrase: "
+	pwPromptOb := "Enter passphrase: "
 
 	// required args
 	assertHelpError(t, ca(
@@ -169,8 +168,8 @@ func Test_ca(t *testing.T) {
 	eb.Reset()
 	args = []string{"-version", "1", "-encrypt", "-name", "test", "-duration", "100m", "-groups", "1,2,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()}
 	require.NoError(t, ca(args, ob, eb, testpw))
-	assert.Empty(t, ob.String())
+	assert.Equal(t, pwPromptOb, ob.String())
-	assert.Equal(t, pwPromptEB, eb.String())
+	assert.Empty(t, eb.String())
 
 	// test encrypted key with passphrase environment variable
 	os.Remove(keyF.Name())
@@ -208,8 +207,8 @@ func Test_ca(t *testing.T) {
 	eb.Reset()
 	args = []string{"-version", "1", "-encrypt", "-name", "test", "-duration", "100m", "-groups", "1,2,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()}
 	require.Error(t, ca(args, ob, eb, errpw))
-	assert.Empty(t, ob.String())
+	assert.Equal(t, pwPromptOb, ob.String())
-	assert.Equal(t, pwPromptEB, eb.String())
+	assert.Empty(t, eb.String())
 
 	// test when user fails to enter a password
 	os.Remove(keyF.Name())
@@ -218,8 +217,8 @@ func Test_ca(t *testing.T) {
 	eb.Reset()
 	args = []string{"-version", "1", "-encrypt", "-name", "test", "-duration", "100m", "-groups", "1,2,3,4,5", "-out-crt", crtF.Name(), "-out-key", keyF.Name()}
 	require.EqualError(t, ca(args, ob, eb, nopw), "no passphrase specified, remove -encrypt flag to write out-key in plaintext")
-	assert.Empty(t, ob.String())
+	assert.Equal(t, strings.Repeat(pwPromptOb, 5), ob.String()) // prompts 5 times before giving up
-	assert.Equal(t, strings.Repeat(pwPromptEB, 5), eb.String()) // prompts 5 times before giving up
+	assert.Empty(t, eb.String())
 
 	// create valid cert/key for overwrite tests
 	os.Remove(keyF.Name())
@@ -248,67 +247,3 @@ func Test_ca(t *testing.T) {
 	os.Remove(keyF.Name())
 
 }
-
-func Test_ca_stdio(t *testing.T) {
-	nopw := &StubPasswordReader{}
-
-	keyF, err := os.CreateTemp("", "ca.key")
-	require.NoError(t, err)
-	os.Remove(keyF.Name())
-	defer os.Remove(keyF.Name())
-
-	crtF, err := os.CreateTemp("", "ca.crt")
-	require.NoError(t, err)
-	os.Remove(crtF.Name())
-	defer os.Remove(crtF.Name())
-
-	// out-crt on stdout, out-key on disk
-	ob := &bytes.Buffer{}
-	eb := &bytes.Buffer{}
-	require.NoError(t, ca([]string{"-name", "test-ca", "-duration", "1h", "-out-crt", "-", "-out-key", keyF.Name()}, ob, eb, nopw))
-	assert.Empty(t, eb.String())
-	c, _, err := cert.UnmarshalCertificateFromPEM(ob.Bytes())
-	require.NoError(t, err)
-	assert.True(t, c.IsCA())
-	assert.Equal(t, "test-ca", c.Name())
-
-	// out-key on stdout, out-crt on disk
-	os.Remove(keyF.Name())
-	ob.Reset()
-	eb.Reset()
-	require.NoError(t, ca([]string{"-name", "test-ca", "-duration", "1h", "-out-crt", crtF.Name(), "-out-key", "-"}, ob, eb, nopw))
-	assert.Empty(t, eb.String())
-	_, _, curve, err := cert.UnmarshalSigningPrivateKeyFromPEM(ob.Bytes())
-	require.NoError(t, err)
-	assert.Equal(t, cert.Curve_CURVE25519, curve)
-
-	// dual stdout is rejected up front
-	os.Remove(crtF.Name())
-	ob.Reset()
-	eb.Reset()
-	require.EqualError(t,
-		ca([]string{"-name", "test-ca", "-duration", "1h", "-out-crt", "-", "-out-key", "-"}, ob, eb, nopw),
-		`-out-key and -out-crt both set to "-", only one output may write to stdout`)
-	assert.Empty(t, ob.String())
-
-	// an output conflict combined with -encrypt must error BEFORE prompting
-	// for a passphrase; pr would record any read attempt
-	tracker := &trackingPasswordReader{}
-	ob.Reset()
-	eb.Reset()
-	require.EqualError(t,
-		ca([]string{"-name", "test-ca", "-duration", "1h", "-encrypt", "-out-crt", "-", "-out-key", "-"}, ob, eb, tracker),
-		`-out-key and -out-crt both set to "-", only one output may write to stdout`)
-	assert.Empty(t, ob.String())
-	assert.Empty(t, eb.String())
-	assert.Zero(t, tracker.calls, "passphrase prompt should not have been called")
-}
-
-type trackingPasswordReader struct {
-	calls int
-}
-
-func (pr *trackingPasswordReader) ReadPassword() ([]byte, error) {
-	pr.calls++
-	return []byte(""), nil
-}

cmd/nebula-cert/keygen.go

@@ -42,8 +42,6 @@ func keygen(args []string, out io.Writer, errOut io.Writer) error {
 		if err = mustFlagString("out-key", cf.outKeyPath); err != nil {
 			return err
 		}
-	} else if *cf.outKeyPath != "" {
-		return newHelpErrorf("cannot set -out-key with -pkcs11")
 	}
 	if err = mustFlagString("out-pub", cf.outPubPath); err != nil {
 		return err
@@ -71,14 +69,6 @@ func keygen(args []string, out io.Writer, errOut io.Writer) error {
 		}
 	}
 
-	var claims ioClaims
-	if err := reserveOutputs(&claims,
-		"out-key", *cf.outKeyPath,
-		"out-pub", *cf.outPubPath,
-	); err != nil {
-		return err
-	}
-
 	if isP11 {
 		p11Client, err := pkclient.FromUrl(*cf.p11url)
 		if err != nil {
@@ -92,12 +82,12 @@ func keygen(args []string, out io.Writer, errOut io.Writer) error {
 			return fmt.Errorf("error while getting public key: %w", err)
 		}
 	} else {
-		err = writeOutput(*cf.outKeyPath, cert.MarshalPrivateKeyToPEM(curve, rawPriv), 0600, out)
+		err = os.WriteFile(*cf.outKeyPath, cert.MarshalPrivateKeyToPEM(curve, rawPriv), 0600)
 		if err != nil {
 			return fmt.Errorf("error while writing out-key: %s", err)
 		}
 	}
-	err = writeOutput(*cf.outPubPath, cert.MarshalPublicKeyToPEM(curve, pub), 0600, out)
+	err = os.WriteFile(*cf.outPubPath, cert.MarshalPublicKeyToPEM(curve, pub), 0600)
 	if err != nil {
 		return fmt.Errorf("error while writing out-pub: %s", err)
 	}
@@ -112,7 +102,6 @@ func keygenSummary() string {
 func keygenHelp(out io.Writer) {
 	cf := newKeygenFlags()
 	_, _ = out.Write([]byte("Usage of " + os.Args[0] + " " + keygenSummary() + "\n"))
-	_, _ = out.Write([]byte(stdioHelpText))
 	cf.set.SetOutput(out)
 	cf.set.PrintDefaults()
 }

cmd/nebula-cert/keygen_test.go

@@ -20,7 +20,6 @@ func Test_keygenHelp(t *testing.T) {
 	assert.Equal(
 		t,
 		"Usage of "+os.Args[0]+" keygen <flags>: create a public/private key pair. the public key can be passed to `nebula-cert sign`\n"+
-			"  Pass \"-\" to any path flag to read from stdin or write to stdout.\n"+
 			"  -curve string\n"+
 			"    \tECDH Curve (25519, P256) (default \"25519\")\n"+
 			"  -out-key string\n"+
@@ -94,43 +93,3 @@ func Test_keygen(t *testing.T) {
 	require.NoError(t, err)
 	assert.Len(t, lPub, 32)
 }
-
-func Test_keygen_stdio(t *testing.T) {
-	keyF, err := os.CreateTemp("", "test.key")
-	require.NoError(t, err)
-	os.Remove(keyF.Name())
-	defer os.Remove(keyF.Name())
-
-	pubF, err := os.CreateTemp("", "test.pub")
-	require.NoError(t, err)
-	os.Remove(pubF.Name())
-	defer os.Remove(pubF.Name())
-
-	// out-pub on stdout, out-key on disk
-	ob := &bytes.Buffer{}
-	eb := &bytes.Buffer{}
-	require.NoError(t, keygen([]string{"-out-pub", "-", "-out-key", keyF.Name()}, ob, eb))
-	assert.Empty(t, eb.String())
-	lPub, _, curve, err := cert.UnmarshalPublicKeyFromPEM(ob.Bytes())
-	require.NoError(t, err)
-	assert.Equal(t, cert.Curve_CURVE25519, curve)
-	assert.Len(t, lPub, 32)
-
-	// out-key on stdout, out-pub on disk
-	os.Remove(keyF.Name())
-	ob.Reset()
-	eb.Reset()
-	require.NoError(t, keygen([]string{"-out-pub", pubF.Name(), "-out-key", "-"}, ob, eb))
-	assert.Empty(t, eb.String())
-	lKey, _, curve, err := cert.UnmarshalPrivateKeyFromPEM(ob.Bytes())
-	require.NoError(t, err)
-	assert.Equal(t, cert.Curve_CURVE25519, curve)
-	assert.Len(t, lKey, 32)
-
-	// both on stdout is a conflict caught up front
-	ob.Reset()
-	eb.Reset()
-	require.EqualError(t, keygen([]string{"-out-pub", "-", "-out-key", "-"}, ob, eb),
-		`-out-key and -out-pub both set to "-", only one output may write to stdout`)
-	assert.Empty(t, ob.String())
-}

cmd/nebula-cert/passwords.go

@@ -22,9 +22,7 @@ func (pr StdinPasswordReader) ReadPassword() ([]byte, error) {
 	}
 
 	password, err := term.ReadPassword(int(os.Stdin.Fd()))
-	// Terminal echo is off while reading, so the user's Enter key does not
+	fmt.Println()
-	// produce a visible newline. Emit one on stderr to match the prompt.
-	fmt.Fprintln(os.Stderr)
 
 	return password, err
 }

cmd/nebula-cert/print.go

@@ -40,23 +40,11 @@ func printCert(args []string, out io.Writer, errOut io.Writer) error {
 		return err
 	}
 
-	var claims ioClaims
+	rawCert, err := os.ReadFile(*pf.path)
-	if err := reserveInputs(&claims, "path", *pf.path); err != nil {
-		return err
-	}
-	if err := reserveOutputs(&claims, "out-qr", *pf.outQRPath); err != nil {
-		return err
-	}
-
-	rawCert, err := readInput("path", *pf.path, &claims)
 	if err != nil {
 		return fmt.Errorf("unable to read cert; %s", err)
 	}
 
-	// When the QR is going to stdout, suppress the human-readable text/json
-	// output so the binary stream is not contaminated.
-	qrToStdout := isStdio(*pf.outQRPath)
-
 	var c cert.Certificate
 	var qrBytes []byte
 	part := 0
@@ -69,13 +57,11 @@ func printCert(args []string, out io.Writer, errOut io.Writer) error {
 			return fmt.Errorf("error while unmarshaling cert: %s", err)
 		}
 
-		if !qrToStdout {
+		if *pf.json {
-			if *pf.json {
+			jsonCerts = append(jsonCerts, c)
-				jsonCerts = append(jsonCerts, c)
+		} else {
-			} else {
+			_, _ = out.Write([]byte(c.String()))
-				_, _ = out.Write([]byte(c.String()))
+			_, _ = out.Write([]byte("\n"))
-				_, _ = out.Write([]byte("\n"))
-			}
 		}
 
 		if *pf.outQRPath != "" {
@@ -93,7 +79,7 @@ func printCert(args []string, out io.Writer, errOut io.Writer) error {
 		part++
 	}
 
-	if *pf.json && !qrToStdout {
+	if *pf.json {
 		b, _ := json.Marshal(jsonCerts)
 		_, _ = out.Write(b)
 		_, _ = out.Write([]byte("\n"))
@@ -105,7 +91,7 @@ func printCert(args []string, out io.Writer, errOut io.Writer) error {
 			return fmt.Errorf("error while generating qr code: %s", err)
 		}
 
-		err = writeOutput(*pf.outQRPath, b, 0600, out)
+		err = os.WriteFile(*pf.outQRPath, b, 0600)
 		if err != nil {
 			return fmt.Errorf("error while writing out-qr: %s", err)
 		}
@@ -121,7 +107,6 @@ func printSummary() string {
 func printHelp(out io.Writer) {
 	pf := newPrintFlags()
 	out.Write([]byte("Usage of " + os.Args[0] + " " + printSummary() + "\n"))
-	out.Write([]byte(stdioHelpText))
 	pf.set.SetOutput(out)
 	pf.set.PrintDefaults()
 }

cmd/nebula-cert/print_test.go

@@ -25,7 +25,6 @@ func Test_printHelp(t *testing.T) {
 	assert.Equal(
 		t,
 		"Usage of "+os.Args[0]+" print <flags>: prints details about a certificate\n"+
-			"  Pass \"-\" to any path flag to read from stdin or write to stdout.\n"+
 			"  -json\n"+
 			"    \tOptional: outputs certificates in json format\n"+
 			"  -out-qr string\n"+
@@ -179,44 +178,6 @@ func Test_printCert(t *testing.T) {
 		ob.String(),
 	)
 	assert.Empty(t, eb.String())
-
-	// read cert from stdin
-	ob.Reset()
-	eb.Reset()
-	withStdin(t, bytes.NewReader(p))
-	err = printCert([]string{"-json", "-path", "-"}, ob, eb)
-	require.NoError(t, err)
-	assert.Equal(
-		t,
-		`[{"details":{"curve":"CURVE25519","groups":["hi"],"isCa":false,"issuer":"`+c.Issuer()+`","name":"test","networks":["10.0.0.123/8"],"notAfter":"0001-01-01T00:00:00Z","notBefore":"0001-01-01T00:00:00Z","publicKey":"`+pk+`","unsafeNetworks":[]},"fingerprint":"`+fp+`","signature":"`+sig+`","version":1}]
-`,
-		ob.String(),
-	)
-	assert.Empty(t, eb.String())
-
-	// -out-qr - sends only the PNG to stdout, suppressing the cert dump
-	ob.Reset()
-	eb.Reset()
-	withStdin(t, bytes.NewReader(p))
-	err = printCert([]string{"-path", "-", "-out-qr", "-"}, ob, eb)
-	require.NoError(t, err)
-	assert.Empty(t, eb.String())
-	stdout := ob.Bytes()
-	require.NotEmpty(t, stdout)
-	// PNG magic, no PEM/JSON noise prepended
-	assert.Equal(t, []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, stdout[:8])
-	assert.NotContains(t, string(stdout), "NebulaCertificate")
-	assert.NotContains(t, string(stdout), `"details"`)
-
-	// json + out-qr - still suppresses json
-	ob.Reset()
-	eb.Reset()
-	withStdin(t, bytes.NewReader(p))
-	err = printCert([]string{"-json", "-path", "-", "-out-qr", "-"}, ob, eb)
-	require.NoError(t, err)
-	assert.Empty(t, eb.String())
-	assert.Equal(t, []byte{0x89, 'P', 'N', 'G'}, ob.Bytes()[:4])
-	assert.NotContains(t, ob.String(), `"details"`)
 }
 
 // NewTestCaCert will generate a CA cert

cmd/nebula-cert/sign.go

@@ -85,9 +85,6 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 	if !isP11 && *sf.inPubPath != "" && *sf.outKeyPath != "" {
 		return newHelpErrorf("cannot set both -in-pub and -out-key")
 	}
-	if isP11 && *sf.outKeyPath != "" {
-		return newHelpErrorf("cannot set -out-key with -pkcs11")
-	}
 
 	var v4Networks []netip.Prefix
 	var v6Networks []netip.Prefix
@@ -105,35 +102,13 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 		return newHelpErrorf("-version must be either %v or %v", cert.Version1, cert.Version2)
 	}
 
-	if *sf.outKeyPath == "" {
-		*sf.outKeyPath = *sf.name + ".key"
-	}
-	if *sf.outCertPath == "" {
-		*sf.outCertPath = *sf.name + ".crt"
-	}
-
-	var claims ioClaims
-	if err := reserveInputs(&claims,
-		"ca-key", *sf.caKeyPath,
-		"ca-crt", *sf.caCertPath,
-		"in-pub", *sf.inPubPath,
-	); err != nil {
-		return err
-	}
-	if err := reserveOutputs(&claims,
-		"out-key", *sf.outKeyPath,
-		"out-crt", *sf.outCertPath,
-		"out-qr", *sf.outQRPath,
-	); err != nil {
-		return err
-	}
-
 	var curve cert.Curve
 	var caKey []byte
 
 	if !isP11 {
 		var rawCAKey []byte
-		rawCAKey, err = readInput("ca-key", *sf.caKeyPath, &claims)
+		rawCAKey, err := os.ReadFile(*sf.caKeyPath)
+
 		if err != nil {
 			return fmt.Errorf("error while reading ca-key: %s", err)
 		}
@@ -146,7 +121,7 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 			if len(passphrase) == 0 {
 				// ask for a passphrase until we get one
 				for i := 0; i < 5; i++ {
-					errOut.Write([]byte("Enter passphrase: "))
+					out.Write([]byte("Enter passphrase: "))
 					passphrase, err = pr.ReadPassword()
 
 					if errors.Is(err, ErrNoTerminal) {
@@ -172,7 +147,7 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 		}
 	}
 
-	rawCACert, err := readInput("ca-crt", *sf.caCertPath, &claims)
+	rawCACert, err := os.ReadFile(*sf.caCertPath)
 	if err != nil {
 		return fmt.Errorf("error while reading ca-crt: %s", err)
 	}
@@ -270,7 +245,7 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 
 	if *sf.inPubPath != "" {
 		var pubCurve cert.Curve
-		rawPub, err := readInput("in-pub", *sf.inPubPath, &claims)
+		rawPub, err := os.ReadFile(*sf.inPubPath)
 		if err != nil {
 			return fmt.Errorf("error while reading in-pub: %s", err)
 		}
@@ -291,10 +266,16 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 		pub, rawPriv = newKeypair(curve)
 	}
 
-	if !isStdio(*sf.outCertPath) {
+	if *sf.outKeyPath == "" {
-		if _, err := os.Stat(*sf.outCertPath); err == nil {
+		*sf.outKeyPath = *sf.name + ".key"
-			return fmt.Errorf("refusing to overwrite existing cert: %s", *sf.outCertPath)
+	}
-		}
+
+	if *sf.outCertPath == "" {
+		*sf.outCertPath = *sf.name + ".crt"
+	}
+
+	if _, err := os.Stat(*sf.outCertPath); err == nil {
+		return fmt.Errorf("refusing to overwrite existing cert: %s", *sf.outCertPath)
 	}
 
 	var crts []cert.Certificate
@@ -379,13 +360,11 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 	}
 
 	if !isP11 && *sf.inPubPath == "" {
-		if !isStdio(*sf.outKeyPath) {
+		if _, err := os.Stat(*sf.outKeyPath); err == nil {
-			if _, err := os.Stat(*sf.outKeyPath); err == nil {
+			return fmt.Errorf("refusing to overwrite existing key: %s", *sf.outKeyPath)
-				return fmt.Errorf("refusing to overwrite existing key: %s", *sf.outKeyPath)
-			}
 		}
 
-		err = writeOutput(*sf.outKeyPath, cert.MarshalPrivateKeyToPEM(curve, rawPriv), 0600, out)
+		err = os.WriteFile(*sf.outKeyPath, cert.MarshalPrivateKeyToPEM(curve, rawPriv), 0600)
 		if err != nil {
 			return fmt.Errorf("error while writing out-key: %s", err)
 		}
@@ -400,7 +379,7 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 		b = append(b, sb...)
 	}
 
-	err = writeOutput(*sf.outCertPath, b, 0600, out)
+	err = os.WriteFile(*sf.outCertPath, b, 0600)
 	if err != nil {
 		return fmt.Errorf("error while writing out-crt: %s", err)
 	}
@@ -411,7 +390,7 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
 			return fmt.Errorf("error while generating qr code: %s", err)
 		}
 
-		err = writeOutput(*sf.outQRPath, b, 0600, out)
+		err = os.WriteFile(*sf.outQRPath, b, 0600)
 		if err != nil {
 			return fmt.Errorf("error while writing out-qr: %s", err)
 		}
@@ -461,7 +440,6 @@ func signSummary() string {
 func signHelp(out io.Writer) {
 	sf := newSignFlags()
 	out.Write([]byte("Usage of " + os.Args[0] + " " + signSummary() + "\n"))
-	out.Write([]byte(stdioHelpText))
 	sf.set.SetOutput(out)
 	sf.set.PrintDefaults()
 }

cmd/nebula-cert/sign_test.go

@@ -27,7 +27,6 @@ func Test_signHelp(t *testing.T) {
 	assert.Equal(
 		t,
 		"Usage of "+os.Args[0]+" sign <flags>: create and sign a certificate\n"+
-			"  Pass \"-\" to any path flag to read from stdin or write to stdout.\n"+
 			"  -ca-crt string\n"+
 			"    \tOptional: path to the signing CA cert (default \"ca.crt\")\n"+
 			"  -ca-key string\n"+
@@ -377,18 +376,15 @@ func Test_signCert(t *testing.T) {
 	// test with the proper password
 	args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, ,   10.2.2.2/32   ,   ,  ,, 10.5.5.5/32", "-groups", "1,,   2    ,        ,,,3,4,5"}
 	require.NoError(t, signCert(args, ob, eb, testpw))
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "Enter passphrase: ", ob.String())
-	assert.Equal(t, "Enter passphrase: ", eb.String())
+	assert.Empty(t, eb.String())
 
 	// test with the proper password in the environment
 	os.Remove(crtF.Name())
 	os.Remove(keyF.Name())
 	args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, ,   10.2.2.2/32   ,   ,  ,, 10.5.5.5/32", "-groups", "1,,   2    ,        ,,,3,4,5"}
 	os.Setenv("NEBULA_CA_PASSPHRASE", string(passphrase))
-	ob.Reset()
-	eb.Reset()
 	require.NoError(t, signCert(args, ob, eb, testpw))
-	assert.Empty(t, ob.String())
 	assert.Empty(t, eb.String())
 	os.Setenv("NEBULA_CA_PASSPHRASE", "")
 
@@ -399,8 +395,8 @@ func Test_signCert(t *testing.T) {
 	testpw.password = []byte("invalid password")
 	args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, ,   10.2.2.2/32   ,   ,  ,, 10.5.5.5/32", "-groups", "1,,   2    ,        ,,,3,4,5"}
 	require.Error(t, signCert(args, ob, eb, testpw))
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "Enter passphrase: ", ob.String())
-	assert.Equal(t, "Enter passphrase: ", eb.String())
+	assert.Empty(t, eb.String())
 
 	// test with the wrong password in environment
 	ob.Reset()
@@ -420,8 +416,8 @@ func Test_signCert(t *testing.T) {
 	args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, ,   10.2.2.2/32   ,   ,  ,, 10.5.5.5/32", "-groups", "1,,   2    ,        ,,,3,4,5"}
 	require.Error(t, signCert(args, ob, eb, nopw))
 	// normally the user hitting enter on the prompt would add newlines between these
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "Enter passphrase: Enter passphrase: Enter passphrase: Enter passphrase: Enter passphrase: ", ob.String())
-	assert.Equal(t, "Enter passphrase: Enter passphrase: Enter passphrase: Enter passphrase: Enter passphrase: ", eb.String())
+	assert.Empty(t, eb.String())
 
 	// test an error condition
 	ob.Reset()
@@ -429,106 +425,6 @@ func Test_signCert(t *testing.T) {
 
 	args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", keyF.Name(), "-duration", "100m", "-subnets", "10.1.1.1/32, ,   10.2.2.2/32   ,   ,  ,, 10.5.5.5/32", "-groups", "1,,   2    ,        ,,,3,4,5"}
 	require.Error(t, signCert(args, ob, eb, errpw))
-	assert.Empty(t, ob.String())
+	assert.Equal(t, "Enter passphrase: ", ob.String())
-	assert.Equal(t, "Enter passphrase: ", eb.String())
+	assert.Empty(t, eb.String())
-}
-
-func Test_signCert_stdio(t *testing.T) {
-	nopw := &StubPasswordReader{
-		password: []byte(""),
-		err:      nil,
-	}
-
-	caPub, caPriv, _ := ed25519.GenerateKey(rand.Reader)
-	rawCAKey := cert.MarshalSigningPrivateKeyToPEM(cert.Curve_CURVE25519, caPriv)
-
-	ca, _ := NewTestCaCert("ca", caPub, caPriv, time.Now(), time.Now().Add(time.Minute*200), nil, nil, nil)
-	rawCACrt, _ := ca.MarshalPEM()
-
-	caCrtF, err := os.CreateTemp("", "sign-cert.crt")
-	require.NoError(t, err)
-	defer os.Remove(caCrtF.Name())
-	caCrtF.Write(rawCACrt)
-
-	caKeyF, err := os.CreateTemp("", "sign-cert.key")
-	require.NoError(t, err)
-	defer os.Remove(caKeyF.Name())
-	caKeyF.Write(rawCAKey)
-
-	keyF, err := os.CreateTemp("", "sign.key")
-	require.NoError(t, err)
-	os.Remove(keyF.Name())
-	defer os.Remove(keyF.Name())
-
-	// ca-key on stdin, cert to stdout
-	withStdin(t, bytes.NewReader(rawCAKey))
-	ob := &bytes.Buffer{}
-	eb := &bytes.Buffer{}
-	args := []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", "-", "-name", "stdin-test", "-ip", "1.1.1.1/24", "-out-crt", "-", "-out-key", keyF.Name(), "-duration", "100m"}
-	require.NoError(t, signCert(args, ob, eb, nopw))
-	assert.Empty(t, eb.String())
-
-	lCrt, _, err := cert.UnmarshalCertificateFromPEM(ob.Bytes())
-	require.NoError(t, err)
-	assert.Equal(t, "stdin-test", lCrt.Name())
-	assert.True(t, lCrt.CheckSignature(caPub))
-
-	// two flags reading from stdin should error before any read attempt;
-	// otherwise an interactive shell would hang on io.ReadAll
-	stdinIn := bytes.NewReader(rawCAKey)
-	withStdin(t, stdinIn)
-	ob.Reset()
-	eb.Reset()
-	args = []string{"-version", "1", "-ca-crt", "-", "-ca-key", "-", "-name", "stdin-test", "-ip", "1.1.1.1/24", "-out-crt", "nope", "-out-key", "nope", "-duration", "100m"}
-	require.EqualError(t, signCert(args, ob, eb, nopw),
-		`-ca-key and -ca-crt both set to "-", only one input may read from stdin`)
-	assert.Equal(t, len(rawCAKey), stdinIn.Len(), "stdin should be untouched when conflict is caught up front")
-
-	// two flags writing to stdout should error before any output is written
-	// AND before stdin is consumed
-	stdinR := bytes.NewReader(rawCAKey)
-	withStdin(t, stdinR)
-	ob.Reset()
-	eb.Reset()
-	args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", "-", "-name", "stdin-test", "-ip", "1.1.1.1/24", "-out-crt", "-", "-out-key", "-", "-duration", "100m"}
-	require.EqualError(t, signCert(args, ob, eb, nopw),
-		`-out-key and -out-crt both set to "-", only one output may write to stdout`)
-	assert.Empty(t, ob.String())
-	// stdin should be untouched because the conflict was caught up front
-	assert.Equal(t, len(rawCAKey), stdinR.Len())
-
-	// out-key on stdout, cert on disk
-	keyF2, err := os.CreateTemp("", "sign.key")
-	require.NoError(t, err)
-	os.Remove(keyF2.Name())
-	defer os.Remove(keyF2.Name())
-	crtF, err := os.CreateTemp("", "sign.crt")
-	require.NoError(t, err)
-	os.Remove(crtF.Name())
-	defer os.Remove(crtF.Name())
-
-	ob.Reset()
-	eb.Reset()
-	args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "stdin-test", "-ip", "1.1.1.1/24", "-out-crt", crtF.Name(), "-out-key", "-", "-duration", "100m"}
-	require.NoError(t, signCert(args, ob, eb, nopw))
-	assert.Empty(t, eb.String())
-	_, _, curve, err := cert.UnmarshalPrivateKeyFromPEM(ob.Bytes())
-	require.NoError(t, err)
-	assert.Equal(t, cert.Curve_CURVE25519, curve)
-
-	// in-pub on stdin (caller already has a keypair, only the cert is generated)
-	inPub, _ := x25519Keypair()
-	rawInPub := cert.MarshalPublicKeyToPEM(cert.Curve_CURVE25519, inPub)
-
-	withStdin(t, bytes.NewReader(rawInPub))
-	os.Remove(crtF.Name())
-	ob.Reset()
-	eb.Reset()
-	args = []string{"-version", "1", "-ca-crt", caCrtF.Name(), "-ca-key", caKeyF.Name(), "-name", "in-pub-test", "-ip", "1.1.1.1/24", "-in-pub", "-", "-out-crt", "-", "-duration", "100m"}
-	require.NoError(t, signCert(args, ob, eb, nopw))
-	assert.Empty(t, eb.String())
-	stdinCrt, _, err := cert.UnmarshalCertificateFromPEM(ob.Bytes())
-	require.NoError(t, err)
-	assert.Equal(t, "in-pub-test", stdinCrt.Name())
-	assert.Equal(t, inPub, stdinCrt.PublicKey())
 }

cmd/nebula-cert/stdio.go

@@ -1,117 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"io"
-	"os"
-)
-
-// stdioPath is the special path value that selects stdin (for inputs) or
-// stdout (for outputs) instead of a file on disk.
-const stdioPath = "-"
-
-// stdioHelpText is rendered just under the Usage line of each subcommand
-// help so the - convention is documented once instead of on every flag.
-const stdioHelpText = "  Pass \"-\" to any path flag to read from stdin or write to stdout.\n"
-
-// stdinReader is the source used when an input flag is set to "-".
-// It is a package level var so tests can swap in a deterministic reader.
-// Tests that mutate stdinReader cannot run with t.Parallel().
-var stdinReader io.Reader = os.Stdin
-
-// ioClaims tracks which flags have claimed stdin and stdout during a single
-// command invocation so we can refuse a second flag asking for the same
-// stream.
-type ioClaims struct {
-	in  string
-	out string
-}
-
-func (c *ioClaims) claimIn(flagName string) error {
-	if c.in != "" && c.in != flagName {
-		return fmt.Errorf("-%s and -%s both set to %q, only one input may read from stdin", c.in, flagName, stdioPath)
-	}
-	c.in = flagName
-	return nil
-}
-
-func (c *ioClaims) claimOut(flagName string) error {
-	if c.out != "" && c.out != flagName {
-		return fmt.Errorf("-%s and -%s both set to %q, only one output may write to stdout", c.out, flagName, stdioPath)
-	}
-	c.out = flagName
-	return nil
-}
-
-// reserveInputs walks alternating (flagName, path) pairs and claims stdin
-// for any path equal to stdioPath. It must be called before any input is
-// read so a conflict can be reported immediately instead of blocking on
-// io.ReadAll while waiting for input that will never arrive.
-func reserveInputs(claims *ioClaims, pairs ...string) error {
-	return reserveStdio(claims, "reserveInputs", (*ioClaims).claimIn, pairs)
-}
-
-// reserveOutputs walks alternating (flagName, path) pairs and claims stdout
-// for any path equal to stdioPath. It must be called before any output is
-// written so a conflict cannot leave one stream half written before the
-// second flag fails.
-func reserveOutputs(claims *ioClaims, pairs ...string) error {
-	return reserveStdio(claims, "reserveOutputs", (*ioClaims).claimOut, pairs)
-}
-
-func reserveStdio(claims *ioClaims, who string, claim func(*ioClaims, string) error, pairs []string) error {
-	if len(pairs)%2 != 0 {
-		panic(who + " requires alternating name, path pairs")
-	}
-	for i := 0; i < len(pairs); i += 2 {
-		name, path := pairs[i], pairs[i+1]
-		if path != stdioPath {
-			continue
-		}
-		if err := claim(claims, name); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// readInput returns the bytes referenced by path, reading from stdin when
-// path is stdioPath.
-func readInput(flagName, path string, claims *ioClaims) ([]byte, error) {
-	if path == stdioPath {
-		if err := claims.claimIn(flagName); err != nil {
-			return nil, err
-		}
-		return io.ReadAll(stdinReader)
-	}
-	return os.ReadFile(path)
-}
-
-// openInput returns a reader for path. When path is stdioPath the returned
-// reader wraps stdin and Close is a no-op.
-func openInput(flagName, path string, claims *ioClaims) (io.ReadCloser, error) {
-	if path == stdioPath {
-		if err := claims.claimIn(flagName); err != nil {
-			return nil, err
-		}
-		return io.NopCloser(stdinReader), nil
-	}
-	return os.Open(path)
-}
-
-// writeOutput writes data to path, or to stdout when path is stdioPath. perm
-// is only used for file output. The caller must have already claimed stdout
-// via reserveOutputs before invoking with stdioPath.
-func writeOutput(path string, data []byte, perm os.FileMode, stdout io.Writer) error {
-	if path == stdioPath {
-		_, err := stdout.Write(data)
-		return err
-	}
-	return os.WriteFile(path, data, perm)
-}
-
-// isStdio reports whether path is the stdio sentinel and so should skip
-// existence checks like "refuse to overwrite".
-func isStdio(path string) bool {
-	return path == stdioPath
-}

cmd/nebula-cert/stdio_test.go

@@ -1,167 +0,0 @@
-package main
-
-import (
-	"bytes"
-	"io"
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// withStdin temporarily replaces stdinReader for the duration of t.
-func withStdin(t *testing.T, r io.Reader) {
-	t.Helper()
-	prev := stdinReader
-	stdinReader = r
-	t.Cleanup(func() { stdinReader = prev })
-}
-
-func Test_readInput_stdin(t *testing.T) {
-	withStdin(t, bytes.NewBufferString("hello"))
-	var claims ioClaims
-
-	got, err := readInput("path", "-", &claims)
-	require.NoError(t, err)
-	assert.Equal(t, []byte("hello"), got)
-	assert.Equal(t, "path", claims.in)
-}
-
-func Test_readInput_file(t *testing.T) {
-	dir := t.TempDir()
-	p := filepath.Join(dir, "f")
-	require.NoError(t, os.WriteFile(p, []byte("file"), 0600))
-	var claims ioClaims
-
-	got, err := readInput("path", p, &claims)
-	require.NoError(t, err)
-	assert.Equal(t, []byte("file"), got)
-	assert.Empty(t, claims.in)
-}
-
-func Test_readInput_doubleStdinErrors(t *testing.T) {
-	withStdin(t, bytes.NewBufferString("hello"))
-	var claims ioClaims
-
-	_, err := readInput("ca-key", "-", &claims)
-	require.NoError(t, err)
-
-	_, err = readInput("ca-crt", "-", &claims)
-	require.EqualError(t, err, `-ca-key and -ca-crt both set to "-", only one input may read from stdin`)
-}
-
-func Test_openInput_stdin(t *testing.T) {
-	withStdin(t, bytes.NewBufferString("hi"))
-	var claims ioClaims
-
-	r, err := openInput("ca", "-", &claims)
-	require.NoError(t, err)
-	defer r.Close()
-	b, err := io.ReadAll(r)
-	require.NoError(t, err)
-	assert.Equal(t, []byte("hi"), b)
-}
-
-func Test_openInput_doubleStdinErrors(t *testing.T) {
-	withStdin(t, bytes.NewBufferString("hi"))
-	var claims ioClaims
-
-	r, err := openInput("ca", "-", &claims)
-	require.NoError(t, err)
-	r.Close()
-
-	_, err = openInput("crt", "-", &claims)
-	require.EqualError(t, err, `-ca and -crt both set to "-", only one input may read from stdin`)
-}
-
-func Test_writeOutput_stdout(t *testing.T) {
-	out := &bytes.Buffer{}
-
-	err := writeOutput("-", []byte("payload"), 0600, out)
-	require.NoError(t, err)
-	assert.Equal(t, "payload", out.String())
-}
-
-func Test_writeOutput_file(t *testing.T) {
-	dir := t.TempDir()
-	p := filepath.Join(dir, "f")
-	out := &bytes.Buffer{}
-
-	err := writeOutput(p, []byte("payload"), 0600, out)
-	require.NoError(t, err)
-	assert.Empty(t, out.String())
-	got, err := os.ReadFile(p)
-	require.NoError(t, err)
-	assert.Equal(t, []byte("payload"), got)
-}
-
-func Test_reserveOutputs_noConflict(t *testing.T) {
-	var claims ioClaims
-	require.NoError(t, reserveOutputs(&claims,
-		"out-key", "/tmp/key",
-		"out-crt", "-",
-		"out-qr", "",
-	))
-	assert.Equal(t, "out-crt", claims.out)
-}
-
-func Test_reserveOutputs_conflict(t *testing.T) {
-	var claims ioClaims
-	err := reserveOutputs(&claims,
-		"out-key", "-",
-		"out-crt", "-",
-	)
-	require.EqualError(t, err, `-out-key and -out-crt both set to "-", only one output may write to stdout`)
-}
-
-func Test_reserveOutputs_panicsOnOddPairs(t *testing.T) {
-	defer func() {
-		r := recover()
-		require.NotNil(t, r)
-	}()
-	var claims ioClaims
-	_ = reserveOutputs(&claims, "out-key")
-}
-
-func Test_reserveInputs_noConflict(t *testing.T) {
-	var claims ioClaims
-	require.NoError(t, reserveInputs(&claims,
-		"ca-key", "/tmp/ca.key",
-		"ca-crt", "-",
-		"in-pub", "",
-	))
-	assert.Equal(t, "ca-crt", claims.in)
-}
-
-func Test_reserveInputs_conflict(t *testing.T) {
-	var claims ioClaims
-	err := reserveInputs(&claims,
-		"ca-key", "-",
-		"ca-crt", "-",
-	)
-	require.EqualError(t, err, `-ca-key and -ca-crt both set to "-", only one input may read from stdin`)
-}
-
-func Test_claimIn_idempotent(t *testing.T) {
-	// pre-claim then a lazy re-claim of the same flag should be a no-op
-	var claims ioClaims
-	require.NoError(t, claims.claimIn("ca-key"))
-	require.NoError(t, claims.claimIn("ca-key"))
-	assert.Equal(t, "ca-key", claims.in)
-}
-
-func Test_claimOut_idempotent(t *testing.T) {
-	var claims ioClaims
-	require.NoError(t, claims.claimOut("out-crt"))
-	require.NoError(t, claims.claimOut("out-crt"))
-	assert.Equal(t, "out-crt", claims.out)
-}
-
-func Test_isStdio(t *testing.T) {
-	assert.True(t, isStdio("-"))
-	assert.False(t, isStdio(""))
-	assert.False(t, isStdio("./-"))
-	assert.False(t, isStdio("foo"))
-}

cmd/nebula-cert/verify.go

@@ -39,26 +39,18 @@ func verify(args []string, out io.Writer, errOut io.Writer) error {
 		return err
 	}
 
-	var claims ioClaims
+	caFile, err := os.Open(*vf.caPath)
-	if err := reserveInputs(&claims,
-		"ca", *vf.caPath,
-		"crt", *vf.certPath,
-	); err != nil {
-		return err
-	}
-
-	caReader, err := openInput("ca", *vf.caPath, &claims)
 	if err != nil {
 		return fmt.Errorf("error while reading ca: %w", err)
 	}
-	defer caReader.Close()
+	defer caFile.Close()
 
-	caPool, err := cert.NewCAPoolFromPEMReader(caReader)
+	caPool, err := cert.NewCAPoolFromPEMReader(caFile)
 	if err != nil && !errors.Is(err, cert.ErrExpired) {
 		return fmt.Errorf("error while adding ca cert to pool: %w", err)
 	}
 
-	rawCert, err := readInput("crt", *vf.certPath, &claims)
+	rawCert, err := os.ReadFile(*vf.certPath)
 	if err != nil {
 		return fmt.Errorf("unable to read crt: %w", err)
 	}
@@ -93,7 +85,6 @@ func verifySummary() string {
 func verifyHelp(out io.Writer) {
 	vf := newVerifyFlags()
 	_, _ = out.Write([]byte("Usage of " + os.Args[0] + " " + verifySummary() + "\n"))
-	_, _ = out.Write([]byte(stdioHelpText))
 	vf.set.SetOutput(out)
 	vf.set.PrintDefaults()
 }

cmd/nebula-cert/verify_test.go

@@ -23,7 +23,6 @@ func Test_verifyHelp(t *testing.T) {
 	assert.Equal(
 		t,
 		"Usage of "+os.Args[0]+" verify <flags>: verifies a certificate isn't expired and was signed by a trusted authority.\n"+
-			"  Pass \"-\" to any path flag to read from stdin or write to stdout.\n"+
 			"  -ca string\n"+
 			"    \tRequired: path to a file containing one or more ca certificates\n"+
 			"  -crt string\n"+
@@ -123,46 +122,3 @@ func Test_verify(t *testing.T) {
 	assert.Empty(t, eb.String())
 	require.NoError(t, err)
 }
-
-func Test_verify_stdio(t *testing.T) {
-	ob := &bytes.Buffer{}
-	eb := &bytes.Buffer{}
-
-	caPub, caPriv, _ := ed25519.GenerateKey(rand.Reader)
-	ca, _ := NewTestCaCert("test-ca", caPub, caPriv, time.Now().Add(time.Hour*-1), time.Now().Add(time.Hour*2), nil, nil, nil)
-	caPEM, _ := ca.MarshalPEM()
-
-	crt, _ := NewTestCert(ca, caPriv, "test-cert", time.Now().Add(time.Hour*-1), time.Now().Add(time.Hour), nil, nil, nil)
-	crtPEM, _ := crt.MarshalPEM()
-
-	caFile, err := os.CreateTemp("", "verify-ca")
-	require.NoError(t, err)
-	defer os.Remove(caFile.Name())
-	caFile.Write(caPEM)
-
-	// crt on stdin, ca on disk
-	withStdin(t, bytes.NewReader(crtPEM))
-	require.NoError(t, verify([]string{"-ca", caFile.Name(), "-crt", "-"}, ob, eb))
-	assert.Empty(t, ob.String())
-	assert.Empty(t, eb.String())
-
-	// ca on stdin, crt on disk
-	certFile, err := os.CreateTemp("", "verify-cert")
-	require.NoError(t, err)
-	defer os.Remove(certFile.Name())
-	certFile.Write(crtPEM)
-
-	withStdin(t, bytes.NewReader(caPEM))
-	ob.Reset()
-	eb.Reset()
-	require.NoError(t, verify([]string{"-ca", "-", "-crt", certFile.Name()}, ob, eb))
-	assert.Empty(t, ob.String())
-	assert.Empty(t, eb.String())
-
-	// both flags on stdin should error
-	withStdin(t, bytes.NewReader(caPEM))
-	ob.Reset()
-	eb.Reset()
-	require.EqualError(t, verify([]string{"-ca", "-", "-crt", "-"}, ob, eb),
-		`-ca and -crt both set to "-", only one input may read from stdin`)
-}

cmd/nebula-service/main.go

@@ -61,12 +61,9 @@ func main() {
 	}
 
 	if *configPath == "" {
-		p, err := config.DefaultPath()
+		fmt.Println("-config flag must be set")
-		if err != nil {
+		flag.Usage()
-			fmt.Println(err)
+		os.Exit(1)
-			os.Exit(1)
-		}
-		*configPath = p
 	}
 
 	c := config.NewC(l)

cmd/nebula-service/service.go

@@ -3,6 +3,8 @@ package main
 import (
 	"fmt"
 	"log"
+	"os"
+	"path/filepath"
 
 	"github.com/kardianos/service"
 	"github.com/slackhq/nebula"
@@ -55,13 +57,24 @@ func (p *program) Stop(s service.Service) error {
 	return nil
 }
 
+func fileExists(filename string) bool {
+	_, err := os.Stat(filename)
+	if os.IsNotExist(err) {
+		return false
+	}
+	return true
+}
+
 func doService(configPath *string, configTest *bool, build string, serviceFlag *string) error {
 	if *configPath == "" {
-		p, err := config.DefaultPath()
+		ex, err := os.Executable()
 		if err != nil {
 			return err
 		}
-		*configPath = p
+		*configPath = filepath.Dir(ex) + "/config.yaml"
+		if !fileExists(*configPath) {
+			*configPath = filepath.Dir(ex) + "/config.yml"
+		}
 	}
 
 	svcConfig := &service.Config{

cmd/nebula/main.go

@@ -50,12 +50,9 @@ func main() {
 	}
 
 	if *configPath == "" {
-		p, err := config.DefaultPath()
+		fmt.Println("-config flag must be set")
-		if err != nil {
+		flag.Usage()
-			fmt.Println(err)
+		os.Exit(1)
-			os.Exit(1)
-		}
-		*configPath = p
 	}
 
 	l := logging.NewLogger(os.Stdout)

config/default.go

@@ -1,29 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-)
-
-// DefaultPath returns a path to a config file alongside the running executable, preferring config.yaml over config.yml.
-// If neither file exists an error is returned that names both paths checked.
-func DefaultPath() (string, error) {
-	ex, err := os.Executable()
-	if err != nil {
-		return "", err
-	}
-	return defaultPathInDir(filepath.Dir(ex))
-}
-
-func defaultPathInDir(dir string) (string, error) {
-	yamlPath := filepath.Join(dir, "config.yaml")
-	if _, err := os.Stat(yamlPath); err == nil {
-		return yamlPath, nil
-	}
-	ymlPath := filepath.Join(dir, "config.yml")
-	if _, err := os.Stat(ymlPath); err == nil {
-		return ymlPath, nil
-	}
-	return "", fmt.Errorf("no default config found at %s or %s", yamlPath, ymlPath)
-}

config/default_test.go

@@ -1,67 +0,0 @@
-package config
-
-import (
-	"os"
-	"path/filepath"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestDefaultPathInDir(t *testing.T) {
-	t.Run("prefers config.yaml when both exist", func(t *testing.T) {
-		dir := t.TempDir()
-		want := filepath.Join(dir, "config.yaml")
-		other := filepath.Join(dir, "config.yml")
-		require.NoError(t, os.WriteFile(want, []byte("a: 1"), 0644))
-		require.NoError(t, os.WriteFile(other, []byte("a: 2"), 0644))
-
-		got, err := defaultPathInDir(dir)
-		require.NoError(t, err)
-		assert.Equal(t, want, got)
-	})
-
-	t.Run("returns config.yaml when only it exists", func(t *testing.T) {
-		dir := t.TempDir()
-		want := filepath.Join(dir, "config.yaml")
-		require.NoError(t, os.WriteFile(want, []byte("a: 1"), 0644))
-
-		got, err := defaultPathInDir(dir)
-		require.NoError(t, err)
-		assert.Equal(t, want, got)
-	})
-
-	t.Run("falls back to config.yml when only it exists", func(t *testing.T) {
-		dir := t.TempDir()
-		want := filepath.Join(dir, "config.yml")
-		require.NoError(t, os.WriteFile(want, []byte("a: 1"), 0644))
-
-		got, err := defaultPathInDir(dir)
-		require.NoError(t, err)
-		assert.Equal(t, want, got)
-	})
-
-	t.Run("errors when neither exists and names both paths", func(t *testing.T) {
-		dir := t.TempDir()
-		got, err := defaultPathInDir(dir)
-		assert.Empty(t, got)
-		require.Error(t, err)
-		assert.Contains(t, err.Error(), filepath.Join(dir, "config.yaml"))
-		assert.Contains(t, err.Error(), filepath.Join(dir, "config.yml"))
-	})
-}
-
-func TestDefaultPath(t *testing.T) {
-	got, err := DefaultPath()
-	if err != nil {
-		ex, exErr := os.Executable()
-		require.NoError(t, exErr)
-		assert.Contains(t, err.Error(), filepath.Dir(ex))
-		return
-	}
-	ex, err := os.Executable()
-	require.NoError(t, err)
-	assert.Equal(t, filepath.Dir(ex), filepath.Dir(got))
-	assert.Contains(t, []string{"config.yaml", "config.yml"}, filepath.Base(got))
-}

connection_manager.go

@@ -11,6 +11,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/rcrowley/go-metrics"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
@@ -44,16 +45,19 @@ type connectionManager struct {
 	inactivityTimeout       atomic.Int64
 	dropInactive            atomic.Bool
 
+	metricsTxPunchy metrics.Counter
+
 	l *slog.Logger
 }
 
 func newConnectionManagerFromConfig(l *slog.Logger, c *config.C, hm *HostMap, p *Punchy) *connectionManager {
 	cm := &connectionManager{
-		hostMap:       hm,
+		hostMap:         hm,
-		l:             l,
+		l:               l,
-		punchy:        p,
+		punchy:          p,
-		relayUsed:     make(map[uint32]struct{}),
+		relayUsed:       make(map[uint32]struct{}),
-		relayUsedLock: &sync.RWMutex{},
+		relayUsedLock:   &sync.RWMutex{},
+		metricsTxPunchy: metrics.GetOrRegisterCounter("messages.tx.punchy", nil),
 	}
 
 	cm.reload(c, true)
@@ -136,13 +140,21 @@ func (cm *connectionManager) getAndResetTrafficCheck(h *HostInfo, now time.Time)
 	return in, out
 }
 
+// AddTrafficWatch must be called for every new HostInfo.
+// We will continue to monitor the HostInfo until the tunnel is dropped.
+func (cm *connectionManager) AddTrafficWatch(h *HostInfo) {
+	if h.out.Swap(true) == false {
+		cm.trafficTimer.Add(h.localIndexId, cm.checkInterval)
+	}
+}
+
 func (cm *connectionManager) Start(ctx context.Context) {
 	clockSource := time.NewTicker(cm.trafficTimer.t.tickDuration)
 	defer clockSource.Stop()
 
 	p := []byte("")
-	nb := make([]byte, 12, 12)
+	// Long-lived buf for the traffic-check goroutine; never released.
-	out := make([]byte, mtu)
+	buf := cm.intf.bufAlloc.Acquire()
 
 	for {
 		select {
@@ -157,13 +169,13 @@ func (cm *connectionManager) Start(ctx context.Context) {
 					break
 				}
 
-				cm.doTrafficCheck(localIndex, p, nb, out, now)
+				cm.doTrafficCheck(localIndex, p, buf, now)
 			}
 		}
 	}
 }
 
-func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte, now time.Time) {
+func (cm *connectionManager) doTrafficCheck(localIndex uint32, p []byte, buf *WireBuffer, now time.Time) {
 	decision, hostinfo, primary := cm.makeTrafficDecision(localIndex, now)
 
 	switch decision {
@@ -187,7 +199,7 @@ func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte
 		cm.tryRehandshake(hostinfo)
 
 	case sendTestPacket:
-		cm.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, p, nb, out)
+		cm.intf.SendMessageToHostInfo(header.Test, header.TestRequest, hostinfo, p, buf)
 	}
 
 	cm.resetRelayTrafficCheck(hostinfo)
@@ -296,10 +308,12 @@ func (cm *connectionManager) migrateRelayUsed(oldhostinfo, newhostinfo *HostInfo
 		if err != nil {
 			cm.l.Error("failed to marshal Control message to migrate relay", "error", err)
 		} else {
-			cm.intf.SendMessageToHostInfo(header.Control, 0, newhostinfo, msg, make([]byte, 12), make([]byte, mtu))
+			migBuf := cm.intf.bufAlloc.Acquire()
+			cm.intf.SendMessageToHostInfo(header.Control, 0, newhostinfo, msg, migBuf)
+			cm.intf.bufAlloc.Release(migBuf)
 			cm.l.Info("send CreateRelayRequest",
-				"relayFrom", relayFrom,
+				"relayFrom", req.RelayFromAddr,
-				"relayTo", relayTo,
+				"relayTo", req.RelayToAddr,
 				"initiatorRelayIndex", req.InitiatorRelayIndex,
 				"responderRelayIndex", req.ResponderRelayIndex,
 				"vpnAddrs", newhostinfo.vpnAddrs,
@@ -357,7 +371,7 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
 
 		if !outTraffic {
 			// Send a punch packet to keep the NAT state alive
-			cm.punchy.SendPunch(hostinfo)
+			cm.sendPunch(hostinfo)
 		}
 
 		return decision, hostinfo, primary
@@ -388,16 +402,17 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
 
 			// If we aren't sending or receiving traffic then its an unused tunnel and we don't to test the tunnel.
 			// Just maintain NAT state if configured to do so.
-			cm.punchy.SendPunch(hostinfo)
+			cm.sendPunch(hostinfo)
 			cm.trafficTimer.Add(hostinfo.localIndexId, cm.checkInterval)
 			return doNothing, nil, nil
 		}
 
-		// We aren't receiving traffic but we are sending it. The outbound
+		if cm.punchy.GetTargetEverything() {
-		// traffic itself refreshes the primary remote's NAT state; this
+			// This is similar to the old punchy behavior with a slight optimization.
-		// fans out to non-primary remotes, but only if target_all_remotes
+			// We aren't receiving traffic but we are sending it, punch on all known
-		// is configured.
+			// ips in case we need to re-prime NAT state
-		cm.punchy.SendPunchToAll(hostinfo)
+			cm.sendPunch(hostinfo)
+		}
 
 		if cm.l.Enabled(context.Background(), slog.LevelDebug) {
 			hostinfo.logger(cm.l).Debug("Tunnel status",
@@ -499,6 +514,31 @@ func (cm *connectionManager) isInvalidCertificate(now time.Time, hostinfo *HostI
 	}
 }
 
+func (cm *connectionManager) sendPunch(hostinfo *HostInfo) {
+	if !cm.punchy.GetPunch() {
+		// Punching is disabled
+		return
+	}
+
+	if cm.intf.lightHouse.IsAnyLighthouseAddr(hostinfo.vpnAddrs) {
+		// Do not punch to lighthouses, we assume our lighthouse update interval is good enough.
+		// In the event the update interval is not sufficient to maintain NAT state then a publicly available lighthouse
+		// would lose the ability to notify us and punchy.respond would become unreliable.
+		return
+	}
+
+	if cm.punchy.GetTargetEverything() {
+		hostinfo.remotes.ForEach(cm.hostMap.GetPreferredRanges(), func(addr netip.AddrPort, preferred bool) {
+			cm.metricsTxPunchy.Inc(1)
+			cm.intf.outside.WriteTo([]byte{1}, addr)
+		})
+
+	} else if hostinfo.remote.IsValid() {
+		cm.metricsTxPunchy.Inc(1)
+		cm.intf.outside.WriteTo([]byte{1}, hostinfo.remote)
+	}
+}
+
 func (cm *connectionManager) tryRehandshake(hostinfo *HostInfo) {
 	cs := cm.intf.pki.getCertState()
 	curCrt := hostinfo.ConnectionState.myCert

connection_manager_test.go

@@ -64,12 +64,12 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 
 	// Create manager
 	conf := config.NewC(test.NewLogger())
-	punchy := NewPunchyFromConfig(test.NewLogger(), conf, nil)
+	punchy := NewPunchyFromConfig(test.NewLogger(), conf)
 	nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy)
 	nc.intf = ifce
+
 	p := []byte("")
-	nb := make([]byte, 12, 12)
+	buf := NewWireBuffer(mtu, 0)
-	out := make([]byte, mtu)
 
 	// Add an ip we have established a connection w/ to hostmap
 	hostinfo := &HostInfo{
@@ -92,7 +92,7 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	assert.True(t, hostinfo.in.Load())
 
 	// Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded
-	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
 	assert.False(t, hostinfo.pendingDeletion.Load())
 	assert.False(t, hostinfo.out.Load())
 	assert.False(t, hostinfo.in.Load())
@@ -100,7 +100,7 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	// Do another traffic check tick, this host should be pending deletion now
 	nc.Out(hostinfo)
 	assert.True(t, hostinfo.out.Load())
-	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
 	assert.True(t, hostinfo.pendingDeletion.Load())
 	assert.False(t, hostinfo.out.Load())
 	assert.False(t, hostinfo.in.Load())
@@ -108,7 +108,7 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
 
 	// Do a final traffic check tick, the host should now be removed
-	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
 	assert.NotContains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs)
 	assert.NotContains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 }
@@ -146,12 +146,12 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 
 	// Create manager
 	conf := config.NewC(test.NewLogger())
-	punchy := NewPunchyFromConfig(test.NewLogger(), conf, nil)
+	punchy := NewPunchyFromConfig(test.NewLogger(), conf)
 	nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy)
 	nc.intf = ifce
+
 	p := []byte("")
-	nb := make([]byte, 12, 12)
+	buf := NewWireBuffer(mtu, 0)
-	out := make([]byte, mtu)
 
 	// Add an ip we have established a connection w/ to hostmap
 	hostinfo := &HostInfo{
@@ -174,14 +174,14 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 	assert.Contains(t, nc.hostMap.Indexes, hostinfo.localIndexId)
 
 	// Do a traffic check tick, should not be pending deletion but should not have any in/out packets recorded
-	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
 	assert.False(t, hostinfo.pendingDeletion.Load())
 	assert.False(t, hostinfo.out.Load())
 	assert.False(t, hostinfo.in.Load())
 
 	// Do another traffic check tick, this host should be pending deletion now
 	nc.Out(hostinfo)
-	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
 	assert.True(t, hostinfo.pendingDeletion.Load())
 	assert.False(t, hostinfo.out.Load())
 	assert.False(t, hostinfo.in.Load())
@@ -190,7 +190,7 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 
 	// We saw traffic, should no longer be pending deletion
 	nc.In(hostinfo)
-	nc.doTrafficCheck(hostinfo.localIndexId, p, nb, out, time.Now())
+	nc.doTrafficCheck(hostinfo.localIndexId, p, buf, time.Now())
 	assert.False(t, hostinfo.pendingDeletion.Load())
 	assert.False(t, hostinfo.out.Load())
 	assert.False(t, hostinfo.in.Load())
@@ -233,7 +233,7 @@ func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
 	conf.Settings["tunnels"] = map[string]any{
 		"drop_inactive": true,
 	}
-	punchy := NewPunchyFromConfig(test.NewLogger(), conf, nil)
+	punchy := NewPunchyFromConfig(test.NewLogger(), conf)
 	nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy)
 	assert.True(t, nc.dropInactive.Load())
 	nc.intf = ifce
@@ -358,7 +358,7 @@ func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) {
 
 	// Create manager
 	conf := config.NewC(test.NewLogger())
-	punchy := NewPunchyFromConfig(test.NewLogger(), conf, nil)
+	punchy := NewPunchyFromConfig(test.NewLogger(), conf)
 	nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy)
 	nc.intf = ifce
 	ifce.connectionManager = nc

connection_state.go

@@ -7,14 +7,13 @@ import (
 
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/handshake"
-	"github.com/slackhq/nebula/noiseutil"
 )
 
 const ReplayWindow = 1024
 
 type ConnectionState struct {
-	eKey           noiseutil.CipherState
+	eKey           *NebulaCipherState
-	dKey           noiseutil.CipherState
+	dKey           *NebulaCipherState
 	myCert         cert.Certificate
 	peerCert       *cert.CachedCertificate
 	initiator      bool
@@ -32,8 +31,8 @@ func newConnectionStateFromResult(r *handshake.Result) *ConnectionState {
 		myCert:    r.MyCert,
 		initiator: r.Initiator,
 		peerCert:  r.RemoteCert,
-		eKey:      noiseutil.NewCipherState(r.EKey, r.Cipher),
+		eKey:      NewNebulaCipherState(r.EKey),
-		dKey:      noiseutil.NewCipherState(r.DKey, r.Cipher),
+		dKey:      NewNebulaCipherState(r.DKey),
 		window:    NewBits(ReplayWindow),
 	}
 	ci.messageCounter.Add(r.MessageIndex)

control.go

@@ -278,15 +278,9 @@ func (c *Control) CloseTunnel(vpnIp netip.Addr, localOnly bool) bool {
 	}
 
 	if !localOnly {
-		c.f.send(
+		buf := c.f.bufAlloc.Acquire()
-			header.CloseTunnel,
+		c.f.send(header.CloseTunnel, 0, hostInfo.ConnectionState, hostInfo, []byte{}, buf)
-			0,
+		c.f.bufAlloc.Release(buf)
-			hostInfo.ConnectionState,
-			hostInfo,
-			[]byte{},
-			make([]byte, 12, 12),
-			make([]byte, mtu),
-		)
 	}
 
 	c.f.closeTunnel(hostInfo)
@@ -296,16 +290,19 @@ func (c *Control) CloseTunnel(vpnIp netip.Addr, localOnly bool) bool {
 // CloseAllTunnels is just like CloseTunnel except it goes through and shuts them all down, optionally you can avoid shutting down lighthouse tunnels
 // the int returned is a count of tunnels closed
 func (c *Control) CloseAllTunnels(excludeLighthouses bool) (closed int) {
+	// One WireBuffer for the whole shutdown loop.
+	buf := c.f.bufAlloc.Acquire()
+	defer c.f.bufAlloc.Release(buf)
 	shutdown := func(h *HostInfo) {
 		if excludeLighthouses && c.f.lightHouse.IsAnyLighthouseAddr(h.vpnAddrs) {
 			return
 		}
-		c.f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, make([]byte, 12, 12), make([]byte, mtu))
+		c.f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, buf)
 		c.f.closeTunnel(h)
 
 		c.l.Debug("Sending close tunnel message",
 			"vpnAddrs", h.vpnAddrs,
-			"udpAddr", h.GetRemote(),
+			"udpAddr", h.remote,
 		)
 		closed++
 	}
@@ -350,7 +347,7 @@ func copyHostInfo(h *HostInfo, preferredRanges []netip.Prefix) ControlHostInfo {
 		RemoteAddrs:            h.remotes.CopyAddrs(preferredRanges),
 		CurrentRelaysToMe:      h.relayState.CopyRelayIps(),
 		CurrentRelaysThroughMe: h.relayState.CopyRelayForIps(),
-		CurrentRemote:          h.GetRemote(),
+		CurrentRemote:          h.remote,
 	}
 
 	for i, a := range h.vpnAddrs {

control_test.go

@@ -1,8 +1,6 @@
 package nebula
 
 import (
-	"bytes"
-	"log/slog"
 	"net"
 	"net/netip"
 	"reflect"
@@ -11,7 +9,6 @@ import (
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestControl_GetHostInfoByVpnIp(t *testing.T) {
@@ -45,7 +42,8 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) {
 	assert.True(t, ok)
 
 	crt := &dummyCert{}
-	hi := &HostInfo{
+	hm.unlockedAddHostInfo(&HostInfo{
+		remote:  remote1,
 		remotes: remotes,
 		ConnectionState: &ConnectionState{
 			peerCert: &cert.CachedCertificate{Certificate: crt},
@@ -58,14 +56,13 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) {
 			relayForByAddr: map[netip.Addr]*Relay{},
 			relayForByIdx:  map[uint32]*Relay{},
 		},
-	}
+	}, &Interface{})
-	hi.remote.Store(&remote1)
-	hm.unlockedAddHostInfo(hi, &Interface{})
 
 	vpnIp2, ok := netip.AddrFromSlice(ipNet2.IP)
 	assert.True(t, ok)
 
-	hi2 := &HostInfo{
+	hm.unlockedAddHostInfo(&HostInfo{
+		remote:  remote1,
 		remotes: remotes,
 		ConnectionState: &ConnectionState{
 			peerCert: nil,
@@ -78,9 +75,7 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) {
 			relayForByAddr: map[netip.Addr]*Relay{},
 			relayForByIdx:  map[uint32]*Relay{},
 		},
-	}
+	}, &Interface{})
-	hi2.remote.Store(&remote1)
-	hm.unlockedAddHostInfo(hi2, &Interface{})
 
 	c := Control{
 		state: StateReady,
@@ -124,153 +119,3 @@ func assertFields(t *testing.T, expected []string, actualStruct any) {
 
 	assert.Equal(t, expected, fields)
 }
-
-// alwaysAllowV4/V6 are check funcs that accept every entry (including nil pointers),
-// letting us inject a nil *V4AddrPort/*V6AddrPort into a RemoteList's reported cache
-// the same way a malformed proto message off the wire could.
-func alwaysAllowV4(netip.Addr, *V4AddrPort) bool { return true }
-func alwaysAllowV6(netip.Addr, *V6AddrPort) bool { return true }
-
-// TestGetRelays_SkipsNilRelayAddrs proves GetRelays tolerates nil entries in the
-// RelayVpnAddrs proto slice (which protoAddrToNetAddr would nil-deref on) and still
-// returns the valid relays, including the legacy OldRelayVpnAddrs.
-func TestGetRelays_SkipsNilRelayAddrs(t *testing.T) {
-	good := netip.MustParseAddr("10.0.0.9")
-
-	d := &NebulaMetaDetails{
-		OldRelayVpnAddrs: []uint32{0x0a000001}, // 10.0.0.1
-		RelayVpnAddrs: []*Addr{
-			nil,
-			netAddrToProtoAddr(good),
-			nil,
-		},
-	}
-
-	var relays []netip.Addr
-	require.NotPanics(t, func() { relays = d.GetRelays() })
-
-	assert.Equal(t, []netip.Addr{
-		netip.MustParseAddr("10.0.0.1"),
-		good,
-	}, relays)
-}
-
-// TestGetRelays_AllNil ensures an all-nil RelayVpnAddrs slice yields no relays and no panic.
-func TestGetRelays_AllNil(t *testing.T) {
-	d := &NebulaMetaDetails{RelayVpnAddrs: []*Addr{nil, nil}}
-	var relays []netip.Addr
-	require.NotPanics(t, func() { relays = d.GetRelays() })
-	assert.Empty(t, relays)
-}
-
-// TestRemoteList_CopyCache_SkipsNilReported proves CopyCache skips nil reported
-// pointers (v4 and v6) instead of nil-dereferencing them in protoV*AddrPortToNetAddrPort.
-func TestRemoteList_CopyCache_SkipsNilReported(t *testing.T) {
-	owner := netip.MustParseAddr("10.0.0.1")
-	rl := NewRemoteList([]netip.Addr{owner}, nil)
-
-	rl.unlockedSetV4(owner, owner, []*V4AddrPort{
-		nil,
-		newIp4AndPortFromString("1.2.3.4:5"),
-		nil,
-	}, alwaysAllowV4)
-
-	rl.unlockedSetV6(owner, owner, []*V6AddrPort{
-		nil,
-		newIp6AndPortFromString("[1::1]:6"),
-		nil,
-	}, alwaysAllowV6)
-
-	var cm *CacheMap
-	require.NotPanics(t, func() { cm = rl.CopyCache() })
-
-	c := (*cm)[owner.String()]
-	require.NotNil(t, c)
-	assert.ElementsMatch(t, []netip.AddrPort{
-		netip.MustParseAddrPort("1.2.3.4:5"),
-		netip.MustParseAddrPort("[1::1]:6"),
-	}, c.Reported)
-}
-
-// TestRemoteList_Rebuild_SkipsNilReported drives unlockedCollect (via Rebuild) with
-// nil reported entries and confirms only the valid addresses survive, with no panic.
-func TestRemoteList_Rebuild_SkipsNilReported(t *testing.T) {
-	owner := netip.MustParseAddr("10.0.0.1")
-	rl := NewRemoteList([]netip.Addr{owner}, nil)
-
-	rl.unlockedSetV4(owner, owner, []*V4AddrPort{
-		nil,
-		newIp4AndPortFromString("1.2.3.4:5"),
-	}, alwaysAllowV4)
-	rl.unlockedSetV6(owner, owner, []*V6AddrPort{
-		newIp6AndPortFromString("[1::1]:6"),
-		nil,
-	}, alwaysAllowV6)
-
-	require.NotPanics(t, func() { rl.Rebuild([]netip.Prefix{}) })
-
-	assert.ElementsMatch(t, []netip.AddrPort{
-		netip.MustParseAddrPort("1.2.3.4:5"),
-		netip.MustParseAddrPort("[1::1]:6"),
-	}, rl.addrs)
-}
-
-// newRelayControl marshals a NebulaControl the way it arrives on the wire so we can feed
-// it through HandleControlMsg's unmarshal + validate path.
-func newRelayControl(t *testing.T, typ NebulaControl_MessageType, from, to *Addr) []byte {
-	t.Helper()
-	msg := &NebulaControl{
-		Type:          typ,
-		RelayFromAddr: from,
-		RelayToAddr:   to,
-	}
-	b, err := msg.Marshal()
-	require.NoError(t, err)
-	return b
-}
-
-// TestRelayManager_HandleControlMsg_NilRelayAddrs verifies the validation block added to
-// HandleControlMsg: CreateRelay{Request,Response} carrying a nil RelayFromAddr or
-// RelayToAddr are dropped with a debug log rather than nil-dereferencing downstream.
-func TestRelayManager_HandleControlMsg_NilRelayAddrs(t *testing.T) {
-	good := netAddrToProtoAddr(netip.MustParseAddr("10.0.0.9"))
-
-	cases := []struct {
-		name    string
-		typ     NebulaControl_MessageType
-		from    *Addr
-		to      *Addr
-		wantLog string // debug substring expected, "" == expect no drop log
-	}{
-		{"request nil from", NebulaControl_CreateRelayRequest, nil, good, "nil RelayFromAddr"},
-		{"request nil to", NebulaControl_CreateRelayRequest, good, nil, "nil RelayToAddr"},
-		{"request both nil", NebulaControl_CreateRelayRequest, nil, nil, "nil RelayFromAddr"},
-		{"response nil from", NebulaControl_CreateRelayResponse, nil, good, "nil RelayFromAddr"},
-		{"response nil to", NebulaControl_CreateRelayResponse, good, nil, "nil RelayToAddr"},
-		// A non-relay control type is not subject to the relay-addr validation and must
-		// pass through it untouched (the final switch simply no-ops on it).
-		{"unrelated type nil addrs", NebulaControl_None, nil, nil, ""},
-	}
-
-	for _, tc := range cases {
-		t.Run(tc.name, func(t *testing.T) {
-			var buf bytes.Buffer
-			l := test.NewLoggerWithOutputAndLevel(&buf, slog.LevelDebug)
-			rm := &relayManager{l: l, hostmap: newHostMap(l)}
-			rm.useRelays.Store(true)
-
-			f := &Interface{l: l}
-			h := &HostInfo{vpnAddrs: []netip.Addr{netip.MustParseAddr("10.0.0.2")}, localIndexId: 1}
-
-			d := newRelayControl(t, tc.typ, tc.from, tc.to)
-
-			require.NotPanics(t, func() { rm.HandleControlMsg(h, d, f) })
-
-			if tc.wantLog == "" {
-				assert.NotContains(t, buf.String(), "nil Relay")
-			} else {
-				assert.Contains(t, buf.String(), tc.wantLog)
-			}
-		})
-	}
-}

dist/wireshark/nebula.lua

@@ -62,7 +62,7 @@ function nebula.dissector(tvbuf, pktinfo, root)
     tree:add(pf_version, tvbuf:range(0,1))
     local type = tree:add(pf_type,    tvbuf:range(0,1))
 
-    local nebula_type = bit.band(tvbuf:range(0,1):uint(), 0x0F)
+    local nebula_type = bit32.band(tvbuf:range(0,1):uint(), 0x0F)
     if nebula_type == 0 then
         local stage = tvbuf(8,8):uint64()
         tree:add(pf_subtype_handshake, tvbuf:range(1,1))

dns_server.go

@@ -11,21 +11,19 @@ import (
 	"sync"
 	"sync/atomic"
 
+	"github.com/gaissmai/bart"
 	"github.com/miekg/dns"
 	"github.com/slackhq/nebula/config"
 )
 
 type dnsServer struct {
 	sync.RWMutex
-	l       *slog.Logger
+	l               *slog.Logger
-	ctx     context.Context
+	ctx             context.Context
-	dnsMap4 map[string]netip.Addr
+	dnsMap4         map[string]netip.Addr
-	dnsMap6 map[string]netip.Addr
+	dnsMap6         map[string]netip.Addr
-	hostMap *HostMap
+	hostMap         *HostMap
-	pki     *PKI
+	myVpnAddrsTable *bart.Lite
-
-	// selfHost is the cached FQDN we last seeded for ourselves
-	selfHost string
 
 	mux *dns.ServeMux
 
@@ -57,14 +55,14 @@ type dnsServer struct {
 // they no-op when DNS isn't enabled. Each Start invocation owns a ctx-cancel
 // watcher that tears the listener down on nebula shutdown. The returned
 // pointer is always non-nil, even on error.
-func newDnsServerFromConfig(ctx context.Context, l *slog.Logger, pki *PKI, hostMap *HostMap, c *config.C) (*dnsServer, error) {
+func newDnsServerFromConfig(ctx context.Context, l *slog.Logger, cs *CertState, hostMap *HostMap, c *config.C) (*dnsServer, error) {
 	ds := &dnsServer{
-		l:       l,
+		l:               l,
-		ctx:     ctx,
+		ctx:             ctx,
-		dnsMap4: make(map[string]netip.Addr),
+		dnsMap4:         make(map[string]netip.Addr),
-		dnsMap6: make(map[string]netip.Addr),
+		dnsMap6:         make(map[string]netip.Addr),
-		hostMap: hostMap,
+		hostMap:         hostMap,
-		pki:     pki,
+		myVpnAddrsTable: cs.myVpnAddrsTable,
 	}
 	ds.mux = dns.NewServeMux()
 	ds.mux.HandleFunc(".", ds.handleDnsRequest)
@@ -78,7 +76,6 @@ func newDnsServerFromConfig(ctx context.Context, l *slog.Logger, pki *PKI, hostM
 	if err := ds.reload(c, true); err != nil {
 		return ds, err
 	}
-	ds.seedSelf()
 	return ds, nil
 }
 
@@ -116,7 +113,7 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
 			d.Stop()
 		}
 		// Drop any records that accumulated while enabled; a later re-enable
-		// will repopulate from fresh handshakes and a fresh seedSelf.
+		// will repopulate from fresh handshakes.
 		d.clearRecords()
 		return nil
 	}
@@ -124,14 +121,17 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
 	if running == nil {
 		// Was disabled (or never started); bring it up now.
 		go d.Start()
-	} else if !sameAddr {
+		return nil
-		d.shutdownServer(running, runningStarted, "reload")
-		// Old Start goroutine has now exited; bring up a fresh listener on the new address.
-		go d.Start()
 	}
 
-	// Refresh the self entry every enabled reload so cert renewals that change our name or VPN addresses are picked up.
+	if sameAddr {
-	d.seedSelf()
+		return nil
+	}
+
+	d.shutdownServer(running, runningStarted, "reload")
+	// Old Start goroutine has now exited; bring up a fresh listener on the
+	// new address.
+	go d.Start()
 	return nil
 }
 
@@ -249,20 +249,6 @@ func (d *dnsServer) QueryCert(data string) string {
 		return ""
 	}
 
-	// The hostmap only ever contains peers we have handshaked with, so it never carries an entry for ourselves.
-	// Answer self lookups straight from the local cert state.
-	if cs := d.certState(); cs != nil && cs.myVpnAddrsTable != nil && cs.myVpnAddrsTable.Contains(ip) {
-		c := cs.GetDefaultCertificate()
-		if c == nil {
-			return ""
-		}
-		b, err := c.MarshalJSON()
-		if err != nil {
-			return ""
-		}
-		return string(b)
-	}
-
 	hostinfo := d.hostMap.QueryVpnAddr(ip)
 	if hostinfo == nil {
 		return ""
@@ -280,60 +266,12 @@ func (d *dnsServer) QueryCert(data string) string {
 	return string(b)
 }
 
-// clearRecords drops all DNS records, including the self entry.
+// clearRecords drops all DNS records.
 func (d *dnsServer) clearRecords() {
 	d.Lock()
 	defer d.Unlock()
 	clear(d.dnsMap4)
 	clear(d.dnsMap6)
-	d.selfHost = ""
-}
-
-// seedSelf inserts (or refreshes) a record for our own cert name pointing at our VPN addresses,
-// so a single-lighthouse network can resolve the lighthouse's own hostname without the two-process workaround.
-func (d *dnsServer) seedSelf() {
-	if !d.enabled.Load() {
-		return
-	}
-	cs := d.certState()
-	if cs == nil {
-		return
-	}
-	c := cs.GetDefaultCertificate()
-	if c == nil {
-		return
-	}
-	newHost := strings.ToLower(c.Name()) + "."
-
-	d.Lock()
-	defer d.Unlock()
-	if d.selfHost != "" && d.selfHost != newHost {
-		delete(d.dnsMap4, d.selfHost)
-		delete(d.dnsMap6, d.selfHost)
-	}
-	d.selfHost = newHost
-	delete(d.dnsMap4, newHost)
-	delete(d.dnsMap6, newHost)
-	haveV4, haveV6 := false, false
-	for _, addr := range cs.myVpnAddrs {
-		if addr.Is4() && !haveV4 {
-			d.dnsMap4[newHost] = addr
-			haveV4 = true
-		} else if addr.Is6() && !haveV6 {
-			d.dnsMap6[newHost] = addr
-			haveV6 = true
-		}
-		if haveV4 && haveV6 {
-			break
-		}
-	}
-}
-
-func (d *dnsServer) certState() *CertState {
-	if d.pki == nil {
-		return nil
-	}
-	return d.pki.getCertState()
 }
 
 // Add adds the first IPv4 and IPv6 address that appears in `addresses` as the record for `host`
@@ -371,12 +309,8 @@ func (d *dnsServer) isSelfNebulaOrLocalhost(addr string) bool {
 		return true
 	}
 
-	cs := d.certState()
-	if cs == nil || cs.myVpnAddrsTable == nil {
-		return false
-	}
 	//if we found it in this table, it's good
-	return cs.myVpnAddrsTable.Contains(b)
+	return d.myVpnAddrsTable.Contains(b)
 }
 
 func (d *dnsServer) parseQuery(m *dns.Msg, w dns.ResponseWriter) {

dns_server_test.go

@@ -9,10 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gaissmai/bart"
 	"github.com/miekg/dns"
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/cert_test"
 	"github.com/slackhq/nebula/config"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -279,92 +276,6 @@ func TestDnsServer_Stop_beforeBind_doesNotHang(t *testing.T) {
 	}
 }
 
-// newTestPKI builds a minimal *PKI with a single v1 cert whose name and
-// VPN addresses are caller-provided, suitable for exercising seedSelf and
-// QueryCert self handling.
-func newTestPKI(t *testing.T, name string, addrs []netip.Addr) *PKI {
-	t.Helper()
-	networks := make([]netip.Prefix, 0, len(addrs))
-	for _, a := range addrs {
-		bits := 32
-		if a.Is6() {
-			bits = 128
-		}
-		networks = append(networks, netip.PrefixFrom(a, bits))
-	}
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil)
-	c, _, _, _ := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, name, time.Time{}, time.Time{}, networks, nil, nil)
-
-	addrsTable := new(bart.Lite)
-	for _, a := range addrs {
-		addrsTable.Insert(netip.PrefixFrom(a, a.BitLen()))
-	}
-
-	cs := &CertState{
-		v2Cert:            c,
-		initiatingVersion: cert.Version2,
-		myVpnAddrs:        addrs,
-		myVpnAddrsTable:   addrsTable,
-	}
-	pki := &PKI{}
-	pki.cs.Store(cs)
-	return pki
-}
-
-func TestDnsServer_seedSelf_addsOwnRecord(t *testing.T) {
-	ds, c := newTestDnsServer(t)
-	myV4 := netip.MustParseAddr("10.0.0.1")
-	myV6 := netip.MustParseAddr("fd00::1")
-	ds.pki = newTestPKI(t, "lighthouse", []netip.Addr{myV4, myV6})
-	setDnsConfig(c, "127.0.0.1", "0", true, true)
-	require.NoError(t, ds.reload(c, true))
-
-	ds.seedSelf()
-	got4, exists := ds.Query(dns.TypeA, "lighthouse.")
-	assert.True(t, exists)
-	assert.Equal(t, myV4, got4)
-	got6, exists := ds.Query(dns.TypeAAAA, "lighthouse.")
-	assert.True(t, exists)
-	assert.Equal(t, myV6, got6)
-}
-
-func TestDnsServer_seedSelf_disabled_noOp(t *testing.T) {
-	ds, c := newTestDnsServer(t)
-	ds.pki = newTestPKI(t, "lighthouse", []netip.Addr{netip.MustParseAddr("10.0.0.1")})
-	setDnsConfig(c, "127.0.0.1", "0", true, false)
-	require.NoError(t, ds.reload(c, true))
-
-	ds.seedSelf()
-	_, exists := ds.Query(dns.TypeA, "lighthouse.")
-	assert.False(t, exists)
-}
-
-func TestDnsServer_clearRecords_dropsSelfHost(t *testing.T) {
-	ds, c := newTestDnsServer(t)
-	ds.pki = newTestPKI(t, "lighthouse", []netip.Addr{netip.MustParseAddr("10.0.0.1")})
-	setDnsConfig(c, "127.0.0.1", "0", true, true)
-	require.NoError(t, ds.reload(c, true))
-	ds.seedSelf()
-	require.NotEmpty(t, ds.selfHost)
-
-	ds.clearRecords()
-	assert.Empty(t, ds.selfHost)
-	_, exists := ds.Query(dns.TypeA, "lighthouse.")
-	assert.False(t, exists)
-}
-
-func TestDnsServer_QueryCert_returnsOwnCert(t *testing.T) {
-	ds, _ := newTestDnsServer(t)
-	myV4 := netip.MustParseAddr("10.0.0.1")
-	ds.pki = newTestPKI(t, "lighthouse", []netip.Addr{myV4})
-
-	got := ds.QueryCert(myV4.String() + ".")
-	assert.NotEmpty(t, got, "TXT lookup of our own VPN address should return our cert")
-
-	other := netip.MustParseAddr("10.0.0.99")
-	assert.Empty(t, ds.QueryCert(other.String()+"."), "unknown peer IP should return nothing")
-}
-
 func TestDnsServer_reload_disable_stopsRunningServer(t *testing.T) {
 	port := freeUDPPort(t)
 	ds, c := newTestDnsServer(t)

docker/Dockerfile

@@ -1,16 +1,6 @@
 FROM gcr.io/distroless/static:latest
 
 ARG TARGETOS TARGETARCH
-
-ARG VERSION=dev
-ARG REVISION=unknown
-LABEL org.opencontainers.image.title="nebula" \
-      org.opencontainers.image.description="A scalable overlay networking tool with a focus on performance, simplicity and security" \
-      org.opencontainers.image.vendor="Nebula OSS" \
-      org.opencontainers.image.source="https://github.com/slackhq/nebula" \
-      org.opencontainers.image.version="${VERSION}" \
-      org.opencontainers.image.revision="${REVISION}"
-
 COPY build/$TARGETOS-$TARGETARCH/nebula /nebula
 COPY build/$TARGETOS-$TARGETARCH/nebula-cert /nebula-cert
 

e2e/bench_test.go

@@ -0,0 +1,68 @@
+//go:build e2e_testing
+// +build e2e_testing
+
+package e2e
+
+import (
+	"testing"
+	"time"
+
+	"github.com/slackhq/nebula/cert"
+	"github.com/slackhq/nebula/cert_test"
+	"github.com/slackhq/nebula/e2e/router"
+)
+
+// BenchmarkHandshake measures end-to-end tunnel establishment time. The two
+// nodes and the router are constructed once before the loop so the timed window
+// is just the handshake itself: trigger packet -> handshake1 -> handshake2 ->
+// cached packet replay -> arrival on the remote TUN. Between iterations we
+// tear down both sides locally (no CloseTunnel notification on the wire) and
+// re-inject the lighthouse address that closeTunnel cleared, so the next
+// iteration runs through a fresh handshake against the same harness.
+func BenchmarkHandshake(b *testing.B) {
+	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
+
+	// Default try_interval is 100ms. The handshake manager schedules handshake1
+	// on its OutboundHandshakeTimer rather than firing immediately on trigger
+	// (the trigger channel only fast-paths static hosts), so a 100ms default
+	// drowns the actual handshake cost. Drop it to 1ms so the bench reflects
+	// the computation, not the wheel cadence.
+	bovr := m{"handshakes": m{"try_interval": "1ms"}}
+	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.1/24", bovr)
+	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.2/24", bovr)
+
+	myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+
+	myControl.Start()
+	theirControl.Start()
+	defer myControl.Stop()
+	defer theirControl.Stop()
+
+	r := router.NewR(b, myControl, theirControl)
+	r.CancelFlowLogs()
+	r.EnableFanIn()
+
+	trigger := BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me"))
+
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	for n := 0; n < b.N; n++ {
+		myControl.InjectTunPacket(trigger)
+		// RouteForAllUntilTxTun returns the moment the cached packet arrives at
+		// the remote TUN, which is also when both sides are fully established.
+		_ = r.RouteForAllUntilTxTun(theirControl)
+
+		b.StopTimer()
+		// Local-only close removes hostmap state on both sides without putting a
+		// CloseTunnel packet on the wire that we'd then have to drain. The
+		// closeTunnel path also clears learned lighthouse state for the peer
+		// when the last hostinfo for that addr goes away, so we re-inject.
+		myControl.CloseTunnel(theirVpnIpNet[0].Addr(), true)
+		theirControl.CloseTunnel(myVpnIpNet[0].Addr(), true)
+		myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
+		theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
+		b.StartTimer()
+	}
+}

e2e/echo_test.go

@@ -1,85 +0,0 @@
-//go:build e2e_testing
-// +build e2e_testing
-
-package e2e
-
-import (
-	"testing"
-	"time"
-
-	"github.com/slackhq/nebula"
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/cert_test"
-	"github.com/slackhq/nebula/e2e/router"
-	"github.com/slackhq/nebula/header"
-	"github.com/slackhq/nebula/udp"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func assertTestRequestEchoed(t *testing.T, cipher string) {
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	over := m{"cipher": cipher}
-	a, aNet, aUdp, _ := newSimpleServer(cert.Version1, ca, caKey, "a", "10.128.0.1/24", over)
-	b, bNet, bUdp, _ := newSimpleServer(cert.Version1, ca, caKey, "b", "10.128.0.2/24", over)
-
-	a.InjectLightHouseAddr(bNet[0].Addr(), bUdp)
-	b.InjectLightHouseAddr(aNet[0].Addr(), aUdp)
-	a.Start()
-	b.Start()
-	t.Cleanup(func() { a.Stop(); b.Stop() })
-	r := router.NewR(t, a, b)
-	defer r.RenderFlow()
-
-	assertTunnel(t, aNet[0].Addr(), bNet[0].Addr(), a, b, r)
-	drainUDPTx(a)
-	drainUDPTx(b)
-
-	payload := []byte("a test payload well over sixteen bytes long, wow it's so very long long long!")
-	require.Greater(t, len(payload), header.Len)
-	a.GetF().SendMessageToVpnAddr(header.Test, header.TestRequest, bNet[0].Addr(), payload, make([]byte, 12, 12), make([]byte, udp.MTU))
-
-	// Deliver A's request to B; B must echo a reply back
-	b.InjectUDPPacket(a.GetFromUDP(true))
-	reply := nextUDPTxOfType(t, b, header.Test, header.TestReply, 2*time.Second)
-
-	assert.Equal(t, aUdp, reply.To, "the reply must go back to the requester")
-	// header + echoed payload + 16-byte AEAD tag: proves the whole payload
-	// round-tripped rather than being dropped or truncated.
-	assert.Equal(t, header.Len+len(payload)+16, len(reply.Data), "the full payload must be echoed back")
-}
-
-func TestTestRequestEchoesLongPayloadAES(t *testing.T) {
-	assertTestRequestEchoed(t, "aes")
-}
-
-func TestTestRequestEchoesLongPayloadChaChaPoly(t *testing.T) {
-	assertTestRequestEchoed(t, "chachapoly")
-}
-
-// drainUDPTx empties a control's UDP tx queue without blocking.
-func drainUDPTx(c *nebula.Control) {
-	for c.GetFromUDP(false) != nil {
-	}
-}
-
-// nextUDPTxOfType returns the next packet a control transmits whose nebula
-// header matches (wantType, wantSub), skipping unrelated packets.
-// It fails the test if none arrives within the timeout.
-func nextUDPTxOfType(t *testing.T, c *nebula.Control, wantType header.MessageType, wantSub header.MessageSubType, within time.Duration) *udp.Packet {
-	t.Helper()
-	ch := c.GetUDPTxChan()
-	timeout := time.After(within)
-	for {
-		select {
-		case p := <-ch:
-			var h header.H
-			if err := h.Parse(p.Data); err == nil && h.Type == wantType && h.Subtype == wantSub {
-				return p
-			}
-		case <-timeout:
-			t.Fatalf("timed out waiting for a %v/%v packet on the udp tx queue", wantType, wantSub)
-			return nil
-		}
-	}
-}

e2e/handshakes_test.go

@@ -165,7 +165,7 @@ func TestGoodHandshakeNoOverlap(t *testing.T) {
 
 	empty := []byte{}
 	t.Log("do something to cause a handshake")
-	myControl.GetF().SendMessageToVpnAddr(header.Test, header.MessageNone, theirVpnIpNet[0].Addr(), empty, empty, empty)
+	myControl.GetF().SendMessageToVpnAddr(header.Test, header.MessageNone, theirVpnIpNet[0].Addr(), empty, nebula.NewWireBuffer(9001, 0))
 
 	t.Log("Have them consume my stage 0 packet. They have a tunnel now")
 	theirControl.InjectUDPPacket(myControl.GetFromUDP(true))
@@ -1535,78 +1535,3 @@ func TestGoodHandshakeUnsafeDest(t *testing.T) {
 	myControl.Stop()
 	theirControl.Stop()
 }
-
-func TestMultiVpnAddrDeletePrimaryKeepsSecondAddr(t *testing.T) {
-	t.Parallel()
-	// Regression for the hostmap multi-vpnAddr delete bug. A dual-stack (v4+v6) V2-cert peer that
-	// handshakes twice at once ends up with two hostinfos linked in the shared next/prev chain, with the
-	// primary owning both addresses. Deleting that primary (e.g. connection manager dropping it, a
-	// CloseTunnel, a collision) must promote the surviving sibling for EVERY address. The pre-fix code
-	// unlinked the chain once per address, so it promoted the sibling for the first address and orphaned
-	// the second: the peer stayed reachable at its v4 addr but not its v6 addr despite a live tunnel.
-
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me  ", "10.128.0.1/24,fd00::1/64", nil)
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.2/24,fd00::2/64", nil)
-
-	// This bug only exists for peers carrying more than one vpn address
-	require.Len(t, theirVpnIpNet, 2)
-	theirV4 := theirVpnIpNet[0].Addr()
-	theirV6 := theirVpnIpNet[1].Addr()
-
-	// Put their info in our lighthouse and vice versa
-	myControl.InjectLightHouseAddr(theirV4, theirUdpAddr)
-	theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
-
-	// Build a router so we don't have to reason who gets which packet
-	r := router.NewR(t, myControl, theirControl)
-	defer r.RenderFlow()
-
-	myControl.Start()
-	theirControl.Start()
-
-	// Race a handshake so both of us build a hostinfo for the other, leaving my hostmap with a single
-	// host (them) backed by two linked hostinfos, just like TestStage1Race.
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirV4, 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirV4, 80, []byte("Hi from them")))
-
-	myHsForThem := myControl.GetFromUDP(true)
-	theirHsForMe := theirControl.GetFromUDP(true)
-
-	r.InjectUDPPacket(theirControl, myControl, theirHsForMe)
-	r.InjectUDPPacket(myControl, theirControl, myHsForThem)
-
-	r.RouteForAllUntilTxTun(theirControl)
-	r.RouteForAllUntilTxTun(myControl)
-
-	r.RenderHostmaps("Racing hostmaps", myControl, theirControl)
-
-	// Two hostinfos for them means the shared next/prev chain has a sibling to promote. The Hosts map has
-	// one entry per vpn address (two, for dual stack), so the index count is what tells us there are two
-	// hostinfos.
-	require.Len(t, myControl.ListHostmapIndexes(false), 2)
-
-	// The primary owns both of their addresses
-	primaryV4 := myControl.GetHostInfoByVpnAddr(theirV4, false)
-	primaryV6 := myControl.GetHostInfoByVpnAddr(theirV6, false)
-	require.NotNil(t, primaryV4)
-	require.NotNil(t, primaryV6)
-	require.Equal(t, primaryV4.LocalIndex, primaryV6.LocalIndex, "both addrs should point at the same primary")
-
-	// Delete the primary tunnel. localOnly so we don't perturb their side, we only care about my hostmap.
-	require.True(t, myControl.CloseTunnel(theirV4, true))
-
-	// The surviving sibling must still serve BOTH addresses.
-	survivorV4 := myControl.GetHostInfoByVpnAddr(theirV4, false)
-	survivorV6 := myControl.GetHostInfoByVpnAddr(theirV6, false)
-	require.NotNil(t, survivorV4, "v4 addr should still resolve to the surviving tunnel")
-	// Pre-fix this is nil: the second address was orphaned when the primary was deleted.
-	require.NotNil(t, survivorV6, "v6 addr was orphaned after deleting the primary (multi-vpnAddr delete bug)")
-	assert.Equal(t, survivorV4.LocalIndex, survivorV6.LocalIndex, "both addrs should promote to the same survivor")
-	assert.NotEqual(t, primaryV4.LocalIndex, survivorV4.LocalIndex, "a different hostinfo should now be primary")
-
-	r.RenderHostmaps("Final hostmaps", myControl, theirControl)
-
-	myControl.Stop()
-	theirControl.Stop()
-}

e2e/leak_test.go

@@ -18,10 +18,14 @@ import (
 // retry mechanism gives the wg.Wait()-driven goroutines a moment to drain
 // before failing the assertion.
 //
-// Intentionally NOT t.Parallel()'d: concurrent tests would have their own
+// IgnoreCurrent is necessary in the parallelized suite: other tests can
-// goroutines running and trip the assertion.
+// leave goroutines mid-shutdown when this one runs (Stop is async, the
+// wg.Wait() drain is not blocking on test return). We're checking that
+// *this* test's setup tears down cleanly, not that the whole suite is
+// idle at this moment. Intentionally NOT t.Parallel()'d for the same
+// reason — concurrent test goroutines would always show up.
 func TestNoGoroutineLeaks(t *testing.T) {
-	defer goleak.VerifyNone(t)
+	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
 
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
 	myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "me", "10.128.0.1/24", nil)

e2e/sshd_test.go

@@ -1,125 +0,0 @@
-//go:build e2e_testing
-// +build e2e_testing
-
-package e2e
-
-import (
-	"crypto/ed25519"
-	"crypto/rand"
-	"encoding/pem"
-	"net"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/cert_test"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	"golang.org/x/crypto/ssh"
-)
-
-func TestSSHDLifecycle(t *testing.T) {
-	// TestSSHDLifecycle exercises the in-process sshd through several config reloads and a Control.Stop.
-	ca, _, caKey, _ := cert_test.NewTestCaCert(
-		cert.Version1, cert.Curve_CURVE25519,
-		time.Now(), time.Now().Add(10*time.Minute),
-		nil, nil, []string{},
-	)
-
-	hostKeyPEM := generateSSHHostKey(t)
-	clientSigner, clientAuthKey := generateSSHClientKey(t)
-	sshdAddr := allocLoopbackPort(t)
-
-	overrides := m{
-		"sshd": m{
-			"enabled":  true,
-			"listen":   sshdAddr,
-			"host_key": hostKeyPEM,
-			"authorized_users": []m{{
-				"user": "tester",
-				"keys": []string{clientAuthKey},
-			}},
-		},
-	}
-	control, _, _, _ := newSimpleServer(cert.Version1, ca, caKey, "sshd-test", "10.222.0.1/24", overrides)
-	control.Start()
-	t.Cleanup(func() { control.Stop() })
-
-	// sshd binds in a goroutine after Start returns; wait for it.
-	require.Eventually(t, func() bool { return canDial(sshdAddr) }, 2*time.Second, 25*time.Millisecond,
-		"sshd never started listening")
-
-	for i := 1; i <= 3; i++ {
-		out := sshExecReload(t, sshdAddr, clientSigner)
-		assert.Contains(t, out, "Reloading config", "reload cycle %d", i)
-		require.Eventually(t, func() bool { return canDial(sshdAddr) }, 2*time.Second, 25*time.Millisecond,
-			"sshd not listening after reload cycle %d", i)
-	}
-
-	control.Stop()
-	require.Eventually(t, func() bool { return !canDial(sshdAddr) }, 2*time.Second, 25*time.Millisecond,
-		"sshd still listening after Control.Stop")
-}
-
-func canDial(addr string) bool {
-	c, err := net.DialTimeout("tcp", addr, 100*time.Millisecond)
-	if err != nil {
-		return false
-	}
-	_ = c.Close()
-	return true
-}
-
-// allocLoopbackPort grabs an unused TCP port on 127.0.0.1, closes it, and returns the address. There
-// is a small race between releasing the port and the sshd reclaiming it; in practice the OS keeps the
-// port available long enough for the test to bind it.
-func allocLoopbackPort(t *testing.T) string {
-	t.Helper()
-	l, err := net.Listen("tcp", "127.0.0.1:0")
-	require.NoError(t, err)
-	addr := l.Addr().String()
-	require.NoError(t, l.Close())
-	return addr
-}
-
-func generateSSHHostKey(t *testing.T) string {
-	t.Helper()
-	_, priv, err := ed25519.GenerateKey(rand.Reader)
-	require.NoError(t, err)
-	block, err := ssh.MarshalPrivateKey(priv, "nebula-e2e-host")
-	require.NoError(t, err)
-	return string(pem.EncodeToMemory(block))
-}
-
-func generateSSHClientKey(t *testing.T) (ssh.Signer, string) {
-	t.Helper()
-	_, priv, err := ed25519.GenerateKey(rand.Reader)
-	require.NoError(t, err)
-	signer, err := ssh.NewSignerFromKey(priv)
-	require.NoError(t, err)
-	auth := strings.TrimSpace(string(ssh.MarshalAuthorizedKey(signer.PublicKey())))
-	return signer, auth
-}
-
-func sshExecReload(t *testing.T, addr string, signer ssh.Signer) string {
-	t.Helper()
-	cfg := &ssh.ClientConfig{
-		User:            "tester",
-		Auth:            []ssh.AuthMethod{ssh.PublicKeys(signer)},
-		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
-		Timeout:         2 * time.Second,
-	}
-	client, err := ssh.Dial("tcp", addr, cfg)
-	require.NoError(t, err)
-	defer client.Close()
-
-	sess, err := client.NewSession()
-	require.NoError(t, err)
-	defer sess.Close()
-
-	// reload tears the channel down before sending exit-status, so Output returns an error on the
-	// channel close. The output buffer still has whatever the reload callback wrote before that.
-	out, _ := sess.Output("reload")
-	return string(out)
-}

e2e/tunnels_test.go

@@ -15,7 +15,6 @@ import (
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/udp"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 	"gopkg.in/yaml.v3"
 )
 
@@ -374,100 +373,6 @@ func TestCrossStackRelaysWork(t *testing.T) {
 	//relayControl.Stop()
 }
 
-// TestRelayReplayProtection asserts that a relay (forwarding-type) node rejects
-// replayed relay frames. A captured relay frame, re-injected with the same
-// message counter, must be dropped by the replay window rather than re-forwarded
-// to the relay target. Before the fix, handleOutsideRelayPacket authenticated the
-// frame but never advanced the replay window, so every replay was re-forwarded.
-func TestRelayReplayProtection(t *testing.T) {
-	t.Parallel()
-	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
-	myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me     ", "10.128.0.1/24,fc00::1/64", m{"relay": m{"use_relays": true}})
-	relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "relay  ", "10.128.0.128/24,fc00::128/64", m{"relay": m{"am_relay": true}})
-	theirUdp := netip.MustParseAddrPort("10.0.0.2:4242")
-	theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServerWithUdp(cert.Version2, ca, caKey, "them   ", "fc00::2/64", theirUdp, m{"relay": m{"use_relays": true}})
-
-	myVpnV6 := myVpnIpNet[1]
-	relayVpnV4 := relayVpnIpNet[0]
-	relayVpnV6 := relayVpnIpNet[1]
-	theirVpnV6 := theirVpnIpNet[0]
-
-	// Teach me how to reach the relay and that them is reachable via the relay
-	myControl.InjectLightHouseAddr(relayVpnV4.Addr(), relayUdpAddr)
-	myControl.InjectLightHouseAddr(relayVpnV6.Addr(), relayUdpAddr)
-	myControl.InjectRelays(theirVpnV6.Addr(), []netip.Addr{relayVpnV6.Addr()})
-	relayControl.InjectLightHouseAddr(theirVpnV6.Addr(), theirUdpAddr)
-
-	r := router.NewR(t, myControl, relayControl, theirControl)
-	defer r.RenderFlow()
-
-	myControl.Start()
-	relayControl.Start()
-	theirControl.Start()
-
-	// Establish the relayed tunnel in both directions so all handshakes complete.
-	t.Log("Establish the relayed tunnel")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnV6.Addr(), 80, myVpnV6.Addr(), 80, []byte("Hi from me")))
-	p := r.RouteForAllUntilTxTun(theirControl)
-	assertUdpPacket(t, []byte("Hi from me"), p, myVpnV6.Addr(), theirVpnV6.Addr(), 80, 80)
-	theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnV6.Addr(), 80, theirVpnV6.Addr(), 80, []byte("Hi from them")))
-	p = r.RouteForAllUntilTxTun(myControl)
-	assertUdpPacket(t, []byte("Hi from them"), p, theirVpnV6.Addr(), myVpnV6.Addr(), 80, 80)
-
-	// Drain anything still queued on me's UDP tx so the next packet we pull is the
-	// relay frame we are about to generate.
-	for myControl.GetFromUDP(false) != nil {
-	}
-
-	// Capture a single legitimate relay frame that me transmits toward the relay.
-	t.Log("Capture a relay frame from me -> relay")
-	myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnV6.Addr(), 80, myVpnV6.Addr(), 80, []byte("replay me")))
-	relayFrame := myControl.GetFromUDP(true)
-	require.Equal(t, relayUdpAddr, relayFrame.To, "captured frame should be addressed to the relay")
-	var fh header.H
-	require.NoError(t, fh.Parse(relayFrame.Data))
-	require.Equal(t, header.Message, fh.Type)
-	require.Equal(t, header.MessageRelay, fh.Subtype)
-
-	// drainForwards counts relay frames the relay forwards toward them within the
-	// settle window. We match on destination + (Message, MessageRelay) so the
-	// relay's own direct traffic to them can't be miscounted.
-	drainForwards := func(settle time.Duration) int {
-		ch := relayControl.GetUDPTxChan()
-		count := 0
-		for {
-			select {
-			case pkt := <-ch:
-				var ph header.H
-				if pkt.To == theirUdpAddr && ph.Parse(pkt.Data) == nil &&
-					ph.Type == header.Message && ph.Subtype == header.MessageRelay {
-					count++
-				}
-				pkt.Release()
-			case <-time.After(settle):
-				return count
-			}
-		}
-	}
-
-	// First delivery of the captured frame: the relay should forward it once.
-	t.Log("Deliver the captured frame once; relay forwards it to them")
-	relayControl.InjectUDPPacket(relayFrame)
-	require.Equal(t, 1, drainForwards(200*time.Millisecond), "relay should forward the first, legitimate copy")
-
-	// Replay the exact same frame several times. A correct replay window rejects
-	// these duplicates so the relay forwards none of them.
-	t.Log("Replay the captured frame; relay must drop the duplicates")
-	const replays = 3
-	for i := 0; i < replays; i++ {
-		relayControl.InjectUDPPacket(relayFrame)
-	}
-	forwarded := drainForwards(200 * time.Millisecond)
-	assert.Equal(t, 0, forwarded, "relay re-forwarded %d/%d replayed relay frames; replay protection is ineffective on relay tunnels", forwarded, replays)
-
-	r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
-}
-
 func TestCloseTunnelAuthenticated(t *testing.T) {
 	t.Parallel()
 	ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})

examples/config.yml

@@ -138,14 +138,6 @@ listen:
   # max, net.core.rmem_max and net.core.wmem_max
   #read_buffer: 10485760
   #write_buffer: 10485760
-
-  # On Windows only
-  # When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port.
-  # WFP sits below Windows Defender Firewall, so this lets peer handshakes reach Nebula's outside socket regardless
-  # of WDF's inbound rules.
-  # Default true; set to false to leave WDF in charge of inbound decisions on the listener port. Not reloadable.
-  #windows_bypass_wdf: true
-
   # By default, Nebula replies to packets it has no tunnel for with a "recv_error" packet. This packet helps speed up reconnection
   # in the case that Nebula on either side did not shut down cleanly. This response can be abused as a way to discover if Nebula is running
   # on a host though. This option lets you configure if you want to send "recv_error" packets always, never, or only to private network remotes.
@@ -171,21 +163,17 @@ listen:
 
 punchy:
   # Continues to punch inbound/outbound at a regular interval to avoid expiration of firewall nat mappings
-  # This setting is reloadable.
   punch: true
 
   # respond means that a node you are trying to reach will connect back out to you if your hole punching fails
   # this is extremely useful if one node is behind a difficult nat, such as a symmetric NAT
   # Default is false
-  # This setting is reloadable.
   #respond: true
 
   # delays a punch response for misbehaving NATs, default is 1 second.
-  # This setting is reloadable.
   #delay: 1s
 
   # set the delay before attempting punchy.respond. Default is 5 seconds. respond must be true to take effect.
-  # This setting is reloadable.
   #respond_delay: 5s
 
 # Cipher allows you to choose between the available ciphers for your network. Options are chachapoly or aes
@@ -242,10 +230,6 @@ tun:
   # When tun is disabled, a lighthouse can be started without a local tun interface (and therefore without root)
   disabled: false
   # Name of the device. If not set, a default will be chosen by the OS.
-  # For Linux: a single `%d` anywhere in the name is treated as a template and replaced with the
-  #   lowest number that yields an unused device name (e.g. `nebula%d` becomes `nebula0`, then `nebula1`, and so on, `neb%dprod` becomes `neb0prod`).
-  #   Only on Linux: `nebula%d` is the default if tun.dev is unset.
-  #   The name, both before and after %d substitution, must be shorter than the kernel limit of 16 characters.
   # For macOS: if set, must be in the form `utun[0-9]+`.
   # For NetBSD: Required to be set, must be in the form `tun[0-9]+`
   dev: nebula1
@@ -298,24 +282,6 @@ tun:
     #  metric: 100
     #  install: true
 
-  # On Windows only, sets the network category of the nebula interface. Without this, Windows often
-  # leaves the network as "Unidentified" and treats it as Public, which makes the host firewall more
-  # restrictive than you usually want for an overlay between trusted peers. Valid values:
-  #   private             - treat the nebula network as a private/trusted network (default)
-  #   public              - treat it as a public/untrusted network
-  #   domain              - treat it as a domain-authenticated network
-  #   unset               - leave whatever Windows decided alone
-  # Not reloadable.
-  #network_category: private
-
-  # On Windows only
-  # When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to the nebula adapter LUID.
-  # WFP sits below Windows Defender Firewall, so this lets inbound traffic through regardless of WDF rules.
-  # Filters are auto-removed when the adapter goes away.
-  # See listen.windows_bypass_wdf for the matching control over inbound to nebula's outside UDP listener.
-  # Default true; set to false to leave WDF in charge of inbound decisions on the nebula interface. Not reloadable.
-  #windows_bypass_wdf: true
-
   # On linux only, set to true to manage unsafe routes directly on the system route table with gateway routes instead of
   # in nebula configuration files. Default false, not reloadable.
   #use_system_route_table: false
@@ -401,7 +367,7 @@ firewall:
   #   `drop` (default): silently drop the packet.
   #   `reject`: send a reject reply.
   #     - For TCP, this will be a RST "Connection Reset" packet.
-  #     - For other protocols, this will be an ICMP "Destination unreachable: Communication administratively prohibited" packet.
+  #     - For other protocols, this will be an ICMP port unreachable packet.
   outbound_action: drop
   inbound_action: drop
 

firewall.go

@@ -58,9 +58,8 @@ type Firewall struct {
 	routableNetworks *bart.Lite
 
 	// assignedNetworks is a list of vpn networks assigned to us in the certificate.
-	assignedNetworks []netip.Prefix
+	assignedNetworks  []netip.Prefix
-	// unsafeNetworks is the list of unsafe networks issued to us in the certificate
+	hasUnsafeNetworks bool
-	unsafeNetworks []netip.Prefix
 
 	rules        string
 	rulesVersion uint16
@@ -159,9 +158,10 @@ func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Dur
 		assignedNetworks = append(assignedNetworks, network)
 	}
 
-	unsafeNetworks := c.UnsafeNetworks()
+	hasUnsafeNetworks := false
-	for _, n := range unsafeNetworks {
+	for _, n := range c.UnsafeNetworks() {
 		routableNetworks.Insert(n)
+		hasUnsafeNetworks = true
 	}
 
 	return &Firewall{
@@ -169,15 +169,15 @@ func NewFirewall(l *slog.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Dur
 			Conns:      make(map[firewall.Packet]*conn),
 			TimerWheel: NewTimerWheel[firewall.Packet](tmin, tmax),
 		},
-		InRules:          newFirewallTable(),
+		InRules:           newFirewallTable(),
-		OutRules:         newFirewallTable(),
+		OutRules:          newFirewallTable(),
-		TCPTimeout:       tcpTimeout,
+		TCPTimeout:        tcpTimeout,
-		UDPTimeout:       UDPTimeout,
+		UDPTimeout:        UDPTimeout,
-		DefaultTimeout:   defaultTimeout,
+		DefaultTimeout:    defaultTimeout,
-		routableNetworks: routableNetworks,
+		routableNetworks:  routableNetworks,
-		assignedNetworks: assignedNetworks,
+		assignedNetworks:  assignedNetworks,
-		unsafeNetworks:   unsafeNetworks,
+		hasUnsafeNetworks: hasUnsafeNetworks,
-		l:                l,
+		l:                 l,
 
 		incomingMetrics: firewallMetrics{
 			droppedLocalAddr:  metrics.GetOrRegisterCounter("firewall.incoming.dropped.local_addr", nil),
@@ -423,6 +423,11 @@ var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
 // Drop returns an error if the packet should be dropped, explaining why. It
 // returns nil if the packet should not be dropped.
 func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.CAPool, localCache firewall.ConntrackCache) error {
+	// Check if we spoke to this tuple, if we did then allow this packet
+	if f.inConns(fp, h, caPool, localCache) {
+		return nil
+	}
+
 	// Make sure remote address matches nebula certificate, and determine how to treat it
 	if h.networks == nil {
 		// Simple case: Certificate has one address and no unsafe networks
@@ -456,11 +461,6 @@ func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *
 		return ErrInvalidLocalIP
 	}
 
-	// Check if we spoke to this tuple, if we did then allow this packet
-	if f.inConns(fp, h, caPool, localCache) {
-		return nil
-	}
-
 	table := f.OutRules
 	if incoming {
 		table = f.InRules
@@ -897,7 +897,7 @@ func (flc *firewallLocalCIDR) addRule(f *Firewall, localCidr string) error {
 	}
 
 	if localCidr == "" {
-		if len(f.unsafeNetworks) == 0 || f.defaultLocalCIDRAny {
+		if !f.hasUnsafeNetworks || f.defaultLocalCIDRAny {
 			flc.Any = true
 			return nil
 		}
@@ -1055,6 +1055,7 @@ func (r *rule) sanity() error {
 }
 
 func parsePort(s string) (int32, int32, error) {
+	var err error
 	const notAPort int32 = -2
 	if s == "any" {
 		return firewall.PortAny, firewall.PortAny, nil
@@ -1063,11 +1064,11 @@ func parsePort(s string) (int32, int32, error) {
 		return firewall.PortFragment, firewall.PortFragment, nil
 	}
 	if !strings.Contains(s, `-`) {
-		rPort, err := parsePortValue("", s)
+		rPort, err := strconv.Atoi(s)
 		if err != nil {
-			return notAPort, notAPort, err
+			return notAPort, notAPort, fmt.Errorf("was not a number; `%s`", s)
 		}
-		return rPort, rPort, nil
+		return int32(rPort), int32(rPort), nil
 	}
 
 	sPorts := strings.SplitN(s, `-`, 2)
@@ -1078,40 +1079,22 @@ func parsePort(s string) (int32, int32, error) {
 		return notAPort, notAPort, fmt.Errorf("appears to be a range but could not be parsed; `%s`", s)
 	}
 
-	startPort, err := parsePortValue("beginning range ", sPorts[0])
+	rStartPort, err := strconv.Atoi(sPorts[0])
 	if err != nil {
-		return notAPort, notAPort, err
+		return notAPort, notAPort, fmt.Errorf("beginning range was not a number; `%s`", sPorts[0])
 	}
 
-	endPort, err := parsePortValue("ending range ", sPorts[1])
+	rEndPort, err := strconv.Atoi(sPorts[1])
 	if err != nil {
-		return notAPort, notAPort, err
+		return notAPort, notAPort, fmt.Errorf("ending range was not a number; `%s`", sPorts[1])
 	}
 
+	startPort := int32(rStartPort)
+	endPort := int32(rEndPort)
+
 	if startPort == firewall.PortAny {
 		endPort = firewall.PortAny
 	}
 
 	return startPort, endPort, nil
 }
-
-// parsePortValue accepts a base-10 decimal in [0, 65535] and returns it
-// widened to int32. Using strconv.ParseUint with bitSize 16 rejects
-// negative input, out-of-range input (>65535), and any non-decimal byte
-// by construction, so the int32 widening that follows is provably safe
-// and cannot collide with firewall.PortAny (0) or firewall.PortFragment
-// (-1) via integer truncation.
-//
-// prefix is prepended to both error messages so callers can disambiguate
-// the single-port path (prefix="") from the range bounds (prefix="beginning
-// range " / "ending range "), preserving the historical error strings.
-func parsePortValue(prefix, s string) (int32, error) {
-	n, err := strconv.ParseUint(s, 10, 16)
-	if err == nil {
-		return int32(n), nil
-	}
-	if errors.Is(err, strconv.ErrRange) {
-		return 0, fmt.Errorf("%sout of range [0,65535]; `%s`", prefix, s)
-	}
-	return 0, fmt.Errorf("%swas not a number; `%s`", prefix, s)
-}

firewall_test.go

@@ -916,159 +916,6 @@ func TestFirewall_DropIPSpoofing(t *testing.T) {
 	assert.Equal(t, fw.Drop(p, true, &h1, cp, nil), ErrInvalidRemoteIP)
 }
 
-func TestFirewall_ConntrackSourceSpoofingAcrossPeers(t *testing.T) {
-	l := test.NewLoggerWithOutput(&bytes.Buffer{})
-
-	myVpnNetworksTable := new(bart.Lite)
-	myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.1/24"))
-
-	owner := &dummyCert{
-		name:     "owner",
-		networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")},
-	}
-
-	victim := &cert.CachedCertificate{
-		Certificate: &dummyCert{
-			name:     "victim",
-			networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.2/24")},
-		},
-	}
-	victimHI := HostInfo{
-		ConnectionState: &ConnectionState{peerCert: victim},
-		vpnAddrs:        []netip.Addr{netip.MustParseAddr("192.0.2.2")},
-	}
-	victimHI.buildNetworks(myVpnNetworksTable, victim.Certificate)
-
-	attacker := &cert.CachedCertificate{
-		Certificate: &dummyCert{
-			name:     "attacker",
-			networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.3/24")},
-		},
-	}
-	attackerHI := HostInfo{
-		ConnectionState: &ConnectionState{peerCert: attacker},
-		vpnAddrs:        []netip.Addr{netip.MustParseAddr("192.0.2.3")},
-	}
-	attackerHI.buildNetworks(myVpnNetworksTable, attacker.Certificate)
-
-	fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner)
-	// Allow any inbound traffic that passes the cert / source-IP checks.
-	require.NoError(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
-	cp := cert.NewCAPool()
-
-	flow := firewall.Packet{
-		LocalAddr:  netip.MustParseAddr("192.0.2.1"),
-		RemoteAddr: netip.MustParseAddr("192.0.2.2"),
-		LocalPort:  443,
-		RemotePort: 55000,
-		Protocol:   firewall.ProtoUDP,
-	}
-
-	require.NoError(t, fw.Drop(flow, true, &victimHI, cp, nil),
-		"victim's own traffic from its own overlay IP must be allowed")
-
-	unseen := flow
-	unseen.RemotePort = 55001
-	assert.Equal(t, ErrInvalidRemoteIP, fw.Drop(unseen, true, &attackerHI, cp, nil),
-		"sanity: attacker forging victim's source IP must be rejected when no conntrack entry exists")
-
-	got := fw.Drop(flow, true, &attackerHI, cp, nil)
-	t.Logf("attacker replaying victim's 4-tuple: Drop returned %v (nil == packet ALLOWED == spoof succeeded)", got)
-	assert.Equal(t, ErrInvalidRemoteIP, got,
-		"SECURITY: attacker spoofed victim's overlay source IP (192.0.2.2) by reusing an existing conntrack 4-tuple; Drop returned %v instead of rejecting", got)
-}
-
-// BenchmarkFirewallDropConntrackHit measures Drop on an already-established flow
-// (a conntrack hit). This is the fast path that the source-IP<->cert binding
-// reordering adds work to, so it quantifies the cost of moving the address checks
-// ahead of the conntrack lookup. Cases:
-//   - simple:  peer cert has one address, no unsafe networks (h.networks == nil),
-//     so the remote-address check is a single netip.Addr compare.
-//   - complex: peer cert has unsafe networks (h.networks populated), so the
-//     remote-address check is a BART lookup.
-//   - noCache/localCache: whether a per-batch ConntrackCache is supplied, which in
-//     the original code let the fast path skip straight past the address checks.
-func BenchmarkFirewallDropConntrackHit(b *testing.B) {
-	l := test.NewLoggerWithOutput(&bytes.Buffer{})
-
-	myVpnNetworksTable := new(bart.Lite)
-	myVpnNetworksTable.Insert(netip.MustParsePrefix("192.0.2.1/24"))
-
-	owner := &dummyCert{
-		name:     "owner",
-		networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.1/24")},
-	}
-
-	simpleCert := &cert.CachedCertificate{
-		Certificate: &dummyCert{
-			name:     "simple",
-			networks: []netip.Prefix{netip.MustParsePrefix("192.0.2.2/24")},
-		},
-	}
-	simpleHost := &HostInfo{
-		ConnectionState: &ConnectionState{peerCert: simpleCert},
-		vpnAddrs:        []netip.Addr{netip.MustParseAddr("192.0.2.2")},
-	}
-	simpleHost.buildNetworks(myVpnNetworksTable, simpleCert.Certificate)
-
-	complexCert := &cert.CachedCertificate{
-		Certificate: &dummyCert{
-			name:           "complex",
-			networks:       []netip.Prefix{netip.MustParsePrefix("192.0.2.2/24")},
-			unsafeNetworks: []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")},
-		},
-	}
-	complexHost := &HostInfo{
-		ConnectionState: &ConnectionState{peerCert: complexCert},
-		vpnAddrs:        []netip.Addr{netip.MustParseAddr("192.0.2.2")},
-	}
-	complexHost.buildNetworks(myVpnNetworksTable, complexCert.Certificate)
-
-	cp := cert.NewCAPool()
-
-	flow := firewall.Packet{
-		LocalAddr:  netip.MustParseAddr("192.0.2.1"),
-		RemoteAddr: netip.MustParseAddr("192.0.2.2"),
-		LocalPort:  443,
-		RemotePort: 55000,
-		Protocol:   firewall.ProtoUDP,
-	}
-
-	cases := []struct {
-		name     string
-		host     *HostInfo
-		useCache bool
-	}{
-		{"simple/noCache", simpleHost, false},
-		{"simple/localCache", simpleHost, true},
-		{"complex/noCache", complexHost, false},
-		{"complex/localCache", complexHost, true},
-	}
-
-	for _, tc := range cases {
-		b.Run(tc.name, func(b *testing.B) {
-			fw := NewFirewall(l, time.Second, time.Minute, time.Hour, owner)
-			require.NoError(b, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"any"}, "", "", "", "", ""))
-
-			// Establish the conntrack entry so every benchmarked Drop is a hit.
-			require.NoError(b, fw.Drop(flow, true, tc.host, cp, nil))
-
-			var cache firewall.ConntrackCache
-			if tc.useCache {
-				cache = firewall.ConntrackCache{}
-			}
-
-			b.ReportAllocs()
-			b.ResetTimer()
-			for i := 0; i < b.N; i++ {
-				if err := fw.Drop(flow, true, tc.host, cp, cache); err != nil {
-					b.Fatal(err)
-				}
-			}
-		})
-	}
-}
-
 func BenchmarkLookup(b *testing.B) {
 	ml := func(m map[string]struct{}, a [][]string) {
 		for n := 0; n < b.N; n++ {
@@ -1182,75 +1029,6 @@ func Test_parsePort(t *testing.T) {
 	require.NoError(t, err)
 }
 
-// Test_parsePort_invalid covers inputs that must error. The named bug is
-// that int32(strconv.Atoi("4294967296")) truncates to 0 == firewall.PortAny,
-// silently turning a typo into a match-all-ports rule; the rest are
-// representative syntax/range probes.
-func Test_parsePort_invalid(t *testing.T) {
-	tests := []struct {
-		name            string
-		input           string
-		wantErrContains string
-	}{
-		// Numeric overflow (the named bug + boundary).
-		{"named bug: 2^32 truncates to PortAny", "4294967296", "out of range"},
-		{"just above max real port", "65536", "out of range"},
-
-		// Negatives route through the range branch and hit the empty-half
-		// guard; included as defense in depth so a future refactor cannot
-		// accidentally reach the int32 cast.
-		{"negative", "-1", "could not be parsed"},
-
-		// Syntax probes.
-		{"NUL between digits", "4\x002", "was not a number"},
-		{"hex notation", "0x10", "was not a number"},
-		{"scientific notation", "1e3", "was not a number"},
-		{"leading whitespace", " 42", "was not a number"},
-		{"fullwidth digits", "４２", "was not a number"},
-
-		// Range branch.
-		{"range upper out of range", "1-65536", "ending range out of range"},
-		{"range lower out of range", "65536-65537", "beginning range out of range"},
-		{"range with negative upper", "1--1", "ending range was not a number"},
-	}
-
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			_, _, err := parsePort(tc.input)
-			require.Error(t, err, "input %q must error", tc.input)
-			require.ErrorContains(t, err, tc.wantErrContains)
-		})
-	}
-}
-
-// Test_parsePort_valid_boundaries locks in success cases at 0, 1, and 65535
-// so a future refactor cannot regress the boundaries.
-func Test_parsePort_valid_boundaries(t *testing.T) {
-	tests := []struct {
-		name      string
-		input     string
-		wantStart int32
-		wantEnd   int32
-	}{
-		{"zero is PortAny", "0", 0, 0},
-		{"min real port", "1", 1, 1},
-		{"max real port", "65535", 65535, 65535},
-		{"range zero to max forces end to zero", "0-65535", 0, 0},
-		{"range max to max", "65535-65535", 65535, 65535},
-		{"range one to max", "1-65535", 1, 65535},
-		{"range with whitespace inside", " 1 - 2    ", 1, 2},
-	}
-
-	for _, tc := range tests {
-		t.Run(tc.name, func(t *testing.T) {
-			s, e, err := parsePort(tc.input)
-			require.NoError(t, err)
-			assert.Equal(t, tc.wantStart, s, "start port")
-			assert.Equal(t, tc.wantEnd, e, "end port")
-		})
-	}
-}
-
 func TestNewFirewallFromConfig(t *testing.T) {
 	l := test.NewLogger()
 	// Test a bad rule definition

go.mod

@@ -9,10 +9,10 @@ require (
 	github.com/armon/go-radix v1.0.0
 	github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432
 	github.com/flynn/noise v1.1.0
-	github.com/gaissmai/bart v0.28.0
+	github.com/gaissmai/bart v0.26.0
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/gopacket v1.1.19
-	github.com/kardianos/service v1.3.0
+	github.com/kardianos/service v1.2.4
 	github.com/miekg/dns v1.1.72
 	github.com/miekg/pkcs11 v1.1.2
 	github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f
@@ -24,15 +24,15 @@ require (
 	github.com/vishvananda/netlink v1.3.1
 	go.uber.org/goleak v1.3.0
 	go.yaml.in/yaml/v3 v3.0.4
-	golang.org/x/crypto v0.53.0
+	golang.org/x/crypto v0.50.0
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
-	golang.org/x/net v0.56.0
+	golang.org/x/net v0.52.0
-	golang.org/x/sync v0.21.0
+	golang.org/x/sync v0.20.0
-	golang.org/x/sys v0.46.0
+	golang.org/x/sys v0.43.0
-	golang.org/x/term v0.44.0
+	golang.org/x/term v0.42.0
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
 	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
-	golang.zx2c4.com/wireguard/windows v1.0.1
+	golang.zx2c4.com/wireguard/windows v0.6.1
 	google.golang.org/protobuf v1.36.11
 	gopkg.in/yaml.v3 v3.0.1
 	gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
@@ -50,7 +50,7 @@ require (
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/vishvananda/netns v0.0.5 // indirect
 	go.yaml.in/yaml/v2 v2.4.2 // indirect
-	golang.org/x/mod v0.36.0 // indirect
+	golang.org/x/mod v0.34.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.45.0 // indirect
+	golang.org/x/tools v0.43.0 // indirect
 )

go.sum

@@ -26,8 +26,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg=
 github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
-github.com/gaissmai/bart v0.28.0 h1:89yZLo8NmyqD0RYgJ3QO9HhqqGGw+oWhf90cZm69Lko=
+github.com/gaissmai/bart v0.26.0 h1:xOZ57E9hJLBiQaSyeZa9wgWhGuzfGACgqp4BE77OkO0=
-github.com/gaissmai/bart v0.28.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
+github.com/gaissmai/bart v0.26.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -66,8 +66,8 @@ github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/
 github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
-github.com/kardianos/service v1.3.0 h1:/LGy+xPP2TM+GLTiCZ2di7cy0Jd/qrawlTUfqKYFdTI=
+github.com/kardianos/service v1.2.4 h1:XNlGtZOYNx2u91urOdg/Kfmc+gfmuIo1Dd3rEi2OgBk=
-github.com/kardianos/service v1.3.0/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc=
+github.com/kardianos/service v1.2.4/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
@@ -162,16 +162,16 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
+golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
-golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
+golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
+golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
-golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
+golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -182,8 +182,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
+golang.org/x/net v0.52.0 h1:He/TN1l0e4mmR3QqHMT2Xab3Aj3L9qjbhRm78/6jrW0=
-golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
+golang.org/x/net v0.52.0/go.mod h1:R1MAz7uMZxVMualyPXb+VaqGSa3LIaUqk0eEt3w36Sw=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -191,8 +191,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
+golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
-golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
+golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -208,11 +208,11 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
+golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI=
-golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
+golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
-golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
+golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -223,8 +223,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
+golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
-golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
+golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -233,8 +233,8 @@ golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeu
 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI=
 golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b h1:J1CaxgLerRR5lgx3wnr6L04cJFbWoceSK9JWBdglINo=
 golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b/go.mod h1:tqur9LnfstdR9ep2LaJT4lFUl0EjlHtge+gAjmsHUG4=
-golang.zx2c4.com/wireguard/windows v1.0.1 h1:eOxiDVbywPC+ZQqvdCK7x+ZwWXKbYv50TtH8ysFIbw8=
+golang.zx2c4.com/wireguard/windows v0.6.1 h1:XMaKojH1Hs/raMrmnir4n35nTvzvWj7NmSYzHn2F4qU=
-golang.zx2c4.com/wireguard/windows v1.0.1/go.mod h1:+fbT3FFdX4zzYDLwJh5+HPEcNN/3HyNdzhNSVsQM+zs=
+golang.zx2c4.com/wireguard/windows v0.6.1/go.mod h1:04aqInu5GYuTFvMuDw/rKBAF7mHrltW/3rekpfbbZDM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=

handshake/errors.go

@@ -13,7 +13,6 @@ var (
 	ErrUnknownSubtype          = errors.New("unknown handshake subtype")
 	ErrMissingContent          = errors.New("expected handshake content but message was empty")
 	ErrUnexpectedContent       = errors.New("received unexpected handshake content")
-	ErrInvalidRemoteIndex      = errors.New("peer sent an invalid index in handshake payload")
 	ErrIndexAllocation         = errors.New("failed to allocate local index")
 	ErrNoCredential            = errors.New("no handshake credential available for cert version")
 	ErrAsymmetricCipherKeys    = errors.New("noise produced only one cipher key")

handshake/machine.go

@@ -31,7 +31,6 @@ type CertVerifier func(cert.Certificate) (*cert.CachedCertificate, error)
 type Result struct {
 	EKey          *noise.CipherState
 	DKey          *noise.CipherState
-	Cipher        noise.CipherFunc // identifies which post-handshake CipherState the data plane should wrap EKey/DKey in
 	MyCert        cert.Certificate
 	RemoteCert    *cert.CachedCertificate
 	RemoteIndex   uint32
@@ -106,7 +105,6 @@ func NewMachine(
 		myVersion:  version,
 		result: &Result{
 			Initiator: initiator,
-			Cipher:    cred.cipherSuite,
 		},
 	}, nil
 }
@@ -312,19 +310,11 @@ func (m *Machine) processPayload(msg []byte, flags msgFlags) error {
 
 	// Process payload
 	if flags.expectsPayload {
-		var remoteIndex uint32
 		if m.result.Initiator {
-			remoteIndex = payload.ResponderIndex
+			m.result.RemoteIndex = payload.ResponderIndex
 		} else {
-			remoteIndex = payload.InitiatorIndex
+			m.result.RemoteIndex = payload.InitiatorIndex
 		}
-		// The payload presence check above can be satisfied by Time alone, so a payload
-		// could still carry a zero index here. We need to reject it.
-		if remoteIndex == 0 {
-			m.failed = true
-			return ErrInvalidRemoteIndex
-		}
-		m.result.RemoteIndex = remoteIndex
 		m.result.HandshakeTime = payload.Time
 		m.payloadSet = true
 	}

handshake/machine_test.go

@@ -229,24 +229,6 @@ func TestMachineProcessPayload(t *testing.T) {
 		require.ErrorIs(t, err, ErrUnexpectedContent)
 		assert.True(t, m.Failed())
 	})
-
-	t.Run("zero initiator index on responder is fatal", func(t *testing.T) {
-		m := newTestMachine(t, cs, v, false, 100)
-		bytes := MarshalPayload(nil, Payload{InitiatorIndex: 0, Time: 1})
-		err := m.processPayload(bytes, msgFlags{expectsPayload: true})
-		require.ErrorIs(t, err, ErrInvalidRemoteIndex)
-		assert.True(t, m.Failed())
-		assert.Zero(t, m.result.RemoteIndex)
-	})
-
-	t.Run("zero responder index on initiator is fatal", func(t *testing.T) {
-		m := newTestMachine(t, cs, v, true, 100)
-		bytes := MarshalPayload(nil, Payload{InitiatorIndex: 100, ResponderIndex: 0, Time: 1})
-		err := m.processPayload(bytes, msgFlags{expectsPayload: true})
-		require.ErrorIs(t, err, ErrInvalidRemoteIndex)
-		assert.True(t, m.Failed())
-		assert.Zero(t, m.result.RemoteIndex)
-	})
 }
 
 // TestMachineRequireComplete checks the fail-on-incomplete-handshake path

handshake_manager.go

@@ -83,7 +83,6 @@ type HandshakeHostInfo struct {
 	initiatingVersionOverride cert.Version     // Should we use a non-default cert version for this handshake?
 	counter                   int64            // How many attempts have we made so far
 	lastRemotes               []netip.AddrPort // Remotes that we sent to during the previous attempt
-	lastRelays                []netip.Addr     // Relays we attempted to use during the previous attempt
 	packetStore               []*cachedPacket  // A set of packets to be transmitted once the handshake completes
 
 	hostinfo *HostInfo
@@ -218,6 +217,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 		fields := []any{
 			"udpAddrs", hh.hostinfo.remotes.CopyAddrs(hm.mainHostMap.GetPreferredRanges()),
 			"initiatorIndex", hh.hostinfo.localIndexId,
+			"remoteIndex", hh.hostinfo.remoteIndexId,
 			"durationNs", time.Since(hh.startTime).Nanoseconds(),
 		}
 		// hh.machine can be nil here if buildStage0Packet never succeeded
@@ -323,7 +323,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
 		)
 	}
 
-	hm.f.relayManager.StartRelays(hm.f, vpnIp, hh, stage0)
+	hm.f.relayManager.StartRelays(hm.f, vpnIp, hostinfo, stage0)
 
 	// If a lighthouse triggered this attempt then we are still in the timer wheel and do not need to re-add
 	if !lighthouseTriggered {
@@ -465,6 +465,7 @@ func (hm *HandshakeManager) CheckAndComplete(hostinfo *HostInfo, handshakePacket
 		// We have a collision, but this can happen since we can't control
 		// the remote ID. Just log about the situation as a note.
 		hostinfo.logger(hm.l).Info("New host shadows existing host remoteIndex",
+			"remoteIndex", hostinfo.remoteIndexId,
 			"collision", existingRemoteIndex.vpnAddrs,
 		)
 	}
@@ -487,6 +488,7 @@ func (hm *HandshakeManager) Complete(hostinfo *HostInfo, f *Interface) {
 		// We have a collision, but this can happen since we can't control
 		// the remote ID. Just log about the situation as a note.
 		hostinfo.logger(hm.l).Info("New host shadows existing host remoteIndex",
+			"remoteIndex", hostinfo.remoteIndexId,
 			"collision", existingRemoteIndex.vpnAddrs,
 		)
 	}
@@ -796,6 +798,7 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
 	}
 
 	hm.sendHandshakeResponse(via, response, hostinfo, false)
+	f.connectionManager.AddTrafficWatch(hostinfo)
 	hostinfo.remotes.RefreshFromHandshake(vpnAddrs)
 
 	// Don't wait for UpdateWorker
@@ -962,16 +965,17 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
 	hostinfo.buildNetworks(f.myVpnNetworksTable, remoteCert.Certificate)
 
 	hm.Complete(hostinfo, f)
+	f.connectionManager.AddTrafficWatch(hostinfo)
 
 	if len(hh.packetStore) > 0 {
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			hostinfo.logger(f.l).Debug("Sending stored packets", "count", len(hh.packetStore))
 		}
-		nb := make([]byte, 12, 12)
+		buf := f.bufAlloc.Acquire()
-		out := make([]byte, mtu)
 		for _, cp := range hh.packetStore {
-			cp.callback(cp.messageType, cp.messageSubType, hostinfo, cp.packet, nb, out)
+			cp.callback(cp.messageType, cp.messageSubType, hostinfo, cp.packet, buf)
 		}
+		f.bufAlloc.Release(buf)
 		f.cachedPacketMetrics.sent.Inc(int64(len(hh.packetStore)))
 	}
 
@@ -1081,7 +1085,9 @@ func (hm *HandshakeManager) sendHandshakeResponse(via ViaSender, msg []byte, hos
 		// We received a valid handshake on this relay, so make sure the relay
 		// state reflects that, in case it had been marked Disestablished.
 		via.relayHI.relayState.UpdateRelayForByIdxState(via.remoteIdx, Established)
-		f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false)
+		buf := f.bufAlloc.Acquire()
+		f.SendVia(via.relayHI, via.relay, msg, buf)
+		f.bufAlloc.Release(buf)
 		f.l.Info("Handshake message sent", append(logFields, "relay", via.relayHI.vpnAddrs[0])...)
 	}
 }
@@ -1098,7 +1104,9 @@ func (hm *HandshakeManager) handleCheckAndCompleteError(err error, existing, hos
 	switch err {
 	case ErrAlreadySeen:
 		if existing.SetRemoteIfPreferred(f.hostMap, via) {
-			f.SendMessageToVpnAddr(header.Test, header.TestRequest, hostinfo.vpnAddrs[0], []byte(""), make([]byte, 12, 12), make([]byte, mtu))
+			buf := f.bufAlloc.Acquire()
+			f.SendMessageToVpnAddr(header.Test, header.TestRequest, hostinfo.vpnAddrs[0], []byte(""), buf)
+			f.bufAlloc.Release(buf)
 		}
 		// Resend the original response. The peer is committed to that response's
 		// ephemeral keys; a freshly-built one would have different keys and break
@@ -1121,7 +1129,9 @@ func (hm *HandshakeManager) handleCheckAndCompleteError(err error, existing, hos
 			"responderIndex", hostinfo.localIndexId,
 			"handshake", hsFields,
 		)
-		f.SendMessageToVpnAddr(header.Test, header.TestRequest, hostinfo.vpnAddrs[0], []byte(""), make([]byte, 12, 12), make([]byte, mtu))
+		buf := f.bufAlloc.Acquire()
+		f.SendMessageToVpnAddr(header.Test, header.TestRequest, hostinfo.vpnAddrs[0], []byte(""), buf)
+		f.bufAlloc.Release(buf)
 
 	case ErrLocalIndexCollision:
 		f.l.Error("Failed to add HostInfo due to localIndex collision",

handshake_manager_test.go

@@ -80,15 +80,15 @@ func testCountTimerWheelEntries(tw *LockingTimerWheel[netip.Addr]) (c int) {
 type mockEncWriter struct {
 }
 
-func (mw *mockEncWriter) SendMessageToVpnAddr(_ header.MessageType, _ header.MessageSubType, _ netip.Addr, _, _, _ []byte) {
+func (mw *mockEncWriter) SendMessageToVpnAddr(_ header.MessageType, _ header.MessageSubType, _ netip.Addr, _ []byte, _ *WireBuffer) {
 	return
 }
 
-func (mw *mockEncWriter) SendVia(_ *HostInfo, _ *Relay, _, _, _ []byte, _ bool) {
+func (mw *mockEncWriter) SendVia(_ *HostInfo, _ *Relay, _ []byte, _ *WireBuffer) {
 	return
 }
 
-func (mw *mockEncWriter) SendMessageToHostInfo(_ header.MessageType, _ header.MessageSubType, _ *HostInfo, _, _, _ []byte) {
+func (mw *mockEncWriter) SendMessageToHostInfo(_ header.MessageType, _ header.MessageSubType, _ *HostInfo, _ []byte, _ *WireBuffer) {
 	return
 }
 

header/header.go

@@ -174,10 +174,6 @@ func (h *H) SubTypeName() string {
 	return SubTypeName(h.Type, h.Subtype)
 }
 
-func (h *H) IsValidSubType() bool {
-	return IsValidSubType(h.Type, h.Subtype)
-}
-
 // SubTypeName will transform a nebula message sub type into a human string
 func SubTypeName(t MessageType, s MessageSubType) string {
 	if n, ok := subTypeMap[t]; ok {
@@ -189,16 +185,6 @@ func SubTypeName(t MessageType, s MessageSubType) string {
 	return "unknown"
 }
 
-func IsValidSubType(t MessageType, s MessageSubType) bool {
-	if n, ok := subTypeMap[t]; ok {
-		if _, ok := (*n)[s]; ok {
-			return true
-		}
-	}
-
-	return false
-}
-
 // NewHeader turns bytes into a header
 func NewHeader(b []byte) (*H, error) {
 	h := new(H)

hostmap.go

@@ -138,9 +138,9 @@ func (rs *RelayState) InsertRelayTo(ip netip.Addr) {
 }
 
 func (rs *RelayState) CopyRelayIps() []netip.Addr {
+	ret := make([]netip.Addr, len(rs.relays))
 	rs.RLock()
 	defer rs.RUnlock()
-	ret := make([]netip.Addr, len(rs.relays))
 	copy(ret, rs.relays)
 	return ret
 }
@@ -229,7 +229,7 @@ const (
 )
 
 type HostInfo struct {
-	remote          atomic.Pointer[netip.AddrPort]
+	remote          netip.AddrPort
 	remotes         *RemoteList
 	promoteCounter  atomic.Uint32
 	ConnectionState *ConnectionState
@@ -308,7 +308,7 @@ type cachedPacket struct {
 	packet         []byte
 }
 
-type packetCallback func(t header.MessageType, st header.MessageSubType, h *HostInfo, p, nb, out []byte)
+type packetCallback func(t header.MessageType, st header.MessageSubType, h *HostInfo, p []byte, buf *WireBuffer)
 
 type cachedPacketMetrics struct {
 	sent    metrics.Counter
@@ -438,29 +438,43 @@ func (hm *HostMap) unlockedMakePrimary(hostinfo *HostInfo) {
 }
 
 func (hm *HostMap) unlockedDeleteHostInfo(hostinfo *HostInfo) {
-	isLastHostinfo := hostinfo.next == nil && hostinfo.prev == nil
-
 	for _, addr := range hostinfo.vpnAddrs {
-		if hm.Hosts[addr] != hostinfo {
+		h := hm.Hosts[addr]
-			continue
+		for h != nil {
-		}
+			if h == hostinfo {
-		if hostinfo.next != nil {
+				hm.unlockedInnerDeleteHostInfo(h, addr)
-			// Promote the next hostinfo in the shared chain to primary for this address
+			}
-			hm.Hosts[addr] = hostinfo.next
+			h = h.next
-		} else {
-			delete(hm.Hosts, addr)
 		}
 	}
-	if len(hm.Hosts) == 0 {
+}
-		hm.Hosts = map[netip.Addr]*HostInfo{}
-	}
 
-	// Splice this hostinfo out of the shared chain exactly once
+func (hm *HostMap) unlockedInnerDeleteHostInfo(hostinfo *HostInfo, addr netip.Addr) {
-	if hostinfo.prev != nil {
+	primary, ok := hm.Hosts[addr]
-		hostinfo.prev.next = hostinfo.next
+	isLastHostinfo := hostinfo.next == nil && hostinfo.prev == nil
-	}
+	if ok && primary == hostinfo {
-	if hostinfo.next != nil {
+		// The vpn addr pointer points to the same hostinfo as the local index id, we can remove it
-		hostinfo.next.prev = hostinfo.prev
+		delete(hm.Hosts, addr)
+		if len(hm.Hosts) == 0 {
+			hm.Hosts = map[netip.Addr]*HostInfo{}
+		}
+
+		if hostinfo.next != nil {
+			// We had more than 1 hostinfo at this vpn addr, promote the next in the list to primary
+			hm.Hosts[addr] = hostinfo.next
+			// It is primary, there is no previous hostinfo now
+			hostinfo.next.prev = nil
+		}
+
+	} else {
+		// Relink if we were in the middle of multiple hostinfos for this vpn addr
+		if hostinfo.prev != nil {
+			hostinfo.prev.next = hostinfo.next
+		}
+
+		if hostinfo.next != nil {
+			hostinfo.next.prev = hostinfo.prev
+		}
 	}
 
 	hostinfo.next = nil
@@ -609,11 +623,6 @@ func (hm *HostMap) unlockedAddHostInfo(hostinfo *HostInfo, f *Interface) {
 	hm.Indexes[hostinfo.localIndexId] = hostinfo
 	hm.RemoteIndexes[hostinfo.remoteIndexId] = hostinfo
 
-	hostinfo.out.Store(true)
-	if f.connectionManager != nil { // f.connectionManager is only nil in some unit tests
-		f.connectionManager.trafficTimer.Add(hostinfo.localIndexId, f.connectionManager.checkInterval)
-	}
-
 	if hm.l.Enabled(context.Background(), slog.LevelDebug) {
 		hm.l.Debug("Hostmap vpnIp added",
 			"hostMap", m{"vpnAddrs": hostinfo.vpnAddrs, "mapTotalSize": len(hm.Hosts),
@@ -670,7 +679,7 @@ func (hm *HostMap) ForEachIndex(f controlEach) {
 func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interface) {
 	c := i.promoteCounter.Add(1)
 	if c%ifce.tryPromoteEvery.Load() == 0 {
-		remote := i.GetRemote()
+		remote := i.remote
 
 		// return early if we are already on a preferred remote
 		if remote.IsValid() {
@@ -682,6 +691,7 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interfac
 			}
 		}
 
+		buf := ifce.bufAlloc.Acquire()
 		i.remotes.ForEach(preferredRanges, func(addr netip.AddrPort, preferred bool) {
 			if remote.IsValid() && (!addr.IsValid() || !preferred) {
 				return
@@ -689,8 +699,9 @@ func (i *HostInfo) TryPromoteBest(preferredRanges []netip.Prefix, ifce *Interfac
 
 			// Try to send a test packet to that host, this should
 			// cause it to detect a roaming event and switch remotes
-			ifce.sendTo(header.Test, header.TestRequest, i.ConnectionState, i, addr, []byte(""), make([]byte, 12, 12), make([]byte, mtu))
+			ifce.sendTo(header.Test, header.TestRequest, i.ConnectionState, i, addr, []byte(""), buf)
 		})
+		ifce.bufAlloc.Release(buf)
 	}
 
 	// Re query our lighthouses for new remotes occasionally
@@ -712,18 +723,11 @@ func (i *HostInfo) GetCert() *cert.CachedCertificate {
 	return nil
 }
 
-func (i *HostInfo) GetRemote() netip.AddrPort {
-	if p := i.remote.Load(); p != nil {
-		return *p
-	}
-	return netip.AddrPort{}
-}
-
 // TODO: Maybe use ViaSender here?
 func (i *HostInfo) SetRemote(remote netip.AddrPort) {
 	// We copy here because we likely got this remote from a source that reuses the object
-	if i.GetRemote() != remote {
+	if i.remote != remote {
-		i.remote.Store(&remote)
+		i.remote = remote
 		i.remotes.LearnRemote(i.vpnAddrs[0], remote)
 	}
 }
@@ -735,7 +739,7 @@ func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, via ViaSender) bool {
 		return false
 	}
 
-	currentRemote := i.GetRemote()
+	currentRemote := i.remote
 	if !currentRemote.IsValid() {
 		i.SetRemote(via.UdpAddr)
 		return true

hostmap_test.go

@@ -194,107 +194,6 @@ func TestHostMap_DeleteHostInfo(t *testing.T) {
 	assert.Nil(t, prim)
 }
 
-// TestHostMap_DeleteHostInfo_MultipleVpnAddrs exercises the case where a hostinfo carries more than one
-// vpnAddr and shares its next/prev chain with a live sibling. Deleting the head must not corrupt the
-// sibling: every address the sibling owns has to keep pointing at it. The pre-fix code unlinked the shared
-// chain once per vpnAddr, so on the first address it nil'd next/prev, and on the second address the node
-// looked already-detached: it dropped the map entry instead of promoting the sibling (and tripped the
-// isLastHostinfo relay teardown). See unlockedDeleteHostInfo.
-func TestHostMap_DeleteHostInfo_MultipleVpnAddrs(t *testing.T) {
-	l := test.NewLogger()
-	hm := newHostMap(l)
-
-	f := &Interface{}
-
-	a := netip.MustParseAddr("0.0.0.1")
-	b := netip.MustParseAddr("0.0.0.2")
-
-	// Two tunnels for the same peer, each reachable at both a and b.
-	other := &HostInfo{vpnAddrs: []netip.Addr{a, b}, localIndexId: 1}
-	head := &HostInfo{vpnAddrs: []netip.Addr{a, b}, localIndexId: 2}
-
-	hm.unlockedAddHostInfo(other, f)
-	hm.unlockedAddHostInfo(head, f)
-
-	// head is primary for both addresses, other is next in the shared chain
-	assert.Equal(t, head.localIndexId, hm.QueryVpnAddr(a).localIndexId)
-	assert.Equal(t, head.localIndexId, hm.QueryVpnAddr(b).localIndexId)
-	assert.Equal(t, other.localIndexId, head.next.localIndexId)
-	assert.Equal(t, head.localIndexId, other.prev.localIndexId)
-
-	// Delete the head. other is still live, so it must become primary for BOTH addresses.
-	hm.DeleteHostInfo(head)
-
-	// Pre-fix: QueryVpnAddr(b) came back nil here because the second address was deleted rather than
-	// promoted, leaving other unreachable at b.
-	require.NotNil(t, hm.QueryVpnAddr(a))
-	require.NotNil(t, hm.QueryVpnAddr(b))
-	assert.Equal(t, other.localIndexId, hm.QueryVpnAddr(a).localIndexId)
-	assert.Equal(t, other.localIndexId, hm.QueryVpnAddr(b).localIndexId)
-
-	// other is now the only hostinfo in the chain
-	assert.Nil(t, other.prev)
-	assert.Nil(t, other.next)
-
-	// head is fully detached
-	assert.Nil(t, head.prev)
-	assert.Nil(t, head.next)
-	assert.Nil(t, hm.QueryIndex(head.localIndexId))
-}
-
-// TestHostMap_MaxHostInfosPerVpnIp_MultipleVpnAddrs verifies the MaxHostInfosPerVpnIp overflow prune
-// (unlockedInnerAddHostInfo calls unlockedDeleteHostInfo on the oldest node once the chain is too long)
-// still behaves when hostinfos carry more than one vpnAddr. The pruned node is always the tail, so it is
-// primary for none of the addresses, and both address chains must stay consistent afterwards.
-func TestHostMap_MaxHostInfosPerVpnIp_MultipleVpnAddrs(t *testing.T) {
-	l := test.NewLogger()
-	hm := newHostMap(l)
-
-	f := &Interface{}
-
-	a := netip.MustParseAddr("0.0.0.1")
-	b := netip.MustParseAddr("0.0.0.2")
-
-	// Add one more than the cap, newest last so it becomes head. Every hostinfo owns both a and b.
-	hostinfos := make([]*HostInfo, 0, MaxHostInfosPerVpnIp+1)
-	for i := 0; i <= MaxHostInfosPerVpnIp; i++ {
-		hostinfos = append(hostinfos, &HostInfo{vpnAddrs: []netip.Addr{a, b}, localIndexId: uint32(i + 1)})
-	}
-	// Add oldest first (highest index in our slice) so the very first one added is the overflow victim.
-	for i := len(hostinfos) - 1; i >= 0; i-- {
-		hm.unlockedAddHostInfo(hostinfos[i], f)
-	}
-
-	oldest := hostinfos[len(hostinfos)-1]
-
-	// The oldest hostinfo should have been pruned and fully detached
-	assert.Nil(t, oldest.next)
-	assert.Nil(t, oldest.prev)
-	assert.Nil(t, hm.QueryIndex(oldest.localIndexId))
-
-	// Both addresses resolve to the same head, and that head is one of the survivors (not the pruned one)
-	primA := hm.QueryVpnAddr(a)
-	primB := hm.QueryVpnAddr(b)
-	require.NotNil(t, primA)
-	require.NotNil(t, primB)
-	assert.Equal(t, primA.localIndexId, primB.localIndexId)
-	assert.NotEqual(t, oldest.localIndexId, primA.localIndexId)
-
-	// Walk the shared chain: exactly MaxHostInfosPerVpnIp survivors, no cycles, oldest absent
-	seen := map[uint32]struct{}{}
-	for h := primA; h != nil; h = h.next {
-		_, dup := seen[h.localIndexId]
-		require.False(t, dup, "cycle detected in hostinfo chain")
-		seen[h.localIndexId] = struct{}{}
-		if h.next != nil {
-			assert.Equal(t, h.localIndexId, h.next.prev.localIndexId, "prev pointer must mirror next")
-		}
-	}
-	assert.Len(t, seen, MaxHostInfosPerVpnIp)
-	_, prunedStillPresent := seen[oldest.localIndexId]
-	assert.False(t, prunedStillPresent)
-}
-
 func TestHostMap_reload(t *testing.T) {
 	l := test.NewLogger()
 	c := config.NewC(test.NewLogger())

inside.go

@@ -8,12 +8,13 @@ import (
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/iputil"
-	"github.com/slackhq/nebula/noiseutil"
 	"github.com/slackhq/nebula/routing"
 )
 
-func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet, nb, out []byte, q int, localCache firewall.ConntrackCache) {
+func (f *Interface) consumeInsidePacket(buf *WireBuffer, q int, localCache firewall.ConntrackCache) {
-	err := newPacket(packet, false, fwPacket)
+	packet := buf.IPPacket()
+
+	err := newPacket(packet, false, buf.FwPacket)
 	if err != nil {
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			f.l.Debug("Error while validating outbound packet",
@@ -26,12 +27,12 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 
 	// Ignore local broadcast packets
 	if f.dropLocalBroadcast {
-		if f.myBroadcastAddrsTable.Contains(fwPacket.RemoteAddr) {
+		if f.myBroadcastAddrsTable.Contains(buf.FwPacket.RemoteAddr) {
 			return
 		}
 	}
 
-	if f.myVpnAddrsTable.Contains(fwPacket.RemoteAddr) {
+	if f.myVpnAddrsTable.Contains(buf.FwPacket.RemoteAddr) {
 		// Immediately forward packets from self to self.
 		// This should only happen on Darwin-based and FreeBSD hosts, which
 		// routes packets from the Nebula addr to the Nebula addr through the Nebula
@@ -48,20 +49,20 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 	}
 
 	// Ignore multicast packets
-	if f.dropMulticast && fwPacket.RemoteAddr.IsMulticast() {
+	if f.dropMulticast && buf.FwPacket.RemoteAddr.IsMulticast() {
 		return
 	}
 
-	hostinfo, ready := f.getOrHandshakeConsiderRouting(fwPacket, func(hh *HandshakeHostInfo) {
+	hostinfo, ready := f.getOrHandshakeConsiderRouting(buf.FwPacket, func(hh *HandshakeHostInfo) {
 		hh.cachePacket(f.l, header.Message, 0, packet, f.sendMessageNow, f.cachedPacketMetrics)
 	})
 
 	if hostinfo == nil {
-		f.rejectInside(packet, out, q)
+		f.rejectInside(packet, buf.Out, q)
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			f.l.Debug("dropping outbound packet, vpnAddr not in our vpn networks or in unsafe networks",
-				"vpnAddr", fwPacket.RemoteAddr,
+				"vpnAddr", buf.FwPacket.RemoteAddr,
-				"fwPacket", fwPacket,
+				"fwPacket", buf.FwPacket,
 			)
 		}
 		return
@@ -71,15 +72,15 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 		return
 	}
 
-	dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
+	dropReason := f.firewall.Drop(*buf.FwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason == nil {
-		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, nb, out, q)
+		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, packet, buf, q)
 
 	} else {
-		f.rejectInside(packet, out, q)
+		f.rejectInside(packet, buf.Out, q)
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			hostinfo.logger(f.l).Debug("dropping outbound packet",
-				"fwPacket", fwPacket,
+				"fwPacket", buf.FwPacket,
 				"reason", dropReason,
 			)
 		}
@@ -102,27 +103,27 @@ func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
 	}
 }
 
-func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *HostInfo, nb, out []byte, q int) {
+func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *HostInfo, scratch []byte, buf *WireBuffer, q int) {
 	if !f.firewall.OutSendReject {
 		return
 	}
 
-	out = iputil.CreateRejectPacket(packet, out)
+	rejectIP := iputil.CreateRejectPacket(packet, scratch)
-	if len(out) == 0 {
+	if len(rejectIP) == 0 {
 		return
 	}
 
-	if len(out) > iputil.MaxRejectPacketSize {
+	if len(rejectIP) > iputil.MaxRejectPacketSize {
 		if f.l.Enabled(context.Background(), slog.LevelInfo) {
 			f.l.Info("rejectOutside: packet too big, not sending",
 				"packet", packet,
-				"outPacket", out,
+				"outPacket", rejectIP,
 			)
 		}
 		return
 	}
 
-	f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, out, nb, packet, q)
+	f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, rejectIP, buf, q)
 }
 
 // Handshake will attempt to initiate a tunnel with the provided vpn address. This is a no-op if the tunnel is already established or being established
@@ -215,7 +216,7 @@ func (f *Interface) getOrHandshakeConsiderRouting(fwPacket *firewall.Packet, cac
 
 }
 
-func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte) {
+func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p []byte, buf *WireBuffer) {
 	fp := &firewall.Packet{}
 	err := newPacket(p, false, fp)
 	if err != nil {
@@ -235,12 +236,12 @@ func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubTyp
 		return
 	}
 
-	f.sendNoMetrics(header.Message, st, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, p, nb, out, 0)
+	f.sendNoMetrics(header.Message, st, hostinfo.ConnectionState, hostinfo, netip.AddrPort{}, p, buf, 0)
 }
 
 // SendMessageToVpnAddr handles real addr:port lookup and sends to the current best known address for vpnAddr.
 // This function ignores myVpnNetworksTable, and will always attempt to treat the address as a vpnAddr
-func (f *Interface) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p, nb, out []byte) {
+func (f *Interface) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p []byte, buf *WireBuffer) {
 	hostInfo, ready := f.handshakeManager.GetOrHandshake(vpnAddr, func(hh *HandshakeHostInfo) {
 		hh.cachePacket(f.l, t, st, p, f.SendMessageToHostInfo, f.cachedPacketMetrics)
 	})
@@ -258,113 +259,73 @@ func (f *Interface) SendMessageToVpnAddr(t header.MessageType, st header.Message
 		return
 	}
 
-	f.SendMessageToHostInfo(t, st, hostInfo, p, nb, out)
+	f.SendMessageToHostInfo(t, st, hostInfo, p, buf)
 }
 
-func (f *Interface) SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hi *HostInfo, p, nb, out []byte) {
+func (f *Interface) SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hi *HostInfo, p []byte, buf *WireBuffer) {
-	f.send(t, st, hi.ConnectionState, hi, p, nb, out)
+	f.send(t, st, hi.ConnectionState, hi, p, buf)
 }
 
-func (f *Interface) send(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, p, nb, out []byte) {
+func (f *Interface) send(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, p []byte, buf *WireBuffer) {
 	f.messageMetrics.Tx(t, st, 1)
-	f.sendNoMetrics(t, st, ci, hostinfo, netip.AddrPort{}, p, nb, out, 0)
+	f.sendNoMetrics(t, st, ci, hostinfo, netip.AddrPort{}, p, buf, 0)
 }
 
-func (f *Interface) sendTo(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte) {
+func (f *Interface) sendTo(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p []byte, buf *WireBuffer) {
 	f.messageMetrics.Tx(t, st, 1)
-	f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out, 0)
+	f.sendNoMetrics(t, st, ci, hostinfo, remote, p, buf, 0)
 }
 
 // SendVia sends a payload through a Relay tunnel. No authentication or encryption is done
 // to the payload for the ultimate target host, making this a useful method for sending
 // handshake messages to peers through relay tunnels.
-// via is the HostInfo through which the message is relayed.
+//
-// ad is the plaintext data to authenticate, but not encrypt
+// via is the HostInfo through which the message is relayed. ad is staged into
-// nb is a buffer used to store the nonce value, re-used for performance reasons.
+// the inner-payload slot of buf and then AAD-only sealed under via's key by
-// out is a buffer used to store the result of the Encrypt operation
+// SealRelayInPlace. The sendNoMetrics relay-forward path skips this entry
-// q indicates which writer to use to send the packet.
+// point and calls sendViaInPlace directly because its inner ciphertext is
-func (f *Interface) SendVia(via *HostInfo,
+// already in place from the encrypt step.
-	relay *Relay,
+func (f *Interface) SendVia(via *HostInfo, relay *Relay, ad []byte, buf *WireBuffer) {
-	ad,
+	if header.Len+len(ad)+via.ConnectionState.eKey.Overhead() > cap(buf.Out) {
-	nb,
-	out []byte,
-	nocopy bool,
-) {
-	if noiseutil.EncryptLockNeeded {
-		// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
-		via.ConnectionState.writeLock.Lock()
-	}
-	c := via.ConnectionState.messageCounter.Add(1)
-
-	out = header.Encode(out, header.Version, header.Message, header.MessageRelay, relay.RemoteIndex, c)
-	f.connectionManager.Out(via)
-
-	// Authenticate the header and payload, but do not encrypt for this message type.
-	// The payload consists of the inner, unencrypted Nebula header, as well as the end-to-end encrypted payload.
-	if len(out)+len(ad)+via.ConnectionState.eKey.Overhead() > cap(out) {
-		if noiseutil.EncryptLockNeeded {
-			via.ConnectionState.writeLock.Unlock()
-		}
 		via.logger(f.l).Error("SendVia out buffer not large enough for relay",
-			"outCap", cap(out),
+			"outCap", cap(buf.Out),
 			"payloadLen", len(ad),
-			"headerLen", len(out),
+			"headerLen", header.Len,
 			"cipherOverhead", via.ConnectionState.eKey.Overhead(),
 		)
 		return
 	}
+	buf.StageRelayInner(ad)
+	f.sendViaInPlace(via, relay, len(ad), buf)
+}
 
-	// The header bytes are written to the 'out' slice; Grow the slice to hold the header and associated data payload.
+// sendViaInPlace stamps the outer relay header, AAD-seals over the [outer
-	offset := len(out)
+// header | inner-already-staged] region, and writes the result to via.remote.
-	out = out[:offset+len(ad)]
+// Called from SendVia (after staging ad) and from sendNoMetrics' relay-forward
-
+// path (where the inner ciphertext is already in place from SealForRelay).
-	// In one call path, the associated data _is_ already stored in out. In other call paths, the associated data must
+func (f *Interface) sendViaInPlace(via *HostInfo, relay *Relay, innerLen int, buf *WireBuffer) {
-	// be copied into 'out'.
+	f.connectionManager.Out(via)
-	if !nocopy {
+	out, err := buf.SealRelayInPlace(via.ConnectionState, relay.RemoteIndex, innerLen)
-		copy(out[offset:], ad)
-	}
-
-	var err error
-	out, err = via.ConnectionState.eKey.EncryptDanger(out, out, nil, c, nb)
-	if noiseutil.EncryptLockNeeded {
-		via.ConnectionState.writeLock.Unlock()
-	}
 	if err != nil {
 		via.logger(f.l).Info("Failed to EncryptDanger in sendVia", "error", err)
 		return
 	}
-	err = f.writers[0].WriteTo(out, via.GetRemote())
+	if err := f.writers[0].WriteTo(out, via.remote); err != nil {
-	if err != nil {
 		via.logger(f.l).Info("Failed to WriteTo in sendVia", "error", err)
 	}
 	f.connectionManager.RelayUsed(relay.LocalIndex)
 }
 
-func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p, nb, out []byte, q int) {
+// sendNoMetrics encrypts and writes one outbound nebula packet (data, control,
+// lighthouse, etc) using buf as the per-call wire scratch. When the hostinfo
+// has no direct remote we encrypt into the relay-reserved slot via
+// SealForRelay so sendViaInPlace can wrap it without an extra copy.
+func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType, ci *ConnectionState, hostinfo *HostInfo, remote netip.AddrPort, p []byte, buf *WireBuffer, q int) {
 	if ci.eKey == nil {
 		return
 	}
-	useRelay := !remote.IsValid() && !hostinfo.GetRemote().IsValid()
+	useRelay := !remote.IsValid() && !hostinfo.remote.IsValid()
-	fullOut := out
 
-	if useRelay {
-		if len(out) < header.Len {
-			// out always has a capacity of mtu, but not always a length greater than the header.Len.
-			// Grow it to make sure the next operation works.
-			out = out[:header.Len]
-		}
-		// Save a header's worth of data at the front of the 'out' buffer.
-		out = out[header.Len:]
-	}
-
-	if noiseutil.EncryptLockNeeded {
-		// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
-		ci.writeLock.Lock()
-	}
-	c := ci.messageCounter.Add(1)
-
-	//l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p)
-	out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
 	f.connectionManager.Out(hostinfo)
 
 	// Query our LH if we haven't since the last time we've been rebound, this will cause the remote to punch against
@@ -381,49 +342,42 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
 		}
 	}
 
+	var out []byte
 	var err error
-	out, err = ci.eKey.EncryptDanger(out, out, p, c, nb)
+	if useRelay {
-	if noiseutil.EncryptLockNeeded {
+		out, err = buf.SealForRelay(ci, t, st, hostinfo.remoteIndexId, p)
-		ci.writeLock.Unlock()
+	} else {
+		out, err = buf.Seal(ci, t, st, hostinfo.remoteIndexId, p)
 	}
 	if err != nil {
 		hostinfo.logger(f.l).Error("Failed to encrypt outgoing packet",
 			"error", err,
 			"udpAddr", remote,
-			"counter", c,
 		)
 		return
 	}
 
-	if remote.IsValid() {
+	switch {
-		err = f.writers[q].WriteTo(out, remote)
+	case remote.IsValid():
-		if err != nil {
+		if err := f.writers[q].WriteTo(out, remote); err != nil {
-			hostinfo.logger(f.l).Error("Failed to write outgoing packet",
+			hostinfo.logger(f.l).Error("Failed to write outgoing packet", "error", err, "udpAddr", remote)
-				"error", err,
-				"udpAddr", remote,
-			)
 		}
-	} else if hr := hostinfo.GetRemote(); hr.IsValid() {
+	case hostinfo.remote.IsValid():
-		err = f.writers[q].WriteTo(out, hr)
+		if err := f.writers[q].WriteTo(out, hostinfo.remote); err != nil {
-		if err != nil {
+			hostinfo.logger(f.l).Error("Failed to write outgoing packet", "error", err, "udpAddr", hostinfo.remote)
-			hostinfo.logger(f.l).Error("Failed to write outgoing packet",
-				"error", err,
-				"udpAddr", remote,
-			)
 		}
-	} else {
+	default:
-		// Try to send via a relay
+		// SealForRelay placed the inner ciphertext at buf.Out[header.Len:],
+		// so sendViaInPlace can wrap it with the outer relay header without
+		// an extra copy.
 		for _, relayIP := range hostinfo.relayState.CopyRelayIps() {
 			relayHostInfo, relay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relayIP)
 			if err != nil {
 				hostinfo.relayState.DeleteRelay(relayIP)
-				hostinfo.logger(f.l).Info("sendNoMetrics failed to find HostInfo",
+				hostinfo.logger(f.l).Info("sendNoMetrics failed to find HostInfo", "relay", relayIP, "error", err)
-					"relay", relayIP,
-					"error", err,
-				)
 				continue
 			}
-			f.SendVia(relayHostInfo, relay, out, nb, fullOut[:header.Len+len(out)], true)
+			f.sendViaInPlace(relayHostInfo, relay, len(out), buf)
 			break
 		}
 	}

interface.go

@@ -7,7 +7,6 @@ import (
 	"io"
 	"log/slog"
 	"net/netip"
-	"slices"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -15,7 +14,6 @@ import (
 	"github.com/gaissmai/bart"
 	"github.com/rcrowley/go-metrics"
 
-	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/firewall"
 	"github.com/slackhq/nebula/header"
@@ -103,19 +101,19 @@ type Interface struct {
 	messageMetrics      *MessageMetrics
 	cachedPacketMetrics *cachedPacketMetrics
 
+	// bufAlloc hands out reusable WireBuffers sized for this interface's
+	// inside Device. All buf consumers (hot-path data-plane goroutines,
+	// long-lived workers, and cold callers) acquire from here so sizing
+	// is centralized and consistent. Long-lived owners just don't release.
+	bufAlloc WireBufferAllocator
+
 	l *slog.Logger
 }
 
 type EncWriter interface {
-	SendVia(via *HostInfo,
+	SendVia(via *HostInfo, relay *Relay, ad []byte, buf *WireBuffer)
-		relay *Relay,
+	SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p []byte, buf *WireBuffer)
-		ad,
+	SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p []byte, buf *WireBuffer)
-		nb,
-		out []byte,
-		nocopy bool,
-	)
-	SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnAddr netip.Addr, p, nb, out []byte)
-	SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, nb, out []byte)
 	Handshake(vpnAddr netip.Addr)
 	GetHostInfo(vpnAddr netip.Addr) *HostInfo
 	GetCertState() *CertState
@@ -206,6 +204,8 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
 			dropped: metrics.GetOrRegisterCounter("hostinfo.cached_packets.dropped", nil),
 		},
 
+		bufAlloc: NewWireBufferPool(mtu, c.Inside.TunPrefixLen()),
+
 		l: c.l,
 	}
 
@@ -313,13 +313,11 @@ func (f *Interface) listenOut(i int) {
 
 	ctCache := firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout)
 	lhh := f.lightHouse.NewRequestHandler()
-	plaintext := make([]byte, udp.MTU)
+	// Long-lived per-receive-goroutine buf; never released back to the pool.
-	h := &header.H{}
+	buf := f.bufAlloc.Acquire()
-	fwPacket := &firewall.Packet{}
-	nb := make([]byte, 12, 12)
 
 	err := li.ListenOut(func(fromUdpAddr netip.AddrPort, payload []byte) {
-		f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, plaintext[:0], payload, h, fwPacket, lhh, nb, i, ctCache.Get())
+		f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, buf, payload, lhh, i, ctCache.Get())
 	})
 
 	if err != nil && !f.closed.Load() {
@@ -331,15 +329,12 @@ func (f *Interface) listenOut(i int) {
 }
 
 func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
-	packet := make([]byte, mtu)
+	// Long-lived per-tun-reader buf; never released back to the pool.
-	out := make([]byte, mtu)
+	buf := f.bufAlloc.Acquire()
-	fwPacket := &firewall.Packet{}
-	nb := make([]byte, 12, 12)
-
 	conntrackCache := firewall.NewConntrackCacheTicker(f.ctx, f.l, f.conntrackCacheTimeout)
 
 	for {
-		n, err := reader.Read(packet)
+		_, err := buf.ReadIPFromTUN(reader)
 		if err != nil {
 			if !f.closed.Load() {
 				f.l.Error("Error while reading outbound packet, closing", "error", err, "reader", i)
@@ -348,7 +343,7 @@ func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
 			break
 		}
 
-		f.consumeInsidePacket(packet[:n], fwPacket, nb, out, i, conntrackCache.Get())
+		f.consumeInsidePacket(buf, i, conntrackCache.Get())
 	}
 
 	f.l.Debug("overlay reader is done", "reader", i)
@@ -377,22 +372,13 @@ func (f *Interface) reloadDisconnectInvalid(c *config.C) {
 }
 
 func (f *Interface) reloadFirewall(c *config.C) {
-	cs := f.pki.getCertState()
+	//TODO: need to trigger/detect if the certificate changed too
-	curCert := cs.getCertificate(cert.Version2)
+	if c.HasChanged("firewall") == false {
-	if curCert == nil {
-		curCert = cs.getCertificate(cert.Version1)
-	}
-
-	// The firewall builds its routableNetworks set from the certificate's UnsafeNetworks at construction.
-	// Check to see if that set has changed, and if so, rebuild the firewall.
-	certUnsafeChanged := curCert != nil && !slices.Equal(curCert.UnsafeNetworks(), f.firewall.unsafeNetworks)
-
-	if !c.HasChanged("firewall") && !certUnsafeChanged {
 		f.l.Debug("No firewall config change detected")
 		return
 	}
 
-	fw, err := NewFirewallFromConfig(f.l, cs, c)
+	fw, err := NewFirewallFromConfig(f.l, f.pki.getCertState(), c)
 	if err != nil {
 		f.l.Error("Error while creating firewall during reload", "error", err)
 		return
@@ -502,34 +488,26 @@ func (f *Interface) emitStats(ctx context.Context, i time.Duration) {
 	certInitiatingVersion := metrics.GetOrRegisterGauge("certificate.initiating_version", nil)
 	certMaxVersion := metrics.GetOrRegisterGauge("certificate.max_version", nil)
 
-	emit := func() {
-		f.firewall.EmitStats()
-		f.handshakeManager.EmitStats()
-		udpStats()
-
-		certState := f.pki.getCertState()
-		defaultCrt := certState.GetDefaultCertificate()
-		certExpirationGauge.Update(int64(defaultCrt.NotAfter().Sub(time.Now()) / time.Second))
-		certInitiatingVersion.Update(int64(defaultCrt.Version()))
-
-		// Report the max certificate version we are capable of using
-		if certState.v2Cert != nil {
-			certMaxVersion.Update(int64(certState.v2Cert.Version()))
-		} else {
-			certMaxVersion.Update(int64(certState.v1Cert.Version()))
-		}
-	}
-
-	// Prime gauges so a Prometheus scrape that lands before the first tick
-	// sees real values instead of the zero defaults (issue #907).
-	emit()
-
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case <-ticker.C:
-			emit()
+			f.firewall.EmitStats()
+			f.handshakeManager.EmitStats()
+			udpStats()
+
+			certState := f.pki.getCertState()
+			defaultCrt := certState.GetDefaultCertificate()
+			certExpirationGauge.Update(int64(defaultCrt.NotAfter().Sub(time.Now()) / time.Second))
+			certInitiatingVersion.Update(int64(defaultCrt.Version()))
+
+			// Report the max certificate version we are capable of using
+			if certState.v2Cert != nil {
+				certMaxVersion.Update(int64(certState.v2Cert.Version()))
+			} else {
+				certMaxVersion.Update(int64(certState.v1Cert.Version()))
+			}
 		}
 	}
 }

interface_emit_test.go

@@ -1,73 +0,0 @@
-//go:build linux || darwin
-
-package nebula
-
-import (
-	"context"
-	"net/netip"
-	"testing"
-	"time"
-
-	"github.com/rcrowley/go-metrics"
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/firewall"
-	"github.com/slackhq/nebula/overlay/overlaytest"
-	"github.com/slackhq/nebula/test"
-	"github.com/slackhq/nebula/udp"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// Test_emitStats_primesGauges covers issue #907: a Prometheus scrape that
-// landed before the first ticker fire used to read 0 for the cert gauges.
-// emitStats now primes the gauges before entering the ticker loop. We assert
-// the gauge is zero before the first call and non-zero after.
-func Test_emitStats_primesGauges(t *testing.T) {
-	defer metrics.DefaultRegistry.UnregisterAll()
-
-	l := test.NewLogger()
-	hostMap := newHostMap(l)
-	preferredRanges := []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}
-	hostMap.preferredRanges.Store(&preferredRanges)
-
-	notAfter := time.Now().Add(time.Hour)
-	cs := &CertState{
-		initiatingVersion: cert.Version1,
-		privateKey:        []byte{},
-		v1Cert:            &dummyCert{version: cert.Version1, notAfter: notAfter},
-		v1Credential:      nil,
-	}
-
-	lh := newTestLighthouse()
-	ifce := &Interface{
-		hostMap:          hostMap,
-		inside:           &overlaytest.NoopTun{},
-		outside:          &udp.NoopConn{},
-		firewall:         &Firewall{Conntrack: &FirewallConntrack{Conns: map[firewall.Packet]*conn{}}},
-		lightHouse:       lh,
-		pki:              &PKI{},
-		handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
-		l:                l,
-		// On linux, udp.NewUDPStatsEmitter indexes writers[0] and asserts to
-		// *udp.StdConn. A zero value works: getMemInfo sees a nil rawConn,
-		// returns an error, and the emitter falls through to a no-op.
-		writers: []udp.Conn{&udp.StdConn{}},
-	}
-	ifce.pki.cs.Store(cs)
-
-	ttlGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil)
-	require.Zero(t, ttlGauge.Value(), "gauge should be zero before emitStats runs")
-
-	// Pre-cancel the context so emitStats returns after priming the gauges
-	// without ever reading from ticker.C. The one hour interval is just a
-	// belt-and-suspenders, the test does not expect the ticker to fire.
-	ctx, cancel := context.WithCancel(context.Background())
-	cancel()
-	ifce.emitStats(ctx, time.Hour)
-
-	ttl := ttlGauge.Value()
-	assert.Positive(t, ttl, "ttl gauge should be primed by emitStats before its first tick")
-	assert.LessOrEqual(t, ttl, int64(3600))
-	assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.initiating_version", nil).Value())
-	assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.max_version", nil).Value())
-}

interface_test.go

@@ -1,120 +0,0 @@
-package nebula
-
-import (
-	"net/netip"
-	"testing"
-
-	"github.com/slackhq/nebula/cert"
-	"github.com/slackhq/nebula/config"
-	"github.com/slackhq/nebula/test"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-// TestReloadFirewall_CertUnsafeNetworksChanged verifies that reloadFirewall
-// rebuilds the firewall when only the certificate's UnsafeNetworks have changed,
-// even if the firewall section of the YAML has not.
-func TestReloadFirewall_CertUnsafeNetworksChanged(t *testing.T) {
-	l := test.NewLogger()
-
-	vpnNet := netip.MustParsePrefix("10.0.0.1/24")
-	initialUnsafe := []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")}
-
-	// dummyCert avoids dragging the real signing pipeline into a unit test.
-	c1 := &dummyCert{
-		version:        cert.Version2,
-		networks:       []netip.Prefix{vpnNet},
-		unsafeNetworks: initialUnsafe,
-	}
-	pki := &PKI{}
-	pki.cs.Store(&CertState{v2Cert: c1, initiatingVersion: cert.Version2})
-
-	rawYAML := `firewall:
-  outbound:
-    - port: any
-      proto: any
-      host: any
-  inbound:
-    - port: any
-      proto: any
-      host: any
-`
-	cfg := config.NewC(l)
-	require.NoError(t, cfg.LoadString(rawYAML))
-
-	fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg)
-	require.NoError(t, err)
-	require.Equal(t, initialUnsafe, fw.unsafeNetworks)
-
-	f := &Interface{
-		pki:      pki,
-		firewall: fw,
-		l:        l,
-	}
-
-	// Swap the cert with a different UnsafeNetworks set.
-	newUnsafe := []netip.Prefix{
-		netip.MustParsePrefix("198.51.100.0/24"),
-		netip.MustParsePrefix("203.0.113.0/24"),
-	}
-	c2 := &dummyCert{
-		version:        cert.Version2,
-		networks:       []netip.Prefix{vpnNet},
-		unsafeNetworks: newUnsafe,
-	}
-	pki.cs.Store(&CertState{v2Cert: c2, initiatingVersion: cert.Version2})
-
-	// Reload with the same YAML so HasChanged("firewall") reports false.
-	require.NoError(t, cfg.ReloadConfigString(rawYAML))
-	require.False(t, cfg.HasChanged("firewall"))
-
-	f.reloadFirewall(cfg)
-
-	assert.NotSame(t, fw, f.firewall, "firewall pointer should have been replaced")
-	assert.Equal(t, newUnsafe, f.firewall.unsafeNetworks)
-	assert.True(t, f.firewall.routableNetworks.Contains(netip.MustParseAddr("203.0.113.5")))
-}
-
-// TestReloadFirewall_NoChange verifies that reloadFirewall is a no-op when
-// neither the firewall config nor the cert's UnsafeNetworks have changed.
-func TestReloadFirewall_NoChange(t *testing.T) {
-	l := test.NewLogger()
-
-	vpnNet := netip.MustParsePrefix("10.0.0.1/24")
-	unsafe := []netip.Prefix{netip.MustParsePrefix("198.51.100.0/24")}
-
-	c1 := &dummyCert{
-		version:        cert.Version2,
-		networks:       []netip.Prefix{vpnNet},
-		unsafeNetworks: unsafe,
-	}
-	pki := &PKI{}
-	pki.cs.Store(&CertState{v2Cert: c1, initiatingVersion: cert.Version2})
-
-	rawYAML := `firewall:
-  outbound:
-    - port: any
-      proto: any
-      host: any
-  inbound:
-    - port: any
-      proto: any
-      host: any
-`
-	cfg := config.NewC(l)
-	require.NoError(t, cfg.LoadString(rawYAML))
-
-	fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg)
-	require.NoError(t, err)
-
-	f := &Interface{
-		pki:      pki,
-		firewall: fw,
-		l:        l,
-	}
-
-	require.NoError(t, cfg.ReloadConfigString(rawYAML))
-	f.reloadFirewall(cfg)
-
-	assert.Same(t, fw, f.firewall, "firewall should not have been replaced")
-}

iputil/packet.go

@@ -4,54 +4,26 @@ import (
 	"encoding/binary"
 
 	"golang.org/x/net/ipv4"
-	"golang.org/x/net/ipv6"
 )
 
 const (
-	// MaxIPv4RejectPacketSize is the largest IPv4 reject packet:
+	// Need 96 bytes for the largest reject packet:
 	// - 20 byte ipv4 header
 	// - 8 byte icmpv4 header
 	// - 68 byte body (60 byte max orig ipv4 header + 8 byte orig icmpv4 header)
-	maxIPv4RejectPacketSize = ipv4.HeaderLen + 8 + 60 + 8
+	MaxRejectPacketSize = ipv4.HeaderLen + 8 + 60 + 8
-
-	// MaxRejectPacketSize is sized for the largest possible reject packet (IPv6):
-	// - 40 byte ipv6 header
-	// - 8 byte icmpv6 header
-	// - up to 1000 byte body (original packet, possibly truncated. We want to stay
-	//   under the MTU with Nebula overhead included)
-	maxIPv6RejectPacketSize = ipv6.HeaderLen + 8 + 1000
-
-	MaxRejectPacketSize = maxIPv6RejectPacketSize
 )
 
 func CreateRejectPacket(packet []byte, out []byte) []byte {
-	if len(packet) < 1 {
+	if len(packet) < ipv4.HeaderLen || int(packet[0]>>4) != ipv4.Version {
 		return nil
 	}
 
-	version := int(packet[0] >> 4)
+	switch packet[9] {
-	switch version {
+	case 6: // tcp
-	case ipv4.Version:
+		return ipv4CreateRejectTCPPacket(packet, out)
-		if len(packet) < ipv4.HeaderLen {
-			return nil
-		}
-		// Do not send reject packets for non-first fragments
-		if packet[6]&0x1f != 0 || packet[7] != 0 {
-			return nil
-		}
-		switch packet[9] {
-		case 6: // tcp
-			return ipv4CreateRejectTCPPacket(packet, out)
-		default:
-			return ipv4CreateRejectICMPPacket(packet, out)
-		}
-	case ipv6.Version:
-		if len(packet) < ipv6.HeaderLen {
-			return nil
-		}
-		return ipv6CreateRejectPacket(packet, out)
 	default:
-		return nil
+		return ipv4CreateRejectICMPPacket(packet, out)
 	}
 }
 
@@ -63,16 +35,11 @@ func ipv4CreateRejectICMPPacket(packet []byte, out []byte) []byte {
 		return nil
 	}
 
-	// Do not generate ICMP errors in response to ICMP error packets
-	if packet[9] == 1 && len(packet) > ihl {
-		icmpType := packet[ihl]
-		if icmpType == 3 || icmpType == 4 || icmpType == 5 || icmpType == 11 || icmpType == 12 {
-			return nil
-		}
-	}
-
 	// ICMP reply includes original header and first 8 bytes of the packet
-	packetLen := min(len(packet), ihl+8)
+	packetLen := len(packet)
+	if packetLen > ihl+8 {
+		packetLen = ihl + 8
+	}
 
 	outLen := ipv4.HeaderLen + 8 + packetLen
 	if outLen > cap(out) {
@@ -104,14 +71,14 @@ func ipv4CreateRejectICMPPacket(packet []byte, out []byte) []byte {
 
 	// ICMP Destination Unreachable
 	icmpOut := out[ipv4.HeaderLen:]
-	icmpOut[0] = 3  // type (Destination unreachable)
+	icmpOut[0] = 3 // type (Destination unreachable)
-	icmpOut[1] = 13 // code (Communication administratively prohibited)
+	icmpOut[1] = 3 // code (Port unreachable error)
-	icmpOut[2] = 0  // checksum
+	icmpOut[2] = 0 // checksum
-	icmpOut[3] = 0  //  .
+	icmpOut[3] = 0 //  .
-	icmpOut[4] = 0  // unused
+	icmpOut[4] = 0 // unused
-	icmpOut[5] = 0  //  .
+	icmpOut[5] = 0 //  .
-	icmpOut[6] = 0  //  .
+	icmpOut[6] = 0 //  .
-	icmpOut[7] = 0  //  .
+	icmpOut[7] = 0 //  .
 
 	// Copy original IP header and first 8 bytes as body
 	copy(icmpOut[8:], packet[:packetLen])
@@ -198,193 +165,7 @@ func ipv4CreateRejectTCPPacket(packet []byte, out []byte) []byte {
 	return out
 }
 
-func ipv6CreateRejectPacket(packet []byte, out []byte) []byte {
-	proto, offset, isFragment := ipv6FindUpperProtocol(packet)
-	if isFragment {
-		return nil
-	}
-	switch proto {
-	case 6: // tcp
-		return ipv6CreateRejectTCPPacket(packet, out, offset)
-	default:
-		return ipv6CreateRejectICMPPacket(packet, out, proto, offset)
-	}
-}
-
-func ipv6CreateRejectICMPPacket(packet []byte, out []byte, proto uint8, offset int) []byte {
-	// Do not generate ICMPv6 errors in response to ICMPv6 error packets
-	if proto == 58 && len(packet) > offset {
-		icmpType := packet[offset]
-		if icmpType >= 1 && icmpType <= 4 {
-			return nil
-		}
-	}
-
-	// Include as much of the original packet as possible, up to 1000 bytes,
-	// so the response fits comfortably within any tunnel MTU.
-	packetLen := min(len(packet), 1000)
-
-	outLen := ipv6.HeaderLen + 8 + packetLen
-	if outLen > cap(out) {
-		return nil
-	}
-
-	out = out[:outLen]
-
-	// IPv6 header
-	ipHdr := out[0:ipv6.HeaderLen]
-	ipHdr[0] = ipv6.Version << 4 // version, traffic class (high bits)
-	ipHdr[1] = 0                 // traffic class (low bits), flow label (high bits)
-	ipHdr[2] = 0                 // flow label
-	ipHdr[3] = 0                 // flow label
-
-	payloadLen := uint16(outLen - ipv6.HeaderLen)
-	binary.BigEndian.PutUint16(ipHdr[4:], payloadLen) // payload length
-	ipHdr[6] = 58                                     // next header (ICMPv6)
-	ipHdr[7] = 64                                     // hop limit
-
-	// Swap dest / src IPs (each 16 bytes, src at 8, dst at 24)
-	copy(ipHdr[8:24], packet[24:40])
-	copy(ipHdr[24:40], packet[8:24])
-
-	// ICMPv6 Destination Unreachable
-	icmpOut := out[ipv6.HeaderLen:]
-	icmpOut[0] = 1 // type (Destination Unreachable)
-	icmpOut[1] = 1 // code (Communication with destination administratively prohibited)
-	icmpOut[2] = 0 // checksum
-	icmpOut[3] = 0 //  .
-	icmpOut[4] = 0 // unused
-	icmpOut[5] = 0 //  .
-	icmpOut[6] = 0 //  .
-	icmpOut[7] = 0 //  .
-
-	copy(icmpOut[8:], packet[:packetLen])
-
-	// ICMPv6 checksum uses a pseudo-header
-	csum := ipv6PseudoheaderChecksum(ipHdr[8:24], ipHdr[24:40], 58, uint32(payloadLen))
-	binary.BigEndian.PutUint16(icmpOut[2:], tcpipChecksum(icmpOut, csum))
-
-	return out
-}
-
-func ipv6CreateRejectTCPPacket(packet []byte, out []byte, offset int) []byte {
-	const tcpLen = 20
-
-	if len(packet) < offset+tcpLen {
-		return nil
-	}
-
-	outLen := ipv6.HeaderLen + tcpLen
-	if outLen > cap(out) {
-		return nil
-	}
-
-	out = out[:outLen]
-
-	// IPv6 header
-	ipHdr := out[0:ipv6.HeaderLen]
-	ipHdr[0] = ipv6.Version << 4 // version, traffic class (high bits)
-	ipHdr[1] = 0                 // traffic class (low bits), flow label (high bits)
-	ipHdr[2] = 0                 // flow label
-	ipHdr[3] = 0                 // flow label
-
-	binary.BigEndian.PutUint16(ipHdr[4:], tcpLen) // payload length
-	ipHdr[6] = 6                                  // next header (TCP)
-	ipHdr[7] = 64                                 // hop limit
-
-	// Swap dest / src IPs
-	copy(ipHdr[8:24], packet[24:40])
-	copy(ipHdr[24:40], packet[8:24])
-
-	// TCP RST
-	tcpIn := packet[offset:]
-	var ackSeq, seq uint32
-	outFlags := byte(0b00000100) // RST
-
-	inAck := tcpIn[13]&0b00010000 != 0
-	if inAck {
-		seq = binary.BigEndian.Uint32(tcpIn[8:])
-	} else {
-		inSyn := uint32((tcpIn[13] & 0b00000010) >> 1)
-		inFin := uint32(tcpIn[13] & 0b00000001)
-		ackSeq = binary.BigEndian.Uint32(tcpIn[4:]) + inSyn + inFin + uint32(len(tcpIn)) - uint32(tcpIn[12]>>4)<<2
-		outFlags |= 0b00010000 // ACK
-	}
-
-	tcpOut := out[ipv6.HeaderLen:]
-	// Swap dest / src ports
-	copy(tcpOut[0:2], tcpIn[2:4])
-	copy(tcpOut[2:4], tcpIn[0:2])
-	binary.BigEndian.PutUint32(tcpOut[4:], seq)
-	binary.BigEndian.PutUint32(tcpOut[8:], ackSeq)
-	tcpOut[12] = (tcpLen >> 2) << 4 // data offset, reserved, NS
-	tcpOut[13] = outFlags           // CWR, ECE, URG, ACK, PSH, RST, SYN, FIN
-	tcpOut[14] = 0                  // window size
-	tcpOut[15] = 0                  //  .
-	tcpOut[16] = 0                  // checksum
-	tcpOut[17] = 0                  //  .
-	tcpOut[18] = 0                  // URG Pointer
-	tcpOut[19] = 0                  //  .
-
-	// Calculate checksum with IPv6 pseudo-header
-	csum := ipv6PseudoheaderChecksum(ipHdr[8:24], ipHdr[24:40], 6, tcpLen)
-	binary.BigEndian.PutUint16(tcpOut[16:], tcpipChecksum(tcpOut, csum))
-
-	return out
-}
-
-func ipv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragment bool) {
-	nextHeader = packet[6]
-	offset = ipv6.HeaderLen
-
-	for {
-		switch nextHeader {
-		case 0, 43, 60: // Hop-by-Hop, Routing, Destination
-			if len(packet) < offset+2 {
-				return nextHeader, offset, isFragment
-			}
-			nextHeader = packet[offset]
-			offset += (int(packet[offset+1]) + 1) << 3
-
-		case 44: // Fragment
-			if len(packet) < offset+8 {
-				return nextHeader, offset, isFragment
-			}
-			if packet[offset+2] != 0 || packet[offset+3]&0xf8 != 0 {
-				isFragment = true
-			}
-			nextHeader = packet[offset]
-			offset += 8
-
-		case 51: // AH
-			if len(packet) < offset+2 {
-				return nextHeader, offset, isFragment
-			}
-			nextHeader = packet[offset]
-			offset += (int(packet[offset+1]) + 2) << 2
-
-		default:
-			return nextHeader, offset, isFragment
-		}
-	}
-}
-
 func CreateICMPEchoResponse(packet, out []byte) []byte {
-	if len(packet) < 1 {
-		return nil
-	}
-
-	switch packet[0] >> 4 {
-	case 4:
-		return createICMPv4EchoResponse(packet, out)
-	case 6:
-		return createICMPv6EchoResponse(packet, out)
-	default:
-		return nil
-	}
-}
-
-func createICMPv4EchoResponse(packet, out []byte) []byte {
 	// Return early if this is not a simple ICMP Echo Request
 	//TODO: make constants out of these
 	if !(len(packet) >= 28 && len(packet) <= 9001 && packet[0] == 0x45 && packet[9] == 0x01 && packet[20] == 0x08) {
@@ -418,43 +199,6 @@ func createICMPv4EchoResponse(packet, out []byte) []byte {
 	return out
 }
 
-func createICMPv6EchoResponse(packet, out []byte) []byte {
-	// IPv6 header (40 bytes) + ICMPv6 header (8 bytes minimum)
-	if len(packet) < ipv6.HeaderLen+8 || len(packet) > 9001 {
-		return nil
-	}
-
-	// Next Header must be ICMPv6 (58)
-	if packet[6] != 58 {
-		return nil
-	}
-
-	// ICMPv6 type must be Echo Request (128)
-	if packet[ipv6.HeaderLen] != 128 {
-		return nil
-	}
-
-	out = out[:len(packet)]
-	copy(out, packet)
-
-	// Swap src/dst addresses (bytes 8-23 and 24-39)
-	copy(out[8:24], packet[24:40])
-	copy(out[24:40], packet[8:24])
-
-	// Change ICMPv6 type to Echo Reply (129)
-	icmp := out[ipv6.HeaderLen:]
-	icmp[0] = 129
-	icmp[2] = 0
-	icmp[3] = 0
-
-	// ICMPv6 checksum uses a pseudo-header with src, dst, length, and next header
-	payloadLen := uint32(len(icmp))
-	csum := ipv6PseudoheaderChecksum(out[8:24], out[24:40], 58, payloadLen)
-	binary.BigEndian.PutUint16(icmp[2:], tcpipChecksum(icmp, csum))
-
-	return out
-}
-
 // calculates the TCP/IP checksum defined in rfc1071. The passed-in
 // csum is any initial checksum data that's already been computed.
 //
@@ -492,18 +236,3 @@ func ipv4PseudoheaderChecksum(src, dst []byte, proto, length uint32) (csum uint3
 	csum += length >> 16
 	return csum
 }
-
-// based on:
-// - https://github.com/google/gopacket/blob/v1.1.19/layers/tcpip.go#L37-L48
-func ipv6PseudoheaderChecksum(src, dst []byte, proto, length uint32) (csum uint32) {
-	for i := 0; i < 16; i += 2 {
-		csum += uint32(src[i]) << 8
-		csum += uint32(src[i+1])
-		csum += uint32(dst[i]) << 8
-		csum += uint32(dst[i+1])
-	}
-	csum += proto
-	csum += length & 0xffff
-	csum += length >> 16
-	return csum
-}

iputil/packet_test.go

@@ -1,13 +1,11 @@
 package iputil
 
 import (
-	"encoding/binary"
 	"net"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/net/ipv4"
-	"golang.org/x/net/ipv6"
 )
 
 func Test_CreateRejectPacket(t *testing.T) {
@@ -45,7 +43,7 @@ func Test_CreateRejectPacket(t *testing.T) {
 	}
 	b = append(b, []byte{0, 3, 0, 4, 0, 0, 0, 0}...)
 
-	expectedLen = maxIPv4RejectPacketSize
+	expectedLen = MaxRejectPacketSize
 	out = make([]byte, MaxRejectPacketSize)
 	rejectPacket = CreateRejectPacket(b, out)
 	assert.NotNil(t, rejectPacket)
@@ -73,404 +71,3 @@ func Test_CreateRejectPacket(t *testing.T) {
 	assert.NotNil(t, rejectPacket)
 	assert.Len(t, rejectPacket, expectedLen)
 }
-
-func Test_CreateRejectPacket_NoFragment(t *testing.T) {
-	out := make([]byte, MaxRejectPacketSize)
-
-	// IPv4: non-zero fragment offset should not generate reject packet
-	h := ipv4.Header{
-		Len:      20,
-		Src:      net.IPv4(10, 0, 0, 1),
-		Dst:      net.IPv4(10, 0, 0, 2),
-		Protocol: 17, // UDP
-	}
-	b, err := h.Marshal()
-	if err != nil {
-		t.Fatalf("h.Marshal: %v", err)
-	}
-	b = append(b, make([]byte, 8)...)
-	// Set fragment offset to non-zero (byte 6-7, offset in 8-byte units)
-	b[6] = 0x00
-	b[7] = 0x01
-	assert.Nil(t, CreateRejectPacket(b, out))
-
-	// MF flag with zero offset (first fragment) should still generate reject
-	b[6] = 0x20 // MF flag set
-	b[7] = 0x00
-	assert.NotNil(t, CreateRejectPacket(b, out))
-
-	// Non-fragment should still generate reject packet
-	b[6] = 0x00
-	b[7] = 0x00
-	assert.NotNil(t, CreateRejectPacket(b, out))
-
-	// DF flag only (not a fragment) should still generate reject packet
-	b[6] = 0x40
-	b[7] = 0x00
-	assert.NotNil(t, CreateRejectPacket(b, out))
-}
-
-func Test_CreateRejectPacketIPv6_NoFragment(t *testing.T) {
-	src := net.ParseIP("fd00::1")
-	dst := net.ParseIP("fd00::2")
-	out := make([]byte, MaxRejectPacketSize)
-
-	// IPv6 with Fragment header and non-zero offset should not generate reject
-	fragHeader := []byte{
-		17,   // next header: UDP
-		0,    // reserved
-		0, 9, // fragment offset=1 (shifted left 3), M=1
-		0, 0, 0, 1, // identification
-	}
-	udpPayload := make([]byte, 8)
-	payload := append(fragHeader, udpPayload...)
-	packet := makeIPv6Packet(src, dst, 44, payload) // next header 44 = Fragment
-	assert.Nil(t, CreateRejectPacket(packet, out))
-
-	// Fragment header with zero offset (first fragment) should still generate reject
-	fragHeader[2] = 0
-	fragHeader[3] = 1 // offset=0, M=1
-	payload = append(fragHeader, udpPayload...)
-	packet = makeIPv6Packet(src, dst, 44, payload)
-	assert.NotNil(t, CreateRejectPacket(packet, out))
-}
-
-func Test_CreateRejectPacket_NoICMPError(t *testing.T) {
-	out := make([]byte, MaxRejectPacketSize)
-
-	// ICMP error types should not generate reject packets
-	icmpErrorTypes := []byte{3, 4, 5, 11, 12}
-	for _, icmpType := range icmpErrorTypes {
-		h := ipv4.Header{
-			Len:      20,
-			Src:      net.IPv4(10, 0, 0, 1),
-			Dst:      net.IPv4(10, 0, 0, 2),
-			Protocol: 1, // ICMP
-		}
-
-		b, err := h.Marshal()
-		if err != nil {
-			t.Fatalf("h.Marshal: %v", err)
-		}
-		b = append(b, icmpType, 0, 0, 0, 0, 0, 0, 0)
-
-		rejectPacket := CreateRejectPacket(b, out)
-		assert.Nil(t, rejectPacket, "ICMP type %d should not generate a reject packet", icmpType)
-	}
-
-	// ICMP non-error types should still generate reject packets
-	icmpNonErrorTypes := []byte{0, 8, 13, 14}
-	for _, icmpType := range icmpNonErrorTypes {
-		h := ipv4.Header{
-			Len:      20,
-			Src:      net.IPv4(10, 0, 0, 1),
-			Dst:      net.IPv4(10, 0, 0, 2),
-			Protocol: 1, // ICMP
-		}
-
-		b, err := h.Marshal()
-		if err != nil {
-			t.Fatalf("h.Marshal: %v", err)
-		}
-		b = append(b, icmpType, 0, 0, 0, 0, 0, 0, 0)
-
-		rejectPacket := CreateRejectPacket(b, out)
-		assert.NotNil(t, rejectPacket, "ICMP type %d should generate a reject packet", icmpType)
-	}
-}
-
-func makeIPv6Packet(src, dst net.IP, nextHeader uint8, payload []byte) []byte {
-	b := make([]byte, ipv6.HeaderLen+len(payload))
-	b[0] = ipv6.Version << 4
-	binary.BigEndian.PutUint16(b[4:], uint16(len(payload)))
-	b[6] = nextHeader
-	b[7] = 64
-	copy(b[8:24], src.To16())
-	copy(b[24:40], dst.To16())
-	copy(b[ipv6.HeaderLen:], payload)
-	return b
-}
-
-func Test_CreateRejectPacketIPv6_ICMP(t *testing.T) {
-	src := net.ParseIP("fd00::1")
-	dst := net.ParseIP("fd00::2")
-
-	// Small UDP packet: entire original included in body
-	udpPayload := make([]byte, 20)
-	udpPayload[0] = 0x00 // src port high
-	udpPayload[1] = 0x50 // src port low (80)
-	udpPayload[2] = 0x01 // dst port high
-	udpPayload[3] = 0xBB // dst port low (443)
-	packet := makeIPv6Packet(src, dst, 17, udpPayload)
-
-	out := make([]byte, MaxRejectPacketSize)
-	rejectPacket := CreateRejectPacket(packet, out)
-	assert.NotNil(t, rejectPacket)
-
-	// Small packet fits entirely: 40 (ipv6 hdr) + 8 (icmpv6 hdr) + 60 (original)
-	expectedLen := ipv6.HeaderLen + 8 + len(packet)
-	assert.Len(t, rejectPacket, expectedLen)
-
-	// Verify version
-	assert.Equal(t, byte(ipv6.Version<<4), rejectPacket[0]&0xf0)
-	// Verify next header is ICMPv6 (58)
-	assert.Equal(t, byte(58), rejectPacket[6])
-	// Verify src/dst are swapped
-	assert.Equal(t, dst.To16(), net.IP(rejectPacket[8:24]))
-	assert.Equal(t, src.To16(), net.IP(rejectPacket[24:40]))
-	// Verify ICMPv6 type=1 (Dest Unreachable), code=1 (Administratively prohibited)
-	assert.Equal(t, byte(1), rejectPacket[ipv6.HeaderLen])
-	assert.Equal(t, byte(1), rejectPacket[ipv6.HeaderLen+1])
-	// Verify entire original packet is included in body
-	assert.Equal(t, packet, rejectPacket[ipv6.HeaderLen+8:])
-
-	// Large packet: body is truncated to 1000 bytes
-	largePkt := makeIPv6Packet(src, dst, 17, make([]byte, 1200))
-	rejectPacket = CreateRejectPacket(largePkt, out)
-	assert.NotNil(t, rejectPacket)
-	assert.Len(t, rejectPacket, ipv6.HeaderLen+8+1000)
-	assert.Equal(t, largePkt[:1000], rejectPacket[ipv6.HeaderLen+8:])
-}
-
-func Test_CreateRejectPacketIPv6_TCP(t *testing.T) {
-	src := net.ParseIP("fd00::1")
-	dst := net.ParseIP("fd00::2")
-
-	// TCP SYN packet (next header 6)
-	tcpPayload := make([]byte, 20)
-	tcpPayload[0] = 0x00                             // src port high
-	tcpPayload[1] = 0x50                             // src port low (80)
-	tcpPayload[2] = 0x01                             // dst port high
-	tcpPayload[3] = 0xBB                             // dst port low (443)
-	binary.BigEndian.PutUint32(tcpPayload[4:], 1000) // seq
-	binary.BigEndian.PutUint32(tcpPayload[8:], 0)    // ack seq
-	tcpPayload[12] = (20 >> 2) << 4                  // data offset
-	tcpPayload[13] = 0b00000010                      // SYN flag
-
-	packet := makeIPv6Packet(src, dst, 6, tcpPayload)
-
-	out := make([]byte, MaxRejectPacketSize)
-	rejectPacket := CreateRejectPacket(packet, out)
-	assert.NotNil(t, rejectPacket)
-
-	// Expected: 40 (ipv6 hdr) + 20 (tcp RST)
-	expectedLen := ipv6.HeaderLen + 20
-	assert.Len(t, rejectPacket, expectedLen)
-
-	// Verify version
-	assert.Equal(t, byte(ipv6.Version<<4), rejectPacket[0]&0xf0)
-	// Verify next header is TCP (6)
-	assert.Equal(t, byte(6), rejectPacket[6])
-	// Verify src/dst are swapped
-	assert.Equal(t, dst.To16(), net.IP(rejectPacket[8:24]))
-	assert.Equal(t, src.To16(), net.IP(rejectPacket[24:40]))
-	// Verify ports are swapped
-	tcpOut := rejectPacket[ipv6.HeaderLen:]
-	assert.Equal(t, uint16(443), binary.BigEndian.Uint16(tcpOut[0:2]))
-	assert.Equal(t, uint16(80), binary.BigEndian.Uint16(tcpOut[2:4]))
-	// RST+ACK flags (since input was SYN without ACK)
-	assert.Equal(t, byte(0b00010100), tcpOut[13])
-	// ack_seq = original seq (1000) + SYN (1) + FIN (0) + segment data (0)
-	assert.Equal(t, uint32(1001), binary.BigEndian.Uint32(tcpOut[8:]))
-}
-
-func Test_CreateRejectPacketIPv6_TCPWithACK(t *testing.T) {
-	src := net.ParseIP("fd00::1")
-	dst := net.ParseIP("fd00::2")
-
-	// TCP packet with ACK set
-	tcpPayload := make([]byte, 20)
-	tcpPayload[0] = 0x00
-	tcpPayload[1] = 0x50
-	tcpPayload[2] = 0x01
-	tcpPayload[3] = 0xBB
-	binary.BigEndian.PutUint32(tcpPayload[4:], 1000) // seq
-	binary.BigEndian.PutUint32(tcpPayload[8:], 2000) // ack seq
-	tcpPayload[12] = (20 >> 2) << 4                  // data offset
-	tcpPayload[13] = 0b00010000                      // ACK flag
-
-	packet := makeIPv6Packet(src, dst, 6, tcpPayload)
-
-	out := make([]byte, MaxRejectPacketSize)
-	rejectPacket := CreateRejectPacket(packet, out)
-	assert.NotNil(t, rejectPacket)
-
-	tcpOut := rejectPacket[ipv6.HeaderLen:]
-	// RST only (no ACK) since input had ACK
-	assert.Equal(t, byte(0b00000100), tcpOut[13])
-	// seq = original ack_seq
-	assert.Equal(t, uint32(2000), binary.BigEndian.Uint32(tcpOut[4:]))
-}
-
-func Test_CreateRejectPacketIPv6_NoICMPError(t *testing.T) {
-	src := net.ParseIP("fd00::1")
-	dst := net.ParseIP("fd00::2")
-	out := make([]byte, MaxRejectPacketSize)
-
-	// ICMPv6 error types (1-4) should not generate reject packets
-	for icmpType := byte(1); icmpType <= 4; icmpType++ {
-		payload := make([]byte, 8)
-		payload[0] = icmpType
-		packet := makeIPv6Packet(src, dst, 58, payload)
-
-		rejectPacket := CreateRejectPacket(packet, out)
-		assert.Nil(t, rejectPacket, "ICMPv6 type %d should not generate a reject packet", icmpType)
-	}
-
-	// ICMPv6 non-error types should still generate reject packets
-	nonErrorTypes := []byte{128, 129, 133, 134}
-	for _, icmpType := range nonErrorTypes {
-		payload := make([]byte, 8)
-		payload[0] = icmpType
-		packet := makeIPv6Packet(src, dst, 58, payload)
-
-		rejectPacket := CreateRejectPacket(packet, out)
-		assert.NotNil(t, rejectPacket, "ICMPv6 type %d should generate a reject packet", icmpType)
-	}
-}
-
-func Test_CreateRejectPacketIPv6_TooShort(t *testing.T) {
-	// Packet too short to be valid IPv6
-	out := make([]byte, MaxRejectPacketSize)
-	assert.Nil(t, CreateRejectPacket([]byte{0x60}, out))
-	assert.Nil(t, CreateRejectPacket(make([]byte, 39), out))
-}
-
-func Test_CreateRejectPacketIPv6_ExtensionHeaders(t *testing.T) {
-	src := net.ParseIP("fd00::1")
-	dst := net.ParseIP("fd00::2")
-
-	// IPv6 + Hop-by-Hop extension header + TCP
-	hopByHop := []byte{
-		6,    // next header: TCP
-		0,    // length (8 bytes total)
-		0, 0, // padding
-		0, 0, 0, 0,
-	}
-	tcpPayload := make([]byte, 20)
-	tcpPayload[0] = 0x00
-	tcpPayload[1] = 0x50
-	tcpPayload[2] = 0x01
-	tcpPayload[3] = 0xBB
-	binary.BigEndian.PutUint32(tcpPayload[4:], 1000)
-	binary.BigEndian.PutUint32(tcpPayload[8:], 2000)
-	tcpPayload[12] = (20 >> 2) << 4
-	tcpPayload[13] = 0b00010000 // ACK
-
-	payload := append(hopByHop, tcpPayload...)
-	packet := makeIPv6Packet(src, dst, 0, payload) // next header 0 = Hop-by-Hop
-
-	out := make([]byte, MaxRejectPacketSize)
-	rejectPacket := CreateRejectPacket(packet, out)
-	assert.NotNil(t, rejectPacket)
-
-	// Should produce TCP RST
-	expectedLen := ipv6.HeaderLen + 20
-	assert.Len(t, rejectPacket, expectedLen)
-	assert.Equal(t, byte(6), rejectPacket[6]) // next header is TCP
-	tcpOut := rejectPacket[ipv6.HeaderLen:]
-	assert.Equal(t, byte(0b00000100), tcpOut[13]) // RST only
-}
-
-func TestCreateICMPEchoResponse_IPv4(t *testing.T) {
-	// Build a simple IPv4 ICMP Echo Request
-	packet := make([]byte, 28)
-	packet[0] = 0x45                                   // version 4, IHL 5
-	binary.BigEndian.PutUint16(packet[2:], uint16(28)) // total length
-	packet[8] = 64                                     // TTL
-	packet[9] = 1                                      // protocol ICMP
-	copy(packet[12:16], net.IPv4(10, 0, 0, 1).To4())   // src
-	copy(packet[16:20], net.IPv4(10, 0, 0, 2).To4())   // dst
-	packet[20] = 8                                     // ICMP Echo Request
-
-	out := make([]byte, len(packet))
-	result := CreateICMPEchoResponse(packet, out)
-	assert.NotNil(t, result)
-	assert.Equal(t, byte(0x45), result[0])
-	// src/dst swapped
-	assert.Equal(t, net.IPv4(10, 0, 0, 2).To4(), net.IP(result[12:16]))
-	assert.Equal(t, net.IPv4(10, 0, 0, 1).To4(), net.IP(result[16:20]))
-	// ICMP Echo Reply
-	assert.Equal(t, byte(0), result[20])
-}
-
-func TestCreateICMPEchoResponse_IPv6(t *testing.T) {
-	src := net.ParseIP("fd00::1").To16()
-	dst := net.ParseIP("fd00::2").To16()
-
-	// Build an IPv6 ICMPv6 Echo Request packet
-	// IPv6 header (40 bytes) + ICMPv6 (8 bytes)
-	packet := make([]byte, 48)
-	packet[0] = 0x60        // version 6
-	payloadLen := uint16(8) // ICMPv6 header only
-	binary.BigEndian.PutUint16(packet[4:], payloadLen)
-	packet[6] = 58           // Next Header: ICMPv6
-	packet[7] = 64           // Hop Limit
-	copy(packet[8:24], src)  // src address
-	copy(packet[24:40], dst) // dst address
-
-	// ICMPv6 Echo Request
-	icmp := packet[40:]
-	icmp[0] = 128                           // type: Echo Request
-	icmp[1] = 0                             // code
-	binary.BigEndian.PutUint16(icmp[4:], 1) // identifier
-	binary.BigEndian.PutUint16(icmp[6:], 1) // sequence number
-
-	// Compute correct checksum for the request
-	csum := ipv6PseudoheaderChecksum(src, dst, 58, uint32(payloadLen))
-	binary.BigEndian.PutUint16(icmp[2:], tcpipChecksum(icmp, csum))
-
-	out := make([]byte, len(packet))
-	result := CreateICMPEchoResponse(packet, out)
-	assert.NotNil(t, result)
-
-	// Version should still be 6
-	assert.Equal(t, byte(6), result[0]>>4)
-	// src/dst swapped
-	assert.Equal(t, dst, net.IP(result[8:24]))
-	assert.Equal(t, src, net.IP(result[24:40]))
-	// ICMPv6 Echo Reply type
-	assert.Equal(t, byte(129), result[40])
-
-	// Verify checksum is valid (tcpipChecksum returns 0 when data+checksum is correct)
-	respIcmp := result[40:]
-	verifyCsum := ipv6PseudoheaderChecksum(result[8:24], result[24:40], 58, uint32(payloadLen))
-	assert.Equal(t, uint16(0), tcpipChecksum(respIcmp, verifyCsum))
-}
-
-func TestCreateICMPEchoResponse_IPv6_NotEchoRequest(t *testing.T) {
-	src := net.ParseIP("fd00::1").To16()
-	dst := net.ParseIP("fd00::2").To16()
-
-	packet := make([]byte, 48)
-	packet[0] = 0x60
-	binary.BigEndian.PutUint16(packet[4:], 8)
-	packet[6] = 58
-	packet[7] = 64
-	copy(packet[8:24], src)
-	copy(packet[24:40], dst)
-
-	// ICMPv6 type 1 (Destination Unreachable) - not Echo Request
-	packet[40] = 1
-
-	out := make([]byte, len(packet))
-	result := CreateICMPEchoResponse(packet, out)
-	assert.Nil(t, result)
-}
-
-func TestCreateICMPEchoResponse_IPv6_NotICMPv6(t *testing.T) {
-	src := net.ParseIP("fd00::1").To16()
-	dst := net.ParseIP("fd00::2").To16()
-
-	packet := make([]byte, 48)
-	packet[0] = 0x60
-	binary.BigEndian.PutUint16(packet[4:], 8)
-	packet[6] = 6 // TCP, not ICMPv6
-	packet[7] = 64
-	copy(packet[8:24], src)
-	copy(packet[24:40], dst)
-
-	out := make([]byte, len(packet))
-	result := CreateICMPEchoResponse(packet, out)
-	assert.Nil(t, result)
-}

lighthouse.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"github.com/gaissmai/bart"
+	"github.com/rcrowley/go-metrics"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
@@ -34,6 +35,7 @@ type LightHouse struct {
 
 	myVpnNetworks      []netip.Prefix
 	myVpnNetworksTable *bart.Lite
+	punchConn          udp.Conn
 	punchy             *Punchy
 
 	// Local cache of answers from light houses
@@ -61,7 +63,11 @@ type LightHouse struct {
 	interval     atomic.Int64
 	updateCancel context.CancelFunc
 	ifce         EncWriter
-	nebulaPort   uint32 // 32 bits because protobuf does not have a uint16
+	// bufAlloc lets the lighthouse query/update workers, request handlers
+	// and punchback goroutines acquire correctly sized WireBuffers from
+	// the same pool as the data plane. Set by main.go alongside ifce.
+	bufAlloc   WireBufferAllocator
+	nebulaPort uint32 // 32 bits because protobuf does not have a uint16
 
 	advertiseAddrs atomic.Pointer[[]netip.AddrPort]
 
@@ -73,8 +79,9 @@ type LightHouse struct {
 
 	calculatedRemotes atomic.Pointer[bart.Table[[]*calculatedRemote]] // Maps VpnAddr to []*calculatedRemote
 
-	metrics *MessageMetrics
+	metrics           *MessageMetrics
-	l       *slog.Logger
+	metricHolepunchTx metrics.Counter
+	l                 *slog.Logger
 }
 
 // NewLightHouseFromConfig will build a Lighthouse struct from the values provided in the config object
@@ -102,10 +109,15 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
 		myVpnNetworksTable: cs.myVpnNetworksTable,
 		addrMap:            make(map[netip.Addr]*RemoteList),
 		nebulaPort:         nebulaPort,
+		punchConn:          pc,
 		punchy:             p,
 		updateTrigger:      make(chan struct{}, 1),
 		queryChan:          make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)),
-		l:                  l,
+		// Default to a no-prefix pool so the query/update workers and
+		// request handlers have a working WireBufferAllocator before
+		// main.go wires up the real one from the Interface.
+		bufAlloc: NewWireBufferPool(mtu, 0),
+		l:        l,
 	}
 	lighthouses := make([]netip.Addr, 0)
 	h.lighthouses.Store(&lighthouses)
@@ -114,6 +126,9 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
 
 	if c.GetBool("stats.lighthouse_metrics", false) {
 		h.metrics = newLighthouseMetrics()
+		h.metricHolepunchTx = metrics.GetOrRegisterCounter("messages.tx.holepunch", nil)
+	} else {
+		h.metricHolepunchTx = metrics.NilCounter{}
 	}
 
 	err := h.reload(c, true)
@@ -272,18 +287,16 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
 	//NOTE: many things will get much simpler when we combine static_host_map and lighthouse.hosts in config
 	if initial || c.HasChanged("static_host_map") || c.HasChanged("static_map.cadence") || c.HasChanged("static_map.network") || c.HasChanged("static_map.lookup_timeout") {
 		// Clean up. Entries still in the static_host_map will be re-built.
-		ourselves := lh.myVpnNetworks[0].Addr()
+		// Entries no longer present must have their (possible) background DNS goroutines stopped.
-		oldStaticList := lh.staticList.Load()
+		if existingStaticList := lh.staticList.Load(); existingStaticList != nil {
-		if oldStaticList != nil {
 			lh.RLock()
-			for staticVpnAddr := range *oldStaticList {
+			for staticVpnAddr := range *existingStaticList {
 				if am, ok := lh.addrMap[staticVpnAddr]; ok && am != nil {
-					am.ResetForOwner(ourselves)
+					am.hr.Cancel()
 				}
 			}
 			lh.RUnlock()
 		}
-
 		// Build a new list based on current config.
 		staticList := make(map[netip.Addr]struct{})
 		err := lh.loadStaticMap(c, staticList)
@@ -291,21 +304,6 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
 			return err
 		}
 
-		// For entries removed from static_host_map, stop the DNS goroutine and drop the cached addrs.
-		// All addrs must come from the lighthouses now that it's no longer a static host.
-		if oldStaticList != nil {
-			lh.RLock()
-			for staticVpnAddr := range *oldStaticList {
-				if _, stillStatic := staticList[staticVpnAddr]; stillStatic {
-					continue
-				}
-				if am, ok := lh.addrMap[staticVpnAddr]; ok && am != nil {
-					am.ClearHostnameResults()
-				}
-			}
-			lh.RUnlock()
-		}
-
 		lh.staticList.Store(&staticList)
 		if !initial {
 			if c.HasChanged("static_host_map") {
@@ -768,21 +766,22 @@ func (lh *LightHouse) startQueryWorker() {
 	}
 
 	go func() {
-		nb := make([]byte, 12, 12)
+		// Long-lived per-worker WireBuffer; reused for every lighthouse query
-		out := make([]byte, mtu)
+		// this worker issues for the life of the goroutine.
+		buf := lh.bufAlloc.Acquire()
 
 		for {
 			select {
 			case <-lh.ctx.Done():
 				return
 			case addr := <-lh.queryChan:
-				lh.innerQueryServer(addr, nb, out)
+				lh.innerQueryServer(addr, buf)
 			}
 		}
 	}()
 }
 
-func (lh *LightHouse) innerQueryServer(addr netip.Addr, nb, out []byte) {
+func (lh *LightHouse) innerQueryServer(addr netip.Addr, buf *WireBuffer) {
 	if lh.IsLighthouseAddr(addr) {
 		return
 	}
@@ -831,7 +830,7 @@ func (lh *LightHouse) innerQueryServer(addr netip.Addr, nb, out []byte) {
 				}
 			}
 
-			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v1Query, nb, out)
+			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v1Query, buf)
 			queried++
 
 		} else if v == cert.Version2 {
@@ -850,7 +849,7 @@ func (lh *LightHouse) innerQueryServer(addr netip.Addr, nb, out []byte) {
 				}
 			}
 
-			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v2Query, nb, out)
+			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v2Query, buf)
 			queried++
 
 		} else {
@@ -879,8 +878,12 @@ func (lh *LightHouse) StartUpdateWorker() {
 	go func() {
 		defer clockSource.Stop()
 
+		// Long-lived per-worker WireBuffer; reused across every periodic
+		// update for the life of this goroutine.
+		buf := lh.bufAlloc.Acquire()
+
 		for {
-			lh.SendUpdate()
+			lh.sendUpdate(buf)
 
 			select {
 			case <-updateCtx.Done():
@@ -894,6 +897,15 @@ func (lh *LightHouse) StartUpdateWorker() {
 	}()
 }
 
+// SendUpdate is the public entry point that triggers a one-shot lighthouse
+// update outside the worker loop (e.g. tests or reload paths). It allocates
+// its own WireBuffer since callers don't already own one.
+func (lh *LightHouse) SendUpdate() {
+	buf := lh.bufAlloc.Acquire()
+	defer lh.bufAlloc.Release(buf)
+	lh.sendUpdate(buf)
+}
+
 // TriggerUpdate requests an immediate lighthouse update. This is a non-blocking
 // operation intended to be called after a handshake completes with a lighthouse,
 // so the lighthouse has our current addresses without waiting for the next
@@ -905,7 +917,7 @@ func (lh *LightHouse) TriggerUpdate() {
 	}
 }
 
-func (lh *LightHouse) SendUpdate() {
+func (lh *LightHouse) sendUpdate(buf *WireBuffer) {
 	var v4 []*V4AddrPort
 	var v6 []*V6AddrPort
 
@@ -931,9 +943,6 @@ func (lh *LightHouse) SendUpdate() {
 		}
 	}
 
-	nb := make([]byte, 12, 12)
-	out := make([]byte, mtu)
-
 	var v1Update, v2Update []byte
 	var err error
 	updated := 0
@@ -984,7 +993,7 @@ func (lh *LightHouse) SendUpdate() {
 				}
 			}
 
-			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v1Update, nb, out)
+			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v1Update, buf)
 			updated++
 
 		} else if v == cert.Version2 {
@@ -1013,7 +1022,7 @@ func (lh *LightHouse) SendUpdate() {
 				}
 			}
 
-			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v2Update, nb, out)
+			lh.ifce.SendMessageToVpnAddr(header.LightHouse, 0, lhVpnAddr, v2Update, buf)
 			updated++
 
 		} else {
@@ -1029,9 +1038,11 @@ func (lh *LightHouse) SendUpdate() {
 }
 
 type LightHouseHandler struct {
-	lh   *LightHouse
+	lh *LightHouse
-	nb   []byte
+	// buf is the long-lived per-handler wire scratch. NewRequestHandler is
-	out  []byte
+	// called once per data-plane receive goroutine, so buf is owned by that
+	// goroutine and reused for every lighthouse send the handler issues.
+	buf  *WireBuffer
 	pb   []byte
 	meta *NebulaMeta
 	l    *slog.Logger
@@ -1040,8 +1051,7 @@ type LightHouseHandler struct {
 func (lh *LightHouse) NewRequestHandler() *LightHouseHandler {
 	lhh := &LightHouseHandler{
 		lh:  lh,
-		nb:  make([]byte, 12, 12),
+		buf: lh.bufAlloc.Acquire(),
-		out: make([]byte, mtu),
 		l:   lh.l,
 		pb:  make([]byte, mtu),
 
@@ -1178,7 +1188,7 @@ func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, fromVpnAddrs []neti
 	}
 
 	lhh.lh.metricTx(NebulaMeta_HostQueryReply, 1)
-	w.SendMessageToVpnAddr(header.LightHouse, 0, fromVpnAddrs[0], lhh.pb[:ln], lhh.nb, lhh.out[:0])
+	w.SendMessageToVpnAddr(header.LightHouse, 0, fromVpnAddrs[0], lhh.pb[:ln], lhh.buf)
 
 	lhh.sendHostPunchNotification(n, fromVpnAddrs, queryVpnAddr, w)
 }
@@ -1238,7 +1248,7 @@ func (lhh *LightHouseHandler) sendHostPunchNotification(n *NebulaMeta, fromVpnAd
 	}
 
 	lhh.lh.metricTx(NebulaMeta_HostPunchNotification, 1)
-	w.SendMessageToVpnAddr(header.LightHouse, 0, punchNotifDest, lhh.pb[:ln], lhh.nb, lhh.out[:0])
+	w.SendMessageToVpnAddr(header.LightHouse, 0, punchNotifDest, lhh.pb[:ln], lhh.buf)
 }
 
 func (lhh *LightHouseHandler) coalesceAnswers(v cert.Version, c *cache, n *NebulaMeta) {
@@ -1395,7 +1405,7 @@ func (lhh *LightHouseHandler) handleHostUpdateNotification(n *NebulaMeta, fromVp
 	}
 
 	lhh.lh.metricTx(NebulaMeta_HostUpdateNotificationAck, 1)
-	w.SendMessageToVpnAddr(header.LightHouse, 0, fromVpnAddrs[0], lhh.pb[:ln], lhh.nb, lhh.out[:0])
+	w.SendMessageToVpnAddr(header.LightHouse, 0, fromVpnAddrs[0], lhh.pb[:ln], lhh.buf)
 }
 
 func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, w EncWriter) {
@@ -1416,31 +1426,61 @@ func (lhh *LightHouseHandler) handleHostPunchNotification(n *NebulaMeta, fromVpn
 		return
 	}
 
+	empty := []byte{0}
+	punch := func(vpnPeer netip.AddrPort, logVpnAddr netip.Addr) {
+		if !vpnPeer.IsValid() {
+			return
+		}
+
+		go func() {
+			time.Sleep(lhh.lh.punchy.GetDelay())
+			lhh.lh.metricHolepunchTx.Inc(1)
+			lhh.lh.punchConn.WriteTo(empty, vpnPeer)
+		}()
+
+		if lhh.l.Enabled(context.Background(), slog.LevelDebug) {
+			lhh.l.Debug("Punching",
+				"vpnPeer", vpnPeer,
+				"logVpnAddr", logVpnAddr,
+			)
+		}
+	}
+
 	remoteAllowList := lhh.lh.GetRemoteAllowList()
 	for _, a := range n.Details.V4AddrPorts {
-		if a == nil {
-			continue
-		}
 		b := protoV4AddrPortToNetAddrPort(a)
 		if remoteAllowList.Allow(detailsVpnAddr, b.Addr()) {
-			lhh.lh.punchy.Schedule(b, detailsVpnAddr)
+			punch(b, detailsVpnAddr)
 		}
 	}
 
 	for _, a := range n.Details.V6AddrPorts {
-		if a == nil {
-			continue
-		}
 		b := protoV6AddrPortToNetAddrPort(a)
 		if remoteAllowList.Allow(detailsVpnAddr, b.Addr()) {
-			lhh.lh.punchy.Schedule(b, detailsVpnAddr)
+			punch(b, detailsVpnAddr)
 		}
 	}
 
 	// This sends a nebula test packet to the host trying to contact us. In the case
 	// of a double nat or other difficult scenario, this may help establish
-	// a tunnel. ScheduleRespond is a no-op when punchy.respond is disabled.
+	// a tunnel.
-	lhh.lh.punchy.ScheduleRespond(detailsVpnAddr)
+	if lhh.lh.punchy.GetRespond() {
+		go func() {
+			time.Sleep(lhh.lh.punchy.GetRespondDelay())
+			if lhh.l.Enabled(context.Background(), slog.LevelDebug) {
+				lhh.l.Debug("Sending a nebula test packet",
+					"vpnAddr", detailsVpnAddr,
+				)
+			}
+			// We acquire and release a fresh buf within this goroutine so it
+			// returns to the pool once the punchback send completes. We
+			// should move this into a timerwheel or a single goroutine
+			// managed by a channel.
+			pbuf := lhh.lh.bufAlloc.Acquire()
+			defer lhh.lh.bufAlloc.Release(pbuf)
+			w.SendMessageToVpnAddr(header.Test, header.TestRequest, detailsVpnAddr, []byte(""), pbuf)
+		}()
+	}
 }
 
 func protoAddrToNetAddr(addr *Addr) netip.Addr {
@@ -1460,7 +1500,7 @@ func protoV6AddrPortToNetAddrPort(ap *V6AddrPort) netip.AddrPort {
 	b := [16]byte{}
 	binary.BigEndian.PutUint64(b[:8], ap.Hi)
 	binary.BigEndian.PutUint64(b[8:], ap.Lo)
-	return netip.AddrPortFrom(netip.AddrFrom16(b).Unmap(), uint16(ap.Port))
+	return netip.AddrPortFrom(netip.AddrFrom16(b), uint16(ap.Port))
 }
 
 func netAddrToProtoAddr(addr netip.Addr) *Addr {
@@ -1500,9 +1540,7 @@ func (d *NebulaMetaDetails) GetRelays() []netip.Addr {
 
 	if len(d.RelayVpnAddrs) > 0 {
 		for _, r := range d.RelayVpnAddrs {
-			if r != nil {
+			relays = append(relays, protoAddrToNetAddr(r))
-				relays = append(relays, protoAddrToNetAddr(r))
-			}
 		}
 	}
 	return relays

lighthouse_test.go

@@ -303,132 +303,6 @@ func TestLighthouse_reload(t *testing.T) {
 	require.NoError(t, err)
 }
 
-// TestLighthouse_reloadStaticHostMap verifies that reloading static_host_map applies the new
-// config rather than appending to it. See issue #718.
-func TestLighthouse_reloadStaticHostMap(t *testing.T) {
-	l := test.NewLogger()
-	c := config.NewC(l)
-	c.Settings["lighthouse"] = map[string]any{"am_lighthouse": true}
-	c.Settings["listen"] = map[string]any{"port": 4242}
-	c.Settings["static_host_map"] = map[string]any{
-		"10.128.0.2": []any{"1.1.1.1:4242"},
-	}
-
-	myVpnNet := netip.MustParsePrefix("10.128.0.1/24")
-	nt := new(bart.Lite)
-	nt.Insert(myVpnNet)
-	cs := &CertState{
-		myVpnNetworks:      []netip.Prefix{myVpnNet},
-		myVpnNetworksTable: nt,
-	}
-
-	lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
-	require.NoError(t, err)
-
-	staticHost := netip.MustParseAddr("10.128.0.2")
-	otherHost := netip.MustParseAddr("10.128.0.3")
-
-	// Capture the RemoteList pointer up front; an in-flight handshake would hold the same one
-	// on hostinfo.remotes, so it must reflect every reload below.
-	pinned := lh.Query(staticHost)
-	require.NotNil(t, pinned)
-	assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}, pinned.CopyAddrs([]netip.Prefix{}))
-
-	// Replace the remote address. The new address should be the only entry.
-	nc := map[string]any{
-		"static_host_map": map[string]any{
-			"10.128.0.2": []any{"2.2.2.2:4242"},
-		},
-	}
-	rc, err := yaml.Marshal(nc)
-	require.NoError(t, err)
-	require.NoError(t, c.ReloadConfigString(string(rc)))
-
-	rl := lh.Query(staticHost)
-	require.NotNil(t, rl)
-	assert.Same(t, pinned, rl, "RemoteList pointer must stay stable so in-flight handshakes pick up the change")
-	assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("2.2.2.2:4242")}, rl.CopyAddrs([]netip.Prefix{}))
-
-	// Reload back to the original IP. Mirrors the round-trip in issue #718 step 6-8 where
-	// the buggy reload produced [1.1.1.1, 2.2.2.2, 1.1.1.1] instead of [1.1.1.1].
-	nc = map[string]any{
-		"static_host_map": map[string]any{
-			"10.128.0.2": []any{"1.1.1.1:4242"},
-		},
-	}
-	rc, err = yaml.Marshal(nc)
-	require.NoError(t, err)
-	require.NoError(t, c.ReloadConfigString(string(rc)))
-
-	rl = lh.Query(staticHost)
-	require.NotNil(t, rl)
-	assert.Same(t, pinned, rl)
-	assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}, rl.CopyAddrs([]netip.Prefix{}))
-
-	// Reload with the same config. An unchanged entry must not duplicate.
-	require.NoError(t, c.ReloadConfigString(string(rc)))
-
-	rl = lh.Query(staticHost)
-	require.NotNil(t, rl)
-	assert.Same(t, pinned, rl)
-	assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}, rl.CopyAddrs([]netip.Prefix{}))
-
-	// Switch back to 2.2.2.2 so the rest of the test continues against a known address.
-	nc = map[string]any{
-		"static_host_map": map[string]any{
-			"10.128.0.2": []any{"2.2.2.2:4242"},
-		},
-	}
-	rc, err = yaml.Marshal(nc)
-	require.NoError(t, err)
-	require.NoError(t, c.ReloadConfigString(string(rc)))
-
-	// Add a second host alongside the first. Both should be present, neither duplicated.
-	nc = map[string]any{
-		"static_host_map": map[string]any{
-			"10.128.0.2": []any{"2.2.2.2:4242"},
-			"10.128.0.3": []any{"3.3.3.3:4242"},
-		},
-	}
-	rc, err = yaml.Marshal(nc)
-	require.NoError(t, err)
-	require.NoError(t, c.ReloadConfigString(string(rc)))
-
-	rl = lh.Query(staticHost)
-	require.NotNil(t, rl)
-	assert.Same(t, pinned, rl, "adding a sibling entry must not displace the existing RemoteList")
-	assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("2.2.2.2:4242")}, rl.CopyAddrs([]netip.Prefix{}))
-
-	rl = lh.Query(otherHost)
-	require.NotNil(t, rl)
-	assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("3.3.3.3:4242")}, rl.CopyAddrs([]netip.Prefix{}))
-
-	// Drop the first host entirely. The vpnAddr is no longer marked static, our owner
-	// contribution is cleared, but the addrMap entry stays in place so non-static cache
-	// data (from lighthouse queries) on the same RemoteList isn't lost. In-flight handshakes
-	// that already had the pointer see an empty address list rather than retrying stale ones.
-	nc = map[string]any{
-		"static_host_map": map[string]any{
-			"10.128.0.3": []any{"3.3.3.3:4242"},
-		},
-	}
-	rc, err = yaml.Marshal(nc)
-	require.NoError(t, err)
-	require.NoError(t, c.ReloadConfigString(string(rc)))
-
-	_, isStatic := lh.GetStaticHostList()[staticHost]
-	assert.False(t, isStatic)
-
-	rl = lh.Query(staticHost)
-	require.NotNil(t, rl)
-	assert.Same(t, pinned, rl)
-	assert.Empty(t, rl.CopyAddrs([]netip.Prefix{}))
-
-	rl = lh.Query(otherHost)
-	require.NotNil(t, rl)
-	assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("3.3.3.3:4242")}, rl.CopyAddrs([]netip.Prefix{}))
-}
-
 func newLHHostRequest(fromAddr netip.AddrPort, myVpnIp, queryVpnIp netip.Addr, lhh *LightHouseHandler) testLhReply {
 	req := &NebulaMeta{
 		Type:    NebulaMeta_HostQuery,
@@ -498,12 +372,12 @@ type testEncWriter struct {
 	protocolVersion cert.Version
 }
 
-func (tw *testEncWriter) SendVia(via *HostInfo, relay *Relay, ad, nb, out []byte, nocopy bool) {
+func (tw *testEncWriter) SendVia(via *HostInfo, relay *Relay, ad []byte, buf *WireBuffer) {
 }
 func (tw *testEncWriter) Handshake(vpnIp netip.Addr) {
 }
 
-func (tw *testEncWriter) SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p, _, _ []byte) {
+func (tw *testEncWriter) SendMessageToHostInfo(t header.MessageType, st header.MessageSubType, hostinfo *HostInfo, p []byte, _ *WireBuffer) {
 	msg := &NebulaMeta{}
 	err := msg.Unmarshal(p)
 	if tw.metaFilter == nil || msg.Type == *tw.metaFilter {
@@ -520,7 +394,7 @@ func (tw *testEncWriter) SendMessageToHostInfo(t header.MessageType, st header.M
 	}
 }
 
-func (tw *testEncWriter) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnIp netip.Addr, p, _, _ []byte) {
+func (tw *testEncWriter) SendMessageToVpnAddr(t header.MessageType, st header.MessageSubType, vpnIp netip.Addr, p []byte, _ *WireBuffer) {
 	msg := &NebulaMeta{}
 	err := msg.Unmarshal(p)
 	if tw.metaFilter == nil || msg.Type == *tw.metaFilter {

main.go

@@ -55,7 +55,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 	}
 	l.Info("Firewall started", "firewallHashes", fw.GetRuleHashes())
 
-	ssh, err := sshd.NewSSHServer(ctx, l.With("subsystem", "sshd"))
+	ssh, err := sshd.NewSSHServer(l.With("subsystem", "sshd"))
 	if err != nil {
 		return nil, util.ContextualizeIfNeeded("Error while creating SSH server", err)
 	}
@@ -170,7 +170,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 	}
 
 	hostMap := NewHostMapFromConfig(l, c)
-	punchy := NewPunchyFromConfig(l, c, udpConns[0])
+	punchy := NewPunchyFromConfig(l, c)
 	connManager := newConnectionManagerFromConfig(l, c, hostMap, punchy)
 	lightHouse, err := NewLightHouseFromConfig(ctx, l, c, pki.getCertState(), udpConns[0], punchy)
 	if err != nil {
@@ -194,7 +194,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 	handshakeManager := NewHandshakeManager(l, hostMap, lightHouse, udpConns[0], handshakeConfig)
 	lightHouse.handshakeTrigger = handshakeManager.trigger
 
-	ds, err := newDnsServerFromConfig(ctx, l, pki, hostMap, c)
+	ds, err := newDnsServerFromConfig(ctx, l, pki.getCertState(), hostMap, c)
 	if err != nil {
 		l.Warn("Failed to start DNS responder", "error", err)
 	}
@@ -232,6 +232,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 
 		ifce.writers = udpConns
 		lightHouse.ifce = ifce
+		lightHouse.bufAlloc = ifce.bufAlloc
 
 		ifce.RegisterConfigChangeCallbacks(c)
 		ifce.reloadDisconnectInvalid(c)
@@ -240,8 +241,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
 
 		handshakeManager.f = ifce
 		go handshakeManager.Run(ctx)
-
-		punchy.Start(ctx, ifce, hostMap, lightHouse)
 	}
 
 	stats, err := newStatsServerFromConfig(ctx, l, c, buildVersion, configTest)

message_metrics.go

@@ -13,8 +13,6 @@ type MessageMetrics struct {
 
 	rxUnknown metrics.Counter
 	txUnknown metrics.Counter
-
-	rxInvalid metrics.Counter
 }
 
 func (m *MessageMetrics) Rx(t header.MessageType, s header.MessageSubType, i int64) {
@@ -35,11 +33,6 @@ func (m *MessageMetrics) Tx(t header.MessageType, s header.MessageSubType, i int
 		}
 	}
 }
-func (m *MessageMetrics) RxInvalid(i int64) {
-	if m != nil && m.rxInvalid != nil {
-		m.rxInvalid.Inc(i)
-	}
-}
 
 func newMessageMetrics() *MessageMetrics {
 	gen := func(t string) [][]metrics.Counter {
@@ -63,7 +56,6 @@ func newMessageMetrics() *MessageMetrics {
 
 		rxUnknown: metrics.GetOrRegisterCounter("messages.rx.other", nil),
 		txUnknown: metrics.GetOrRegisterCounter("messages.tx.other", nil),
-		rxInvalid: metrics.GetOrRegisterCounter("messages.rx.invalid", nil),
 	}
 }
 

noise.go

@@ -0,0 +1,86 @@
+package nebula
+
+import (
+	"crypto/cipher"
+	"encoding/binary"
+	"errors"
+
+	"github.com/flynn/noise"
+)
+
+type endianness interface {
+	PutUint64(b []byte, v uint64)
+}
+
+var noiseEndianness endianness = binary.BigEndian
+
+// NonceSize is the AEAD nonce length used by all ciphers nebula supports
+// today (AES-GCM and ChaCha20-Poly1305 both use 96-bit nonces). Encrypt-
+// and DecryptDanger lay out the nonce as 4 zero bytes followed by an 8-byte
+// big-endian counter; if a future cipher with a different nonce size is
+// added, this constant and those layouts must change together.
+const NonceSize = 12
+
+// AEADOverhead is the AEAD authentication tag length the ciphers nebula
+// supports append to ciphertext. Both AES-GCM and ChaCha20-Poly1305 use
+// 128-bit tags. NebulaCipherState.Overhead() returns this dynamically from
+// the cipher; the constant is for sizing buffers at construction time.
+const AEADOverhead = 16
+
+type NebulaCipherState struct {
+	c cipher.AEAD
+}
+
+func NewNebulaCipherState(s *noise.CipherState) *NebulaCipherState {
+	x := s.Cipher()
+	return &NebulaCipherState{c: x.(cipher.AEAD)}
+}
+
+// EncryptDanger encrypts and authenticates a given payload.
+//
+// out is a destination slice to hold the output of the EncryptDanger operation.
+//   - ad is additional data, which will be authenticated and appended to out, but not encrypted.
+//   - plaintext is encrypted, authenticated and appended to out.
+//   - n is a nonce value which must never be re-used with this key.
+//   - nb is a buffer used for temporary storage in the implementation of this call, which should
+//     be re-used by callers to minimize garbage collection.
+func (s *NebulaCipherState) EncryptDanger(out, ad, plaintext []byte, n uint64, nb []byte) ([]byte, error) {
+	if s != nil {
+		// TODO: Is this okay now that we have made messageCounter atomic?
+		// Alternative may be to split the counter space into ranges
+		//if n <= s.n {
+		//	return nil, errors.New("CRITICAL: a duplicate counter value was used")
+		//}
+		//s.n = n
+		nb[0] = 0
+		nb[1] = 0
+		nb[2] = 0
+		nb[3] = 0
+		noiseEndianness.PutUint64(nb[4:], n)
+		out = s.c.Seal(out, nb, plaintext, ad)
+		//l.Debugf("Encryption: outlen: %d, nonce: %d, ad: %s, plainlen %d", len(out), n, ad, len(plaintext))
+		return out, nil
+	} else {
+		return nil, errors.New("no cipher state available to encrypt")
+	}
+}
+
+func (s *NebulaCipherState) DecryptDanger(out, ad, ciphertext []byte, n uint64, nb []byte) ([]byte, error) {
+	if s != nil {
+		nb[0] = 0
+		nb[1] = 0
+		nb[2] = 0
+		nb[3] = 0
+		noiseEndianness.PutUint64(nb[4:], n)
+		return s.c.Open(out, nb, ciphertext, ad)
+	} else {
+		return []byte{}, nil
+	}
+}
+
+func (s *NebulaCipherState) Overhead() int {
+	if s != nil {
+		return s.c.Overhead()
+	}
+	return 0
+}

noiseutil/aesgcm.go

@@ -1,53 +0,0 @@
-package noiseutil
-
-import (
-	"crypto/cipher"
-	"encoding/binary"
-	"errors"
-
-	"github.com/flynn/noise"
-)
-
-// CipherStateAESGCM is the data-plane wrapper for the AES-GCM AEAD cipher.
-// AES-GCM uses big-endian nonce encoding per the Noise spec.
-type CipherStateAESGCM struct {
-	c cipher.AEAD
-}
-
-// NewCipherStateAESGCM extracts the underlying AEAD from the post-handshake noise.CipherState.
-// The caller is responsible for ensuring the noise cipher is actually AES-GCM,
-// otherwise the type assertion still succeeds but the nonce endianness will be wrong on the wire.
-func NewCipherStateAESGCM(s *noise.CipherState) *CipherStateAESGCM {
-	return &CipherStateAESGCM{c: s.Cipher().(cipher.AEAD)}
-}
-
-func (s *CipherStateAESGCM) EncryptDanger(out, ad, plaintext []byte, n uint64, nb []byte) ([]byte, error) {
-	if s == nil {
-		return nil, errors.New("no cipher state available to encrypt")
-	}
-	nb[0] = 0
-	nb[1] = 0
-	nb[2] = 0
-	nb[3] = 0
-	binary.BigEndian.PutUint64(nb[4:], n)
-	return s.c.Seal(out, nb, plaintext, ad), nil
-}
-
-func (s *CipherStateAESGCM) DecryptDanger(out, ad, ciphertext []byte, n uint64, nb []byte) ([]byte, error) {
-	if s == nil {
-		return []byte{}, nil
-	}
-	nb[0] = 0
-	nb[1] = 0
-	nb[2] = 0
-	nb[3] = 0
-	binary.BigEndian.PutUint64(nb[4:], n)
-	return s.c.Open(out, nb, ciphertext, ad)
-}
-
-func (s *CipherStateAESGCM) Overhead() int {
-	if s == nil {
-		return 0
-	}
-	return s.c.Overhead()
-}

noiseutil/chachapoly.go

@@ -1,52 +0,0 @@
-package noiseutil
-
-import (
-	"crypto/cipher"
-	"encoding/binary"
-	"errors"
-
-	"github.com/flynn/noise"
-)
-
-// CipherStateChaChaPoly is the data-plane wrapper for the ChaCha20-Poly1305 AEAD cipher.
-// ChaCha20-Poly1305 uses little-endian nonce encoding per the Noise spec.
-type CipherStateChaChaPoly struct {
-	c cipher.AEAD
-}
-
-// NewCipherStateChaChaPoly extracts the underlying AEAD from the post-handshake noise.CipherState.
-// The caller is responsible for ensuring the noise cipher is actually ChaCha20-Poly1305.
-func NewCipherStateChaChaPoly(s *noise.CipherState) *CipherStateChaChaPoly {
-	return &CipherStateChaChaPoly{c: s.Cipher().(cipher.AEAD)}
-}
-
-func (s *CipherStateChaChaPoly) EncryptDanger(out, ad, plaintext []byte, n uint64, nb []byte) ([]byte, error) {
-	if s == nil {
-		return nil, errors.New("no cipher state available to encrypt")
-	}
-	nb[0] = 0
-	nb[1] = 0
-	nb[2] = 0
-	nb[3] = 0
-	binary.LittleEndian.PutUint64(nb[4:], n)
-	return s.c.Seal(out, nb, plaintext, ad), nil
-}
-
-func (s *CipherStateChaChaPoly) DecryptDanger(out, ad, ciphertext []byte, n uint64, nb []byte) ([]byte, error) {
-	if s == nil {
-		return []byte{}, nil
-	}
-	nb[0] = 0
-	nb[1] = 0
-	nb[2] = 0
-	nb[3] = 0
-	binary.LittleEndian.PutUint64(nb[4:], n)
-	return s.c.Open(out, nb, ciphertext, ad)
-}
-
-func (s *CipherStateChaChaPoly) Overhead() int {
-	if s == nil {
-		return 0
-	}
-	return s.c.Overhead()
-}

noiseutil/cipher_state.go

@@ -1,40 +0,0 @@
-package noiseutil
-
-import (
-	"fmt"
-
-	"github.com/flynn/noise"
-)
-
-// CipherState is the post-handshake AEAD cipher used for the data plane.
-// Each supported cipher has its own concrete implementation in this package with the nonce endianness hardcoded,
-// so the encrypt/decrypt fast path avoids interface dispatch on the byte order.
-type CipherState interface {
-	// EncryptDanger encrypts and authenticates a given payload.
-	//
-	// out is a destination slice to hold the output of the EncryptDanger operation.
-	//   - ad is additional data, which will be authenticated and appended to out, but not encrypted.
-	//   - plaintext is encrypted, authenticated and appended to out.
-	//   - n is a nonce value which must never be re-used with this key.
-	//   - nb is a scratch buffer used to assemble the nonce.
-	EncryptDanger(out, ad, plaintext []byte, n uint64, nb []byte) ([]byte, error)
-
-	// DecryptDanger authenticates and decrypts a given payload, with the same argument shape as EncryptDanger.
-	DecryptDanger(out, ad, ciphertext []byte, n uint64, nb []byte) ([]byte, error)
-
-	// Overhead returns the AEAD tag size, or 0 if the receiver is nil.
-	Overhead() int
-}
-
-// NewCipherState wraps the post-handshake noise.CipherState in the per-cipher type that matches cipherFunc.
-// cipherFunc must be the same cipher used to build the noise CipherSuite that produced s.
-func NewCipherState(s *noise.CipherState, cipherFunc noise.CipherFunc) CipherState {
-	switch cipherFunc.CipherName() {
-	case CipherAESGCM.CipherName():
-		return NewCipherStateAESGCM(s)
-	case noise.CipherChaChaPoly.CipherName():
-		return NewCipherStateChaChaPoly(s)
-	default:
-		panic(fmt.Sprintf("noiseutil: unsupported cipher %q", cipherFunc.CipherName()))
-	}
-}

noiseutil/cipher_state_test.go

@@ -1,166 +0,0 @@
-package noiseutil
-
-import (
-	"testing"
-
-	"github.com/flynn/noise"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-)
-
-func TestCipherStateAESGCMRoundtrip(t *testing.T) {
-	enc, dec := buildCipherStates(t, CipherAESGCM)
-	roundtrip(t, NewCipherStateAESGCM(enc), NewCipherStateAESGCM(dec))
-}
-
-func TestCipherStateChaChaPolyRoundtrip(t *testing.T) {
-	enc, dec := buildCipherStates(t, noise.CipherChaChaPoly)
-	roundtrip(t, NewCipherStateChaChaPoly(enc), NewCipherStateChaChaPoly(dec))
-}
-
-func TestNewCipherStateDispatch(t *testing.T) {
-	encA, _ := buildCipherStates(t, CipherAESGCM)
-	encC, _ := buildCipherStates(t, noise.CipherChaChaPoly)
-
-	assert.IsType(t, &CipherStateAESGCM{}, NewCipherState(encA, CipherAESGCM))
-	assert.IsType(t, &CipherStateChaChaPoly{}, NewCipherState(encC, noise.CipherChaChaPoly))
-}
-
-func TestNewCipherStateUnsupportedPanics(t *testing.T) {
-	enc, _ := buildCipherStates(t, CipherAESGCM)
-	assert.Panics(t, func() {
-		NewCipherState(enc, fakeCipher{})
-	})
-}
-
-type fakeCipher struct{}
-
-func (fakeCipher) Cipher(k [32]byte) noise.Cipher { return nil }
-func (fakeCipher) CipherName() string             { return "Fake" }
-
-// buildCipherStates runs an in-memory NN handshake with the requested cipher
-// to produce a pair of post-handshake CipherStates that share keys.
-func buildCipherStates(t *testing.T, c noise.CipherFunc) (*noise.CipherState, *noise.CipherState) {
-	t.Helper()
-	suite := noise.NewCipherSuite(noise.DH25519, c, noise.HashSHA256)
-	cfg := noise.Config{CipherSuite: suite, Pattern: noise.HandshakeNN}
-	cfg.Initiator = true
-	hsI, err := noise.NewHandshakeState(cfg)
-	require.NoError(t, err)
-	cfg.Initiator = false
-	hsR, err := noise.NewHandshakeState(cfg)
-	require.NoError(t, err)
-
-	msg, _, _, err := hsI.WriteMessage(nil, nil)
-	require.NoError(t, err)
-	_, _, _, err = hsR.ReadMessage(nil, msg)
-	require.NoError(t, err)
-
-	msg, dR, _, err := hsR.WriteMessage(nil, nil)
-	require.NoError(t, err)
-	_, eI, _, err := hsI.ReadMessage(nil, msg)
-	require.NoError(t, err)
-	require.NotNil(t, eI)
-	require.NotNil(t, dR)
-
-	// noise returns (cs1, cs2) where cs1 is the initiator->responder cipher.
-	return eI, dR
-}
-
-func roundtrip(t *testing.T, enc, dec CipherState) {
-	t.Helper()
-	plaintext := []byte("nebula cipher state roundtrip")
-	ad := []byte("aad")
-	nb := make([]byte, 12)
-
-	ct, err := enc.EncryptDanger(nil, ad, plaintext, 1, nb)
-	require.NoError(t, err)
-	assert.NotEqual(t, plaintext, ct)
-
-	pt, err := dec.DecryptDanger(nil, ad, ct, 1, nb)
-	require.NoError(t, err)
-	assert.Equal(t, plaintext, pt)
-
-	// Wrong nonce must fail authentication.
-	_, err = dec.DecryptDanger(nil, ad, ct, 2, nb)
-	require.Error(t, err)
-
-	assert.Equal(t, enc.Overhead(), dec.Overhead())
-	assert.Equal(t, 16, enc.Overhead())
-}
-
-func BenchmarkCipherStateEncryptAESGCM(b *testing.B) {
-	enc, _ := buildCipherStatesB(b, CipherAESGCM)
-	benchEncryptCipherState(b, NewCipherState(enc, CipherAESGCM))
-}
-
-func BenchmarkCipherStateEncryptChaChaPoly(b *testing.B) {
-	enc, _ := buildCipherStatesB(b, noise.CipherChaChaPoly)
-	benchEncryptCipherState(b, NewCipherState(enc, noise.CipherChaChaPoly))
-}
-
-func benchEncryptCipherState(b *testing.B, cs CipherState) {
-	plaintext := make([]byte, 1280)
-	ad := make([]byte, 16)
-	nb := make([]byte, 12)
-	out := make([]byte, 0, len(plaintext)+cs.Overhead())
-	b.ResetTimer()
-	b.ReportAllocs()
-	for i := 0; i < b.N; i++ {
-		var err error
-		out, err = cs.EncryptDanger(out[:0], ad, plaintext, uint64(i+1), nb)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func buildCipherStatesB(b *testing.B, c noise.CipherFunc) (*noise.CipherState, *noise.CipherState) {
-	b.Helper()
-	suite := noise.NewCipherSuite(noise.DH25519, c, noise.HashSHA256)
-	cfg := noise.Config{CipherSuite: suite, Pattern: noise.HandshakeNN}
-	cfg.Initiator = true
-	hsI, err := noise.NewHandshakeState(cfg)
-	if err != nil {
-		b.Fatal(err)
-	}
-	cfg.Initiator = false
-	hsR, err := noise.NewHandshakeState(cfg)
-	if err != nil {
-		b.Fatal(err)
-	}
-	msg, _, _, err := hsI.WriteMessage(nil, nil)
-	if err != nil {
-		b.Fatal(err)
-	}
-	if _, _, _, err := hsR.ReadMessage(nil, msg); err != nil {
-		b.Fatal(err)
-	}
-	msg, dR, _, err := hsR.WriteMessage(nil, nil)
-	if err != nil {
-		b.Fatal(err)
-	}
-	_, eI, _, err := hsI.ReadMessage(nil, msg)
-	if err != nil {
-		b.Fatal(err)
-	}
-	return eI, dR
-}
-
-func TestCipherStateNilSafety(t *testing.T) {
-	var aes *CipherStateAESGCM
-	_, err := aes.EncryptDanger(nil, nil, nil, 0, make([]byte, 12))
-	require.Error(t, err)
-	out, err := aes.DecryptDanger(nil, nil, nil, 0, make([]byte, 12))
-	require.NoError(t, err)
-	assert.Empty(t, out)
-	assert.Equal(t, 0, aes.Overhead())
-
-	var cc *CipherStateChaChaPoly
-	_, err = cc.EncryptDanger(nil, nil, nil, 0, make([]byte, 12))
-	require.Error(t, err)
-	out, err = cc.DecryptDanger(nil, nil, nil, 0, make([]byte, 12))
-	require.NoError(t, err)
-	assert.Empty(t, out)
-	assert.Equal(t, 0, cc.Overhead())
-}

outside.go

@@ -20,46 +20,24 @@ const (
 	minFwPacketLen = 4
 )
 
-var ErrOutOfWindow = errors.New("out of window packet")
+func (f *Interface) readOutsidePackets(via ViaSender, buf *WireBuffer, packet []byte, lhf *LightHouseHandler, q int, localCache firewall.ConntrackCache) {
-
+	h := buf.H
-func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
 	err := h.Parse(packet)
 	if err != nil {
 		// Hole punch packets are 0 or 1 byte big, so lets ignore printing those errors
-		// TODO: record metrics for rx holepunch/punchy packets?
 		if len(packet) > 1 {
-			f.messageMetrics.RxInvalid(1)
+			f.l.Info("Error while parsing inbound packet",
-			if f.l.Enabled(context.Background(), slog.LevelDebug) {
+				"from", via,
-				f.l.Debug("Error while parsing inbound packet",
+				"error", err,
-					"from", via,
+				"packet", packet,
-					"error", err,
+			)
-					"packet", packet,
-				)
-			}
-		}
-		return
-	}
-
-	if h.Version != header.Version {
-		f.messageMetrics.RxInvalid(1)
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			f.l.Debug("Unexpected header version received", "from", via)
-		}
-		return
-	}
-
-	// Check before processing to see if this is a expected type/subtype
-	if !h.IsValidSubType() {
-		f.messageMetrics.RxInvalid(1)
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			f.l.Debug("Unexpected packet received", "from", via)
 		}
 		return
 	}
 
+	//l.Error("in packet ", header, packet[HeaderLen:])
 	if !via.IsRelayed {
 		if f.myVpnNetworksTable.Contains(via.UdpAddr.Addr()) {
-			f.messageMetrics.RxInvalid(1)
 			if f.l.Enabled(context.Background(), slog.LevelDebug) {
 				f.l.Debug("Refusing to process double encrypted packet", "from", via)
 			}
@@ -67,199 +45,217 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
 		}
 	}
 
-	// don't keep Rx metrics for message type, since you can see those in the tun metrics
-	if h.Type != header.Message {
-		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-	}
-
-	// Unencrypted packets
-	switch h.Type {
-	case header.Handshake:
-		f.handshakeManager.HandleIncoming(via, packet, h)
-		return
-
-	case header.RecvError:
-		f.handleRecvError(via.UdpAddr, h)
-		return
-	}
-
-	// Relay packets are special
-	isMessageRelay := (h.Type == header.Message && h.Subtype == header.MessageRelay)
-
 	var hostinfo *HostInfo
-	if isMessageRelay {
+	// verify if we've seen this index before, otherwise respond to the handshake initiation
+	if h.Type == header.Message && h.Subtype == header.MessageRelay {
 		hostinfo = f.hostMap.QueryRelayIndex(h.RemoteIndex)
 	} else {
 		hostinfo = f.hostMap.QueryIndex(h.RemoteIndex)
 	}
 
-	// At this point we should have a valid existing tunnel, verify and send
+	var ci *ConnectionState
-	// recvError if necessary
+	if hostinfo != nil {
-	if hostinfo == nil || hostinfo.ConnectionState == nil {
+		ci = hostinfo.ConnectionState
-		if !via.IsRelayed {
-			f.maybeSendRecvError(via.UdpAddr, h.RemoteIndex)
-		}
-		return
 	}
 
-	// All remaining packets are encrypted
-	ci := hostinfo.ConnectionState
-	if !ci.window.Check(f.l, h.MessageCounter) {
-		return
-	}
-
-	// Relay packets are special
-	if isMessageRelay {
-		f.handleOutsideRelayPacket(hostinfo, via, out, packet, h, fwPacket, lhf, nb, q, localCache)
-
-		return
-	}
-
-	out, err = f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
-	if err != nil {
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			hostinfo.logger(f.l).Debug("Failed to decrypt packet",
-				"error", err,
-				"from", via,
-				"header", h,
-			)
-		}
-		return
-	}
-
-	// Roam before we respond
-	f.handleHostRoaming(hostinfo, via)
-	f.connectionManager.In(hostinfo)
-
 	switch h.Type {
 	case header.Message:
+		if !f.handleEncrypted(ci, via, h) {
+			return
+		}
+
 		switch h.Subtype {
 		case header.MessageNone:
-			f.handleOutsideMessagePacket(hostinfo, out, packet, fwPacket, nb, q, localCache)
+			if !f.decryptToTun(hostinfo, h.MessageCounter, buf, packet, q, localCache) {
-		default:
+				return
-			hostinfo.logger(f.l).Error("IsValidSubType was true, but unexpected message subtype seen", "from", via, "header", h)
+			}
-			return
+		case header.MessageRelay:
+			// The entire body is sent as AD, not encrypted.
+			// The packet consists of a 16-byte parsed Nebula header, Associated Data-protected payload, and a trailing 16-byte AEAD signature value.
+			// The packet is guaranteed to be at least 16 bytes at this point, b/c it got past the h.Parse() call above. If it's
+			// otherwise malformed (meaning, there is no trailing 16 byte AEAD value), then this will result in at worst a 0-length slice
+			// which will gracefully fail in the DecryptDanger call.
+			signedPayload := packet[:len(packet)-hostinfo.ConnectionState.dKey.Overhead()]
+			signatureValue := packet[len(packet)-hostinfo.ConnectionState.dKey.Overhead():]
+			// AAD-only validation: passing dst=nil since there's no plaintext
+			// to recover (ciphertext is just the trailing AEAD tag).
+			if _, err = hostinfo.ConnectionState.dKey.DecryptDanger(nil, signedPayload, signatureValue, h.MessageCounter, buf.NB); err != nil {
+				return
+			}
+			// Successfully validated the thing. Get rid of the Relay header.
+			signedPayload = signedPayload[header.Len:]
+			// Pull the Roaming parts up here, and return in all call paths.
+			f.handleHostRoaming(hostinfo, via)
+			// Track usage of both the HostInfo and the Relay for the received & authenticated packet
+			f.connectionManager.In(hostinfo)
+			f.connectionManager.RelayUsed(h.RemoteIndex)
+
+			relay, ok := hostinfo.relayState.QueryRelayForByIdx(h.RemoteIndex)
+			if !ok {
+				// The only way this happens is if hostmap has an index to the correct HostInfo, but the HostInfo is missing
+				// its internal mapping. This should never happen.
+				hostinfo.logger(f.l).Error("HostInfo missing remote relay index",
+					"vpnAddrs", hostinfo.vpnAddrs,
+					"remoteIndex", h.RemoteIndex,
+				)
+				return
+			}
+
+			switch relay.Type {
+			case TerminalType:
+				// If I am the target of this relay, process the unwrapped packet
+				// From this recursive point, all these variables are 'burned'. We shouldn't rely on them again.
+				via = ViaSender{
+					UdpAddr:   via.UdpAddr,
+					relayHI:   hostinfo,
+					remoteIdx: relay.RemoteIndex,
+					relay:     relay,
+					IsRelayed: true,
+				}
+				buf.Reset()
+				f.readOutsidePackets(via, buf, signedPayload, lhf, q, localCache)
+				return
+			case ForwardingType:
+				// Find the target HostInfo relay object
+				targetHI, targetRelay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relay.PeerAddr)
+				if err != nil {
+					hostinfo.logger(f.l).Info("Failed to find target host info by ip",
+						"relayTo", relay.PeerAddr,
+						"error", err,
+						"hostinfo.vpnAddrs", hostinfo.vpnAddrs,
+					)
+					return
+				}
+
+				// If that relay is Established, forward the payload through it
+				if targetRelay.State == Established {
+					switch targetRelay.Type {
+					case ForwardingType:
+						// Forward this packet through the relay tunnel
+						// Find the target HostInfo
+						f.SendVia(targetHI, targetRelay, signedPayload, buf)
+						return
+					case TerminalType:
+						hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
+					}
+				} else {
+					hostinfo.logger(f.l).Info("Unexpected target relay state",
+						"relayTo", relay.PeerAddr,
+						"relayFrom", hostinfo.vpnAddrs[0],
+						"targetRelayState", targetRelay.State,
+					)
+					return
+				}
+			}
 		}
 
 	case header.LightHouse:
+		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
+		if !f.handleEncrypted(ci, via, h) {
+			return
+		}
+
+		d, err := f.decrypt(hostinfo, h.MessageCounter, buf, packet, h)
+		if err != nil {
+			hostinfo.logger(f.l).Error("Failed to decrypt lighthouse packet",
+				"error", err,
+				"from", via,
+				"packet", packet,
+			)
+			return
+		}
+
 		//TODO: assert via is not relayed
-		lhf.HandleRequest(via.UdpAddr, hostinfo.vpnAddrs, out, f)
+		lhf.HandleRequest(via.UdpAddr, hostinfo.vpnAddrs, d, f)
+
+		// Fallthrough to the bottom to record incoming traffic
 
 	case header.Test:
-		switch h.Subtype {
+		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
-		case header.TestReply:
+		if !f.handleEncrypted(ci, via, h) {
-			// No-op, useful for the Roaming and connectionManager side-effects above
-		case header.TestRequest:
-			//recycle the input packet ciphertext as our output buffer
-			f.send(header.Test, header.TestReply, ci, hostinfo, out, nb, packet)
-		default:
-			hostinfo.logger(f.l).Error("IsValidSubType was true, but unexpected test subtype seen", "from", via, "header", h)
 			return
 		}
 
+		d, err := f.decrypt(hostinfo, h.MessageCounter, buf, packet, h)
+		if err != nil {
+			hostinfo.logger(f.l).Error("Failed to decrypt test packet",
+				"error", err,
+				"from", via,
+				"packet", packet,
+			)
+			return
+		}
+
+		if h.Subtype == header.TestRequest {
+			// This testRequest might be from TryPromoteBest, so we should roam
+			// to the new IP address before responding.
+			f.handleHostRoaming(hostinfo, via)
+			f.send(header.Test, header.TestReply, ci, hostinfo, d, buf)
+		}
+
+		// Fallthrough to the bottom to record incoming traffic
+
+		// Non encrypted messages below here, they should not fall through to avoid tracking incoming traffic since they
+		// are unauthenticated
+
+	case header.Handshake:
+		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
+		f.handshakeManager.HandleIncoming(via, packet, h)
+		return
+
+	case header.RecvError:
+		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
+		f.handleRecvError(via.UdpAddr, h)
+		return
+
 	case header.CloseTunnel:
+		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
+		if !f.handleEncrypted(ci, via, h) {
+			return
+		}
+		_, err = f.decrypt(hostinfo, h.MessageCounter, buf, packet, h)
+		if err != nil {
+			hostinfo.logger(f.l).Error("Failed to decrypt CloseTunnel packet",
+				"error", err,
+				"from", via,
+				"packet", packet,
+			)
+			return
+		}
+
 		hostinfo.logger(f.l).Info("Close tunnel received, tearing down.", "from", via)
+
 		f.closeTunnel(hostinfo)
+		return
 
 	case header.Control:
-		f.relayManager.HandleControlMsg(hostinfo, out, f)
+		if !f.handleEncrypted(ci, via, h) {
-
+			return
-	default:
-		hostinfo.logger(f.l).Error("IsValidSubType was true, but unexpected message type seen", "from", via, "header", h)
-	}
-}
-
-func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
-	// The entire body is sent as AD, not encrypted.
-	// The packet consists of a 16-byte parsed Nebula header, Associated Data-protected payload, and a trailing 16-byte AEAD signature value.
-	// The packet is guaranteed to be at least 16 bytes at this point, b/c it got past the h.Parse() call above. If it's
-	// otherwise malformed (meaning, there is no trailing 16 byte AEAD value), then this will result in at worst a 0-length slice
-	// which will gracefully fail in the DecryptDanger call.
-	signedPayload := packet[:len(packet)-hostinfo.ConnectionState.dKey.Overhead()]
-	signatureValue := packet[len(packet)-hostinfo.ConnectionState.dKey.Overhead():]
-	var err error
-	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, signedPayload, signatureValue, h.MessageCounter, nb)
-	if err != nil {
-		return
-	}
-	// Advance the replay window now that the frame is authenticated
-	if !hostinfo.ConnectionState.window.Update(f.l, h.MessageCounter) {
-		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			hostinfo.logger(f.l).Debug("dropping out of window relay packet", "header", h)
 		}
-		return
-	}
-	// Successfully validated the thing. Get rid of the Relay header.
-	signedPayload = signedPayload[header.Len:]
-	// Pull the Roaming parts up here, and return in all call paths.
-	f.handleHostRoaming(hostinfo, via)
-	// Track usage of both the HostInfo and the Relay for the received & authenticated packet
-	f.connectionManager.In(hostinfo)
-	f.connectionManager.RelayUsed(h.RemoteIndex)
 
-	relay, ok := hostinfo.relayState.QueryRelayForByIdx(h.RemoteIndex)
+		d, err := f.decrypt(hostinfo, h.MessageCounter, buf, packet, h)
-	if !ok {
-		// The only way this happens is if hostmap has an index to the correct HostInfo, but the HostInfo is missing
-		// its internal mapping. This should never happen.
-		hostinfo.logger(f.l).Error("HostInfo missing remote relay index",
-			"relayRemoteIndex", h.RemoteIndex,
-		)
-		return
-	}
-
-	switch relay.Type {
-	case TerminalType:
-		// If I am the target of this relay, process the unwrapped packet
-		// From this recursive point, all these variables are 'burned'. We shouldn't rely on them again.
-		via = ViaSender{
-			UdpAddr:   via.UdpAddr,
-			relayHI:   hostinfo,
-			remoteIdx: relay.RemoteIndex,
-			relay:     relay,
-			IsRelayed: true,
-		}
-		f.readOutsidePackets(via, out[:0], signedPayload, h, fwPacket, lhf, nb, q, localCache)
-	case ForwardingType:
-		// Find the target HostInfo relay object
-		targetHI, targetRelay, err := f.hostMap.QueryVpnAddrsRelayFor(hostinfo.vpnAddrs, relay.PeerAddr)
 		if err != nil {
-			hostinfo.logger(f.l).Info("Failed to find target host info by ip",
+			hostinfo.logger(f.l).Error("Failed to decrypt Control packet",
-				"relayTo", relay.PeerAddr,
-				"relayFrom", hostinfo.vpnAddrs[0],
 				"error", err,
+				"from", via,
+				"packet", packet,
 			)
 			return
 		}
 
-		// If that relay is Established, forward the payload through it
+		f.relayManager.HandleControlMsg(hostinfo, d, f)
-		if targetRelay.State == Established {
+
-			switch targetRelay.Type {
-			case ForwardingType:
-				// Forward this packet through the relay tunnel
-				// Find the target HostInfo
-				f.SendVia(targetHI, targetRelay, signedPayload, nb, out, false)
-			case TerminalType:
-				hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
-				return
-			default:
-				if f.l.Enabled(context.Background(), slog.LevelDebug) {
-					hostinfo.logger(f.l).Debug("Unexpected targetRelay Type", "from", via, "relayType", targetRelay.Type)
-				}
-				return
-			}
-		} else {
-			hostinfo.logger(f.l).Info("Unexpected target relay state",
-				"relayTo", relay.PeerAddr,
-				"relayFrom", hostinfo.vpnAddrs[0],
-				"targetRelayState", targetRelay.State,
-			)
-			return
-		}
 	default:
+		f.messageMetrics.Rx(h.Type, h.Subtype, 1)
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
-			hostinfo.logger(f.l).Debug("Unexpected relay type", "from", via, "relayType", relay.Type)
+			hostinfo.logger(f.l).Debug("Unexpected packet received", "from", via)
 		}
+		return
 	}
+
+	f.handleHostRoaming(hostinfo, via)
+
+	f.connectionManager.In(hostinfo)
 }
 
 // closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote
@@ -273,12 +269,13 @@ func (f *Interface) closeTunnel(hostInfo *HostInfo) {
 
 // sendCloseTunnel is a helper function to send a proper close tunnel packet to a remote
 func (f *Interface) sendCloseTunnel(h *HostInfo) {
-	f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, make([]byte, 12, 12), make([]byte, mtu))
+	buf := f.bufAlloc.Acquire()
+	defer f.bufAlloc.Release(buf)
+	f.send(header.CloseTunnel, 0, h.ConnectionState, h, []byte{}, buf)
 }
 
 func (f *Interface) handleHostRoaming(hostinfo *HostInfo, via ViaSender) {
-	curRemote := hostinfo.GetRemote()
+	if !via.IsRelayed && hostinfo.remote != via.UdpAddr {
-	if !via.IsRelayed && curRemote != via.UdpAddr {
 		if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, via.UdpAddr.Addr()) {
 			if f.l.Enabled(context.Background(), slog.LevelDebug) {
 				hostinfo.logger(f.l).Debug("lighthouse.remote_allow_list denied roaming", "newAddr", via.UdpAddr)
@@ -290,7 +287,7 @@ func (f *Interface) handleHostRoaming(hostinfo *HostInfo, via ViaSender) {
 			if f.l.Enabled(context.Background(), slog.LevelDebug) {
 				hostinfo.logger(f.l).Debug("Suppressing roam back to previous remote",
 					"suppressSeconds", RoamingSuppressSeconds,
-					"udpAddr", curRemote,
+					"udpAddr", hostinfo.remote,
 					"newAddr", via.UdpAddr,
 				)
 			}
@@ -298,16 +295,33 @@ func (f *Interface) handleHostRoaming(hostinfo *HostInfo, via ViaSender) {
 		}
 
 		hostinfo.logger(f.l).Info("Host roamed to new udp ip/port.",
-			"udpAddr", curRemote,
+			"udpAddr", hostinfo.remote,
 			"newAddr", via.UdpAddr,
 		)
 		hostinfo.lastRoam = time.Now()
-		hostinfo.lastRoamRemote = curRemote
+		hostinfo.lastRoamRemote = hostinfo.remote
 		hostinfo.SetRemote(via.UdpAddr)
 	}
 
 }
 
+// handleEncrypted returns true if a packet should be processed, false otherwise
+func (f *Interface) handleEncrypted(ci *ConnectionState, via ViaSender, h *header.H) bool {
+	// If connectionstate does not exist, send a recv error, if possible, to encourage a fast reconnect
+	if ci == nil {
+		if !via.IsRelayed {
+			f.maybeSendRecvError(via.UdpAddr, h.RemoteIndex)
+		}
+		return false
+	}
+	// If the window check fails, refuse to process the packet, but don't send a recv error
+	if !ci.window.Check(f.l, h.MessageCounter) {
+		return false
+	}
+
+	return true
+}
+
 var (
 	ErrPacketTooShort          = errors.New("packet is too short")
 	ErrUnknownIPVersion        = errors.New("packet is an unknown ip version")
@@ -422,14 +436,16 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
 			if dataLen <= offset+1 {
 				break
 			}
-			next = (int(data[offset+1]) + 2) << 2
+
+			next = int(data[offset+1]+2) << 2
 
 		default:
 			// Normal ipv6 header length processing
 			if dataLen <= offset+1 {
 				break
 			}
-			next = (int(data[offset+1]) + 1) << 3
+
+			next = int(data[offset+1]+1) << 3
 		}
 
 		if next <= 0 {
@@ -504,48 +520,65 @@ func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
 	return nil
 }
 
-func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, out []byte, packet []byte, h *header.H, nb []byte) ([]byte, error) {
+func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, buf *WireBuffer, packet []byte, h *header.H) ([]byte, error) {
-	var err error
+	plaintext, err := buf.DecryptForHandler(hostinfo.ConnectionState, packet, mc)
-	out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], mc, nb)
 	if err != nil {
 		return nil, err
 	}
 
 	if !hostinfo.ConnectionState.window.Update(f.l, mc) {
-		return nil, ErrOutOfWindow
+		if f.l.Enabled(context.Background(), slog.LevelDebug) {
+			hostinfo.logger(f.l).Debug("dropping out of window packet", "header", h)
+		}
+		return nil, errors.New("out of window packet")
 	}
 
-	return out, nil
+	return plaintext, nil
 }
 
-func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache) {
+func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, buf *WireBuffer, packet []byte, q int, localCache firewall.ConntrackCache) bool {
-	err := newPacket(out, true, fwPacket)
+	if err := buf.DecryptDatagram(hostinfo.ConnectionState, packet, messageCounter); err != nil {
-	if err != nil {
+		hostinfo.logger(f.l).Error("Failed to decrypt packet", "error", err)
-		hostinfo.logger(f.l).Warn("Error while validating inbound packet",
+		return false
-			"error", err,
-			"packet", out,
-		)
-		return
 	}
 
-	dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache)
+	ipPacket := buf.IPPacket()
+	if err := newPacket(ipPacket, true, buf.FwPacket); err != nil {
+		hostinfo.logger(f.l).Warn("Error while validating inbound packet",
+			"error", err,
+			"packet", ipPacket,
+		)
+		return false
+	}
+
+	if !hostinfo.ConnectionState.window.Update(f.l, messageCounter) {
+		if f.l.Enabled(context.Background(), slog.LevelDebug) {
+			hostinfo.logger(f.l).Debug("dropping out of window packet", "fwPacket", buf.FwPacket)
+		}
+		return false
+	}
+
+	dropReason := f.firewall.Drop(*buf.FwPacket, true, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason != nil {
-		// NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore
+		// NOTE: We hand `packet` (the original UDP ciphertext we already
-		// This gives us a buffer to build the reject packet in
+		// decrypted from) as the reject-IP scratch since we no longer
-		f.rejectOutside(out, hostinfo.ConnectionState, hostinfo, nb, packet, q)
+		// need its ciphertext, and it's disjoint from buf.Out where
+		// sendNoMetrics will encrypt the wire packet.
+		f.rejectOutside(ipPacket, hostinfo.ConnectionState, hostinfo, packet, buf, q)
 		if f.l.Enabled(context.Background(), slog.LevelDebug) {
 			hostinfo.logger(f.l).Debug("dropping inbound packet",
-				"fwPacket", fwPacket,
+				"fwPacket", buf.FwPacket,
 				"reason", dropReason,
 			)
 		}
-		return
+		return false
 	}
 
-	_, err = f.readers[q].Write(out)
+	f.connectionManager.In(hostinfo)
-	if err != nil {
+	if _, err := buf.WriteIPToTUN(f.readers[q]); err != nil {
 		f.l.Error("Failed to write to tun", "error", err)
 	}
+	return true
 }
 
 func (f *Interface) maybeSendRecvError(endpoint netip.AddrPort, index uint32) {
@@ -589,11 +622,10 @@ func (f *Interface) handleRecvError(addr netip.AddrPort, h *header.H) {
 		return
 	}
 
-	hr := hostinfo.GetRemote()
+	if hostinfo.remote.IsValid() && hostinfo.remote != addr {
-	if hr.IsValid() && hr != addr {
 		f.l.Info("Someone spoofing recv_errors?",
 			"addr", addr,
-			"hostinfoRemote", hr,
+			"hostinfoRemote", hostinfo.remote,
 		)
 		return
 	}

outside_test.go

@@ -640,38 +640,3 @@ func serializeAH(ah *layers.IPSecAH) []byte {
 
 	return buf.Bytes()
 }
-
-// Test_newPacket_v6ExtHeaderOverflow is a regression test for the IPv6 extension-header
-// length uint8 overflow in parseV6. A Destination-Options header with HdrExtLen=255 spans
-// (255+1)*8 = 2048 bytes, so the real transport header sits at offset 2088. Before the fix
-// the advance was computed in uint8 and wrapped to 0 (then clamped to 8), so the firewall
-// read the transport header ~2KB too early from attacker-controlled option bytes while the
-// host OS parses the real header, a firewall port/proto bypass. The fix makes parseV6 land
-// on the same offset the host does.
-func Test_newPacket_v6ExtHeaderOverflow(t *testing.T) {
-	p := &firewall.Packet{}
-
-	const (
-		hdrLen      = 40              // IPv6 header
-		extLen      = 2048            // (255+1)*8, the true Destination-Options header size
-		realTCPAt   = hdrLen + extLen // 2088, where the host reads the transport header
-		forgedTCPAt = hdrLen + 8      // 48, where the pre-fix wrapped+clamped walk landed
-	)
-
-	pkt := make([]byte, realTCPAt+4)
-	pkt[0] = 0x60                                   // version 6
-	pkt[6] = byte(layers.IPProtocolIPv6Destination) // NextHeader -> Destination Options
-	pkt[40] = byte(firewall.ProtoTCP)               // Dest-Options NextHeader -> TCP
-	pkt[41] = 255                                   // HdrExtLen = 255
-
-	// Forged transport header at the pre-fix (wrong) offset: dst port 443.
-	binary.BigEndian.PutUint16(pkt[forgedTCPAt+2:forgedTCPAt+4], 443)
-	// Real transport header at the offset the host actually uses: dst port 22.
-	binary.BigEndian.PutUint16(pkt[realTCPAt+2:realTCPAt+4], 22)
-
-	require.NoError(t, newPacket(pkt, true, p))
-	assert.Equal(t, uint8(firewall.ProtoTCP), p.Protocol)
-	// LocalPort is the destination port for incoming traffic. It must be the real port (22)
-	// the host delivers to, not the forged 443 at the overflowed offset.
-	assert.Equal(t, uint16(22), p.LocalPort, "firewall must parse the real transport header, not the overflowed offset")
-}

overlay/device.go

@@ -15,4 +15,7 @@ type Device interface {
 	RoutesFor(netip.Addr) routing.Gateways
 	SupportsMultiqueue() bool
 	NewMultiQueueReader() (io.ReadWriteCloser, error)
+	// TunPrefixLen reports the number of bytes the device prepends to every IP packet on the wire.
+	// Currently only non zero for the BSD tun devices.
+	TunPrefixLen() int
 }

overlay/network_category_windows.go

@@ -1,358 +0,0 @@
-//go:build !e2e_testing
-// +build !e2e_testing
-
-package overlay
-
-import (
-	"errors"
-	"fmt"
-	"log/slog"
-	"runtime"
-	"strings"
-	"syscall"
-	"time"
-	"unsafe"
-
-	"golang.org/x/sys/windows"
-)
-
-// networkCategory mirrors NLM_NETWORK_CATEGORY from netlistmgr.h.
-type networkCategory int32
-
-const (
-	networkCategoryPublic              networkCategory = 0
-	networkCategoryPrivate             networkCategory = 1
-	networkCategoryDomainAuthenticated networkCategory = 2
-)
-
-func (c networkCategory) String() string {
-	switch c {
-	case networkCategoryPublic:
-		return "public"
-	case networkCategoryPrivate:
-		return "private"
-	case networkCategoryDomainAuthenticated:
-		return "domain"
-	}
-	return fmt.Sprintf("unknown(%d)", c)
-}
-
-// parseNetworkCategory accepts the user-supplied tun.network_category. A
-// second return of false means "leave the category alone".
-func parseNetworkCategory(s string) (networkCategory, bool, error) {
-	switch strings.ToLower(strings.TrimSpace(s)) {
-	case "", "unset":
-		return 0, false, nil
-	case "public":
-		return networkCategoryPublic, true, nil
-	case "private":
-		return networkCategoryPrivate, true, nil
-	case "domain", "domainauthenticated":
-		return networkCategoryDomainAuthenticated, true, nil
-	}
-	return 0, false, fmt.Errorf("unknown tun.network_category %q (expected public, private, domain, or unset)", s)
-}
-
-// CLSID_NetworkListManager {DCB00C01-570F-4A9B-8D69-199FDBA5723B}
-var clsidNetworkListManager = windows.GUID{
-	Data1: 0xDCB00C01, Data2: 0x570F, Data3: 0x4A9B,
-	Data4: [8]byte{0x8D, 0x69, 0x19, 0x9F, 0xDB, 0xA5, 0x72, 0x3B},
-}
-
-// IID_INetworkListManager {DCB00000-570F-4A9B-8D69-199FDBA5723B}
-var iidINetworkListManager = windows.GUID{
-	Data1: 0xDCB00000, Data2: 0x570F, Data3: 0x4A9B,
-	Data4: [8]byte{0x8D, 0x69, 0x19, 0x9F, 0xDB, 0xA5, 0x72, 0x3B},
-}
-
-// x/sys/windows doesn't expose CoCreateInstance, so we bind it ourselves.
-var procCoCreateInstance = windows.NewLazySystemDLL("ole32.dll").NewProc("CoCreateInstance")
-
-const clsCtxAll = windows.CLSCTX_INPROC_SERVER | windows.CLSCTX_INPROC_HANDLER |
-	windows.CLSCTX_LOCAL_SERVER | windows.CLSCTX_REMOTE_SERVER
-
-const (
-	hrSFALSE          = 0x00000001
-	hrRPCEChangedMode = 0x80010106
-)
-
-type hresult uint32
-
-func (h hresult) failed() bool { return int32(h) < 0 }
-func (h hresult) String() string {
-	return fmt.Sprintf("HRESULT 0x%08x", uint32(h))
-}
-
-var errAdapterNotFound = errors.New("adapter not present in network connections enumeration")
-
-// Vtable layouts. Slot order must match the declaration order in netlistmgr.h.
-// All NLM interfaces here derive from IDispatch, which derives from IUnknown.
-
-type iUnknownVtbl struct {
-	QueryInterface uintptr
-	AddRef         uintptr
-	Release        uintptr
-}
-
-type iDispatchVtbl struct {
-	iUnknownVtbl
-	GetTypeInfoCount uintptr
-	GetTypeInfo      uintptr
-	GetIDsOfNames    uintptr
-	Invoke           uintptr
-}
-
-type iNetworkListManagerVtbl struct {
-	iDispatchVtbl
-	GetNetworks           uintptr
-	GetNetwork            uintptr
-	GetNetworkConnections uintptr
-	GetNetworkConnection  uintptr
-	IsConnectedToInternet uintptr
-	IsConnected           uintptr
-	GetConnectivity       uintptr
-}
-
-type iNetworkListManager struct{ Vtbl *iNetworkListManagerVtbl }
-
-func (n *iNetworkListManager) Release() {
-	syscall.SyscallN(n.Vtbl.Release, uintptr(unsafe.Pointer(n)))
-}
-
-func (n *iNetworkListManager) GetNetworkConnections() (*iEnumNetworkConnections, error) {
-	var enum *iEnumNetworkConnections
-	r1, _, _ := syscall.SyscallN(n.Vtbl.GetNetworkConnections,
-		uintptr(unsafe.Pointer(n)), uintptr(unsafe.Pointer(&enum)),
-	)
-	if hr := hresult(r1); hr.failed() {
-		return nil, fmt.Errorf("INetworkListManager.GetNetworkConnections: %s", hr)
-	}
-	return enum, nil
-}
-
-type iEnumNetworkConnectionsVtbl struct {
-	iDispatchVtbl
-	NewEnum uintptr
-	Next    uintptr
-	Skip    uintptr
-	Reset   uintptr
-	Clone   uintptr
-}
-
-type iEnumNetworkConnections struct{ Vtbl *iEnumNetworkConnectionsVtbl }
-
-func (e *iEnumNetworkConnections) Release() {
-	syscall.SyscallN(e.Vtbl.Release, uintptr(unsafe.Pointer(e)))
-}
-
-// Next returns the next connection, or (nil, nil) at the end of the enumeration.
-func (e *iEnumNetworkConnections) Next() (*iNetworkConnection, error) {
-	var conn *iNetworkConnection
-	var fetched uint32
-	r1, _, _ := syscall.SyscallN(e.Vtbl.Next,
-		uintptr(unsafe.Pointer(e)), 1,
-		uintptr(unsafe.Pointer(&conn)), uintptr(unsafe.Pointer(&fetched)),
-	)
-	if hr := hresult(r1); hr.failed() {
-		return nil, fmt.Errorf("IEnumNetworkConnections.Next: %s", hr)
-	}
-	if fetched == 0 {
-		return nil, nil
-	}
-	return conn, nil
-}
-
-type iNetworkConnectionVtbl struct {
-	iDispatchVtbl
-	GetNetwork            uintptr
-	IsConnectedToInternet uintptr
-	IsConnected           uintptr
-	GetConnectivity       uintptr
-	GetConnectionId       uintptr
-	GetAdapterId          uintptr
-	GetDomainType         uintptr
-}
-
-type iNetworkConnection struct{ Vtbl *iNetworkConnectionVtbl }
-
-func (c *iNetworkConnection) Release() {
-	syscall.SyscallN(c.Vtbl.Release, uintptr(unsafe.Pointer(c)))
-}
-
-func (c *iNetworkConnection) GetAdapterId() (windows.GUID, error) {
-	var g windows.GUID
-	r1, _, _ := syscall.SyscallN(c.Vtbl.GetAdapterId,
-		uintptr(unsafe.Pointer(c)), uintptr(unsafe.Pointer(&g)),
-	)
-	if hr := hresult(r1); hr.failed() {
-		return windows.GUID{}, fmt.Errorf("INetworkConnection.GetAdapterId: %s", hr)
-	}
-	return g, nil
-}
-
-func (c *iNetworkConnection) GetNetwork() (*iNetwork, error) {
-	var net *iNetwork
-	r1, _, _ := syscall.SyscallN(c.Vtbl.GetNetwork,
-		uintptr(unsafe.Pointer(c)), uintptr(unsafe.Pointer(&net)),
-	)
-	if hr := hresult(r1); hr.failed() {
-		return nil, fmt.Errorf("INetworkConnection.GetNetwork: %s", hr)
-	}
-	return net, nil
-}
-
-type iNetworkVtbl struct {
-	iDispatchVtbl
-	GetName                    uintptr
-	SetName                    uintptr
-	GetDescription             uintptr
-	SetDescription             uintptr
-	GetNetworkId               uintptr
-	GetDomainType              uintptr
-	GetNetworkConnections      uintptr
-	GetTimeCreatedAndConnected uintptr
-	IsConnectedToInternet      uintptr
-	IsConnected                uintptr
-	GetConnectivity            uintptr
-	GetCategory                uintptr
-	SetCategory                uintptr
-}
-
-type iNetwork struct{ Vtbl *iNetworkVtbl }
-
-func (n *iNetwork) Release() {
-	syscall.SyscallN(n.Vtbl.Release, uintptr(unsafe.Pointer(n)))
-}
-
-func (n *iNetwork) GetCategory() (networkCategory, error) {
-	var c networkCategory
-	r1, _, _ := syscall.SyscallN(n.Vtbl.GetCategory,
-		uintptr(unsafe.Pointer(n)), uintptr(unsafe.Pointer(&c)),
-	)
-	if hr := hresult(r1); hr.failed() {
-		return 0, fmt.Errorf("INetwork.GetCategory: %s", hr)
-	}
-	return c, nil
-}
-
-func (n *iNetwork) SetCategory(c networkCategory) error {
-	r1, _, _ := syscall.SyscallN(n.Vtbl.SetCategory,
-		uintptr(unsafe.Pointer(n)), uintptr(int32(c)),
-	)
-	if hr := hresult(r1); hr.failed() {
-		return fmt.Errorf("INetwork.SetCategory: %s", hr)
-	}
-	return nil
-}
-
-// coInit initializes COM for the current OS thread. The returned function must
-// be deferred to balance a successful init. RPC_E_CHANGED_MODE means COM is
-// already initialized in a different mode on this thread, which is still fine
-// for our calls but we must not Uninitialize in that case.
-func coInit() (func(), error) {
-	err := windows.CoInitializeEx(0, windows.COINIT_MULTITHREADED)
-	if err == nil {
-		return windows.CoUninitialize, nil
-	}
-	if e, ok := err.(syscall.Errno); ok {
-		switch uint32(e) {
-		case hrSFALSE:
-			return windows.CoUninitialize, nil
-		case hrRPCEChangedMode:
-			return func() {}, nil
-		}
-	}
-	return nil, fmt.Errorf("CoInitializeEx: %w", err)
-}
-
-func createNetworkListManager() (*iNetworkListManager, error) {
-	var nlm *iNetworkListManager
-	r1, _, _ := procCoCreateInstance.Call(
-		uintptr(unsafe.Pointer(&clsidNetworkListManager)),
-		0,
-		uintptr(clsCtxAll),
-		uintptr(unsafe.Pointer(&iidINetworkListManager)),
-		uintptr(unsafe.Pointer(&nlm)),
-	)
-	if hr := hresult(r1); hr.failed() {
-		return nil, fmt.Errorf("CoCreateInstance(NetworkListManager): %s", hr)
-	}
-	return nlm, nil
-}
-
-// setNetworkCategory locates the network connection bound to adapterGUID and
-// sets the category of its parent network. Returns errAdapterNotFound if the
-// adapter is not yet visible in the NLM enumeration.
-func setNetworkCategory(adapterGUID windows.GUID, cat networkCategory) error {
-	deinit, err := coInit()
-	if err != nil {
-		return err
-	}
-	defer deinit()
-
-	nlm, err := createNetworkListManager()
-	if err != nil {
-		return err
-	}
-	defer nlm.Release()
-
-	enum, err := nlm.GetNetworkConnections()
-	if err != nil {
-		return err
-	}
-	defer enum.Release()
-
-	for {
-		conn, err := enum.Next()
-		if err != nil {
-			return err
-		}
-		if conn == nil {
-			return errAdapterNotFound
-		}
-
-		guid, err := conn.GetAdapterId()
-		if err != nil || guid != adapterGUID {
-			conn.Release()
-			continue
-		}
-
-		net, err := conn.GetNetwork()
-		conn.Release()
-		if err != nil {
-			return err
-		}
-		err = net.SetCategory(cat)
-		net.Release()
-		return err
-	}
-}
-
-// applyNetworkCategory polls until the wintun adapter shows up in the NLM
-// enumeration, then sets the category. Intended to run in its own goroutine.
-func applyNetworkCategory(l *slog.Logger, adapterGUID windows.GUID, cat networkCategory) {
-	// COM Init/Uninit must be paired on the same OS thread.
-	runtime.LockOSThread()
-	defer runtime.UnlockOSThread()
-
-	const (
-		attempts = 30
-		interval = 500 * time.Millisecond
-	)
-	for i := 0; i < attempts; i++ {
-		err := setNetworkCategory(adapterGUID, cat)
-		if err == nil {
-			l.Info("Set Windows network category", "category", cat.String())
-			return
-		}
-		if !errors.Is(err, errAdapterNotFound) {
-			l.Warn("Failed to set Windows network category", "error", err, "category", cat.String())
-			return
-		}
-		time.Sleep(interval)
-	}
-	l.Warn("Gave up waiting for adapter to appear in NLM enumeration; network category not set",
-		"category", cat.String(),
-		"waited", time.Duration(attempts)*interval,
-	)
-}

overlay/network_category_windows_test.go

@@ -1,109 +0,0 @@
-//go:build !e2e_testing
-// +build !e2e_testing
-
-package overlay
-
-import (
-	"testing"
-)
-
-func Test_parseNetworkCategory(t *testing.T) {
-	cases := []struct {
-		in        string
-		wantCat   networkCategory
-		wantApply bool
-		wantErr   bool
-	}{
-		{"", 0, false, false},
-		{"unset", 0, false, false},
-		{"  UNSET  ", 0, false, false},
-		{"private", networkCategoryPrivate, true, false},
-		{"Private", networkCategoryPrivate, true, false},
-		{"  PRIVATE  ", networkCategoryPrivate, true, false},
-		{"public", networkCategoryPublic, true, false},
-		{"PUBLIC", networkCategoryPublic, true, false},
-		{"domain", networkCategoryDomainAuthenticated, true, false},
-		{"DomainAuthenticated", networkCategoryDomainAuthenticated, true, false},
-		{"garbage", 0, false, true},
-		{"privates", 0, false, true},
-	}
-	for _, tc := range cases {
-		cat, apply, err := parseNetworkCategory(tc.in)
-		if (err != nil) != tc.wantErr {
-			t.Errorf("parseNetworkCategory(%q) err=%v, wantErr=%v", tc.in, err, tc.wantErr)
-			continue
-		}
-		if cat != tc.wantCat || apply != tc.wantApply {
-			t.Errorf("parseNetworkCategory(%q) = (%v, %v), want (%v, %v)", tc.in, cat, apply, tc.wantCat, tc.wantApply)
-		}
-	}
-}
-
-// Test_NLM_round_trip exercises every COM call path used by setNetworkCategory
-// without mutating the host's network state. It validates the CLSID/IID
-// constants and every vtable index by enumerating connections, fetching the
-// adapter id and parent network, reading the current category, and writing it
-// back unchanged.
-//
-// Requires Windows but does not require admin or the wintun driver. Skips if
-// no network connections are available (unlikely outside of an isolated
-// container).
-func Test_NLM_round_trip(t *testing.T) {
-	deinit, err := coInit()
-	if err != nil {
-		t.Fatalf("coInit: %v", err)
-	}
-	defer deinit()
-
-	nlm, err := createNetworkListManager()
-	if err != nil {
-		t.Fatalf("createNetworkListManager: %v", err)
-	}
-	defer nlm.Release()
-
-	enum, err := nlm.GetNetworkConnections()
-	if err != nil {
-		t.Fatalf("GetNetworkConnections: %v", err)
-	}
-	defer enum.Release()
-
-	saw := 0
-	for {
-		conn, err := enum.Next()
-		if err != nil {
-			t.Fatalf("EnumNetworkConnections.Next: %v", err)
-		}
-		if conn == nil {
-			break
-		}
-		saw++
-
-		if _, err := conn.GetAdapterId(); err != nil {
-			conn.Release()
-			t.Fatalf("INetworkConnection.GetAdapterId: %v", err)
-		}
-
-		net, err := conn.GetNetwork()
-		conn.Release()
-		if err != nil {
-			t.Fatalf("INetworkConnection.GetNetwork: %v", err)
-		}
-
-		cat, err := net.GetCategory()
-		if err != nil {
-			net.Release()
-			t.Fatalf("INetwork.GetCategory: %v", err)
-		}
-		// Set to the current value so the host's NLM state is unchanged but
-		// SetCategory's vtable slot is still validated end-to-end.
-		if err := net.SetCategory(cat); err != nil {
-			net.Release()
-			t.Fatalf("INetwork.SetCategory(%v): %v", cat, err)
-		}
-		net.Release()
-	}
-
-	if saw == 0 {
-		t.Skip("no NLM network connections available; skipping round-trip")
-	}
-}

overlay/overlaytest/noop.go

@@ -50,3 +50,5 @@ func (NoopTun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 func (NoopTun) Close() error {
 	return nil
 }
+
+func (NoopTun) TunPrefixLen() int { return 0 }

overlay/tun_android.go

@@ -102,3 +102,5 @@ func (t *tun) SupportsMultiqueue() bool {
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for android")
 }
+
+func (t *tun) TunPrefixLen() int { return 0 }

overlay/tun_bsd.go

@@ -0,0 +1,29 @@
+//go:build (darwin || ios || freebsd || openbsd || netbsd) && !e2e_testing
+
+package overlay
+
+import (
+	"fmt"
+	"syscall"
+)
+
+// StampTunPrefix writes the 4-byte AF_INET / AF_INET6 protocol-family marker into buf[0:4] in place,
+// picking the family from the first byte of the IP packet at buf[4].
+func StampTunPrefix(buf []byte) error {
+	if len(buf) < 5 {
+		return fmt.Errorf("tun write buffer too small for prefix")
+	}
+	ipVer := buf[4] >> 4
+	buf[0] = 0
+	buf[1] = 0
+	buf[2] = 0
+	switch ipVer {
+	case 4:
+		buf[3] = syscall.AF_INET
+	case 6:
+		buf[3] = syscall.AF_INET6
+	default:
+		return fmt.Errorf("unable to determine IP version from packet")
+	}
+	return nil
+}

overlay/tun_bypass_windows.go

@@ -1,23 +0,0 @@
-//go:build (amd64 || arm64) && !e2e_testing
-// +build amd64 arm64
-// +build !e2e_testing
-
-package overlay
-
-import (
-	"log/slog"
-
-	"github.com/slackhq/nebula/wfp"
-)
-
-// installInterfaceBypass installs a WFP PERMIT filter scoped to the wintun interface LUID so inbound traffic on the
-// nebula adapter bypasses Windows Defender Firewall.
-func installInterfaceBypass(l *slog.Logger, luid uint64) closer {
-	s, err := wfp.PermitInterface(luid)
-	if err != nil {
-		l.Warn("Failed to install WFP bypass filters on nebula interface", "error", err)
-		return nil
-	}
-	l.Info("Installed WFP filters bypassing Windows Defender Firewall on nebula interface")
-	return s
-}

overlay/tun_bypass_windows_386.go

@@ -1,11 +0,0 @@
-//go:build !e2e_testing
-// +build !e2e_testing
-
-package overlay
-
-import "log/slog"
-
-// installInterfaceBypass is a no-op on windows-386 because we don't currently build for it.
-func installInterfaceBypass(_ *slog.Logger, _ uint64) closer {
-	return nil
-}

overlay/tun_darwin.go

@@ -11,7 +11,6 @@ import (
 	"net/netip"
 	"os"
 	"sync/atomic"
-	"syscall"
 	"unsafe"
 
 	"github.com/gaissmai/bart"
@@ -23,7 +22,7 @@ import (
 )
 
 type tun struct {
-	f           *os.File
+	io.ReadWriteCloser
 	Device      string
 	vpnNetworks []netip.Prefix
 	DefaultMTU  int
@@ -121,11 +120,11 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 	}
 
 	t := &tun{
-		f:           os.NewFile(uintptr(fd), ""),
+		ReadWriteCloser: os.NewFile(uintptr(fd), ""),
-		Device:      name,
+		Device:          name,
-		vpnNetworks: vpnNetworks,
+		vpnNetworks:     vpnNetworks,
-		DefaultMTU:  c.GetInt("tun.mtu", DefaultMTU),
+		DefaultMTU:      c.GetInt("tun.mtu", DefaultMTU),
-		l:           l,
+		l:               l,
 	}
 
 	err = t.reload(c, true)
@@ -155,8 +154,8 @@ func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, e
 }
 
 func (t *tun) Close() error {
-	if t.f != nil {
+	if t.ReadWriteCloser != nil {
-		return t.f.Close()
+		return t.ReadWriteCloser.Close()
 	}
 	return nil
 }
@@ -499,105 +498,6 @@ func delRoute(prefix netip.Prefix, gateway netroute.Addr) error {
 	return nil
 }
 
-// tunWritev and tunReadv are linkname'd to x/sys/unix's libc-routed writev/readv stubs so the
-// calls go through libSystem's pinned trampoline. A raw syscall.Syscall(SYS_WRITEV/SYS_READV, ...)
-// on darwin/arm64 emits an SVC #0x80 trap (see $GOROOT/src/syscall/asm_darwin_arm64.s), the path
-// Apple keeps warning they will eventually disallow. We pull the low-level stubs instead of calling
-// unix.Writev/unix.Readv because those take [][]byte and rebuild the []Iovec every call, which
-// heap-allocates the header; linkname'ing the stubs lets us hand them our own stack-allocated
-// iovecs. See golang/go#78049.
-
-//go:linkname tunWritev golang.org/x/sys/unix.writev
-//go:noescape
-func tunWritev(fd int, iovecs []unix.Iovec) (n int, err error)
-
-//go:linkname tunReadv golang.org/x/sys/unix.readv
-//go:noescape
-func tunReadv(fd int, iovecs []unix.Iovec) (n int, err error)
-
-// Read pulls one IP packet off the utun device, scattering the 4 byte protocol header away from
-// the packet so the payload lands directly in to.
-func (t *tun) Read(to []byte) (int, error) {
-	var head [4]byte
-
-	rc, err := t.f.SyscallConn()
-	if err != nil {
-		return 0, err
-	}
-
-	var n int
-	var callErr error
-	err = rc.Read(func(fd uintptr) bool {
-		iovecs := []unix.Iovec{
-			{Base: &head[0], Len: 4},
-			{Base: &to[0], Len: uint64(len(to))},
-		}
-		n, callErr = tunReadv(int(fd), iovecs)
-		if errno, ok := callErr.(syscall.Errno); ok && errno.Temporary() {
-			return false
-		}
-		return true
-	})
-	if err != nil {
-		return 0, err
-	}
-	if callErr != nil {
-		return 0, callErr
-	}
-	if n < 4 {
-		return 0, nil
-	}
-	return n - 4, nil
-}
-
-// Write pushes one IP packet onto the utun device.
-func (t *tun) Write(from []byte) (int, error) {
-	if len(from) == 0 {
-		return 0, syscall.EIO
-	}
-
-	ipVer := from[0] >> 4
-	var head [4]byte
-	switch ipVer {
-	case 4:
-		head[3] = syscall.AF_INET
-	case 6:
-		head[3] = syscall.AF_INET6
-	default:
-		return 0, fmt.Errorf("unable to determine IP version from packet")
-	}
-
-	// Grab rc as a local so the compiler can devirtualize the call and keep the closure on the stack.
-	rc, err := t.f.SyscallConn()
-	if err != nil {
-		return 0, err
-	}
-
-	var n int
-	var callErr error
-	err = rc.Write(func(fd uintptr) bool {
-		iovecs := []unix.Iovec{
-			{Base: &head[0], Len: 4},
-			{Base: &from[0], Len: uint64(len(from))},
-		}
-		n, callErr = tunWritev(int(fd), iovecs)
-		// Type-assert to syscall.Errno so the EAGAIN/EWOULDBLOCK/EINTR check doesn't box the errno
-		// constants into error interfaces on every call.
-		if errno, ok := callErr.(syscall.Errno); ok && errno.Temporary() {
-			return false
-		}
-		return true
-	})
-	if err != nil {
-		return 0, err
-	}
-	if callErr != nil {
-		return 0, callErr
-	}
-
-	return n - 4, nil
-}
-
 func (t *tun) Networks() []netip.Prefix {
 	return t.vpnNetworks
 }
@@ -613,3 +513,7 @@ func (t *tun) SupportsMultiqueue() bool {
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for darwin")
 }
+
+// TunPrefixLen reports the 4-byte BSD AF_INET / AF_INET6 protocol-family
+// marker the kernel prepends on read and expects on write.
+func (t *tun) TunPrefixLen() int { return 4 }

overlay/tun_disabled.go

@@ -136,3 +136,5 @@ func (p prettyPacket) String() string {
 
 	return s.String()
 }
+
+func (t *disabledTun) TunPrefixLen() int { return 0 }

overlay/tun_freebsd.go

@@ -158,74 +158,43 @@ func (t *tun) blockOnWrite() error {
 }
 
 func (t *tun) Read(to []byte) (int, error) {
-	// first 4 bytes is protocol family, in network byte order
-	var head [4]byte
-	iovecs := [2]syscall.Iovec{
-		{&head[0], 4},
-		{&to[0], uint64(len(to))},
-	}
 	for {
-		n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
+		n, err := unix.Read(t.fd, to)
-		if errno == 0 {
+		if err == nil {
-			bytesRead := int(n)
+			return n, nil
-			if bytesRead < 4 {
-				return 0, nil
-			}
-			return bytesRead - 4, nil
 		}
-		switch errno {
+		switch err {
 		case unix.EAGAIN:
-			if err := t.blockOnRead(); err != nil {
+			if berr := t.blockOnRead(); berr != nil {
-				return 0, err
+				return 0, berr
 			}
 		case unix.EINTR:
 			// retry
 		case unix.EBADF:
 			return 0, os.ErrClosed
 		default:
-			return 0, errno
+			return 0, err
 		}
 	}
 }
 
-// Write is only valid for single threaded use
 func (t *tun) Write(from []byte) (int, error) {
-	if len(from) <= 1 {
-		return 0, syscall.EIO
-	}
-
-	ipVer := from[0] >> 4
-	var head [4]byte
-	// first 4 bytes is protocol family, in network byte order
-	switch ipVer {
-	case 4:
-		head[3] = syscall.AF_INET
-	case 6:
-		head[3] = syscall.AF_INET6
-	default:
-		return 0, fmt.Errorf("unable to determine IP version from packet")
-	}
-
-	iovecs := [2]syscall.Iovec{
-		{&head[0], 4},
-		{&from[0], uint64(len(from))},
-	}
 	for {
-		n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
+		n, err := unix.Write(t.fd, from)
-		if errno == 0 {
+		if err == nil {
-			return int(n) - 4, nil
+			return n, nil
 		}
-		switch errno {
+		switch err {
 		case unix.EAGAIN:
-			if err := t.blockOnWrite(); err != nil {
+			if berr := t.blockOnWrite(); berr != nil {
-				return 0, err
+				return 0, berr
 			}
 		case unix.EINTR:
 			// retry
 		case unix.EBADF:
 			return 0, os.ErrClosed
 		default:
-			return 0, errno
+			return 0, err
 		}
 	}
 }
@@ -732,3 +701,7 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
 
 	return nil, nil
 }
+
+// TunPrefixLen reports the 4-byte BSD AF_INET / AF_INET6 protocol-family
+// marker the kernel prepends on read and expects on write.
+func (t *tun) TunPrefixLen() int { return 4 }

overlay/tun_ios.go

@@ -4,15 +4,12 @@
 package overlay
 
 import (
-	"errors"
 	"fmt"
 	"io"
 	"log/slog"
 	"net/netip"
 	"os"
-	"sync"
 	"sync/atomic"
-	"syscall"
 
 	"github.com/gaissmai/bart"
 	"github.com/slackhq/nebula/config"
@@ -36,7 +33,7 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
 	file := os.NewFile(uintptr(deviceFd), "/dev/tun")
 	t := &tun{
 		vpnNetworks:     vpnNetworks,
-		ReadWriteCloser: &tunReadCloser{f: file},
+		ReadWriteCloser: file,
 		l:               l,
 	}
 
@@ -85,64 +82,6 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
 	return r
 }
 
-// The following is hoisted up from water, we do this so we can inject our own fd on iOS
-type tunReadCloser struct {
-	f io.ReadWriteCloser
-
-	rMu  sync.Mutex
-	rBuf []byte
-
-	wMu  sync.Mutex
-	wBuf []byte
-}
-
-func (tr *tunReadCloser) Read(to []byte) (int, error) {
-	tr.rMu.Lock()
-	defer tr.rMu.Unlock()
-
-	if cap(tr.rBuf) < len(to)+4 {
-		tr.rBuf = make([]byte, len(to)+4)
-	}
-	tr.rBuf = tr.rBuf[:len(to)+4]
-
-	n, err := tr.f.Read(tr.rBuf)
-	copy(to, tr.rBuf[4:])
-	return n - 4, err
-}
-
-func (tr *tunReadCloser) Write(from []byte) (int, error) {
-	if len(from) == 0 {
-		return 0, syscall.EIO
-	}
-
-	tr.wMu.Lock()
-	defer tr.wMu.Unlock()
-
-	if cap(tr.wBuf) < len(from)+4 {
-		tr.wBuf = make([]byte, len(from)+4)
-	}
-	tr.wBuf = tr.wBuf[:len(from)+4]
-
-	// Determine the IP Family for the NULL L2 Header
-	ipVer := from[0] >> 4
-	if ipVer == 4 {
-		tr.wBuf[3] = syscall.AF_INET
-	} else if ipVer == 6 {
-		tr.wBuf[3] = syscall.AF_INET6
-	} else {
-		return 0, errors.New("unable to determine IP version from packet")
-	}
-
-	copy(tr.wBuf[4:], from)
-
-	n, err := tr.f.Write(tr.wBuf)
-	return n - 4, err
-}
-
-func (tr *tunReadCloser) Close() error {
-	return tr.f.Close()
-}
-
 func (t *tun) Networks() []netip.Prefix {
 	return t.vpnNetworks
 }
@@ -158,3 +97,7 @@ func (t *tun) SupportsMultiqueue() bool {
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for ios")
 }
+
+// TunPrefixLen reports the 4-byte BSD AF_INET / AF_INET6 protocol-family
+// marker the kernel prepends on read and expects on write.
+func (t *tun) TunPrefixLen() int { return 4 }

overlay/tun_linux.go

@@ -5,7 +5,6 @@ package overlay
 
 import (
 	"encoding/binary"
-	"errors"
 	"fmt"
 	"io"
 	"log/slog"
@@ -251,17 +250,6 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
 }
 
 func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) {
-	// Validate the device name up front so a bad tun.dev fails fast, before we
-	// open /dev/net/tun or leak a file descriptor. A single %d in the name is
-	// substituted by the kernel during TUNSETIFF (dev_alloc_name) with the
-	// lowest number that yields an unused device name. Resolving the template
-	// in the kernel keeps the pick-a-name/create-the-device pair atomic, so
-	// concurrent callers can never race each other to the same name.
-	tunName := c.GetString("tun.dev", "nebula%d")
-	if err := validateTunName(tunName); err != nil {
-		return nil, err
-	}
-
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
 	if err != nil {
 		// If /dev/net/tun doesn't exist, try to create it (will happen in docker)
@@ -289,11 +277,12 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue
 	if multiqueue {
 		req.Flags |= unix.IFF_MULTI_QUEUE
 	}
-	copy(req.Name[:], tunName)
+	nameStr := c.GetString("tun.dev", "")
+	copy(req.Name[:], nameStr)
 	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
 		_ = unix.Close(fd)
 		return nil, &NameError{
-			Name:       tunName,
+			Name:       nameStr,
 			Underlying: err,
 		}
 	}
@@ -309,27 +298,6 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue
 	return t, nil
 }
 
-func validateTunName(tunName string) error {
-	if !strings.Contains(tunName, "%d") {
-		if len(tunName) >= unix.IFNAMSIZ {
-			return fmt.Errorf("tun.dev %q is not shorter than the maximum device name length of %d", tunName, unix.IFNAMSIZ)
-		}
-		return nil
-	}
-	if strings.Count(tunName, "%d") > 1 {
-		return fmt.Errorf("tun.dev template %q may only contain a single %%d", tunName)
-	}
-	if tunName == "%d" {
-		return errors.New("please don't name your tun device '%d'")
-	}
-	// The kernel substitutes the %d itself and requires the template, like a
-	// literal name, to be NUL-terminated within IFNAMSIZ bytes.
-	if len(tunName) >= unix.IFNAMSIZ {
-		return fmt.Errorf("tun.dev template %q is not shorter than the maximum device name length of %d", tunName, unix.IFNAMSIZ)
-	}
-	return nil
-}
-
 // newTunGeneric does all the stuff common to different tun initialization paths. It will close your files on error.
 func newTunGeneric(c *config.C, l *slog.Logger, fd int, vpnNetworks []netip.Prefix) (*tun, error) {
 	tfd, err := newTunFd(fd)
@@ -800,6 +768,7 @@ func (t *tun) isGatewayInVpnNetworks(gwAddr netip.Addr) bool {
 
 func (t *tun) getGatewaysFromRoute(r *netlink.Route) routing.Gateways {
 	var gateways routing.Gateways
+
 	link, err := netlink.LinkByName(t.Device)
 	if err != nil {
 		t.l.Error("Ignoring route update: failed to get link by name", "deviceName", t.Device)
@@ -938,3 +907,5 @@ func (t *tun) Close() error {
 	}
 	return err
 }
+
+func (t *tun) TunPrefixLen() int { return 0 }

overlay/tun_linux_test.go

@@ -3,12 +3,7 @@
 
 package overlay
 
-import (
+import "testing"
-	"strings"
-	"testing"
-
-	"golang.org/x/sys/unix"
-)
 
 var runAdvMSSTests = []struct {
 	name     string
@@ -37,39 +32,3 @@ func TestTunAdvMSS(t *testing.T) {
 		})
 	}
 }
-
-func TestValidateTunName(t *testing.T) {
-	// A device name must be shorter than IFNAMSIZ (i.e. IFNAMSIZ-1 chars max).
-	maxLenName := strings.Repeat("a", unix.IFNAMSIZ-1)
-
-	tests := []struct {
-		name    string
-		tmpl    string
-		wantErr bool
-	}{
-		{"short literal name is fine", "nebula1", false},
-		{"literal name at the max length is fine", maxLenName, false},
-		{"literal name at IFNAMSIZ is rejected", strings.Repeat("a", unix.IFNAMSIZ), true},
-		{"trailing template is fine", "nebula%d", false},
-		{"mid-string template is fine", "neb%dprod", false},
-		{"leading template is fine", "%dnebula", false},
-		{"template at the max length is fine", strings.Repeat("a", unix.IFNAMSIZ-3) + "%d", false},
-		{"template at IFNAMSIZ is rejected", strings.Repeat("a", unix.IFNAMSIZ-2) + "%d", true},
-		{"bare %d is rejected", "%d", true},
-		{"multiple %d is rejected", "neb%d%dprod", true},
-		{"over-long template is rejected", strings.Repeat("a", unix.IFNAMSIZ-1) + "%d", true},
-		{"over-long mid-string template is rejected", "neb%d" + strings.Repeat("a", unix.IFNAMSIZ-3), true},
-	}
-
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			err := validateTunName(tt.tmpl)
-			if tt.wantErr && err == nil {
-				t.Fatalf("expected an error for %q, got none", tt.tmpl)
-			}
-			if !tt.wantErr && err != nil {
-				t.Fatalf("unexpected error for %q: %v", tt.tmpl, err)
-			}
-		})
-	}
-}

overlay/tun_netbsd.go

@@ -58,13 +58,13 @@ type addrLifetime struct {
 }
 
 type tun struct {
+	io.ReadWriteCloser
 	Device      string
 	vpnNetworks []netip.Prefix
 	MTU         int
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
 	l           *slog.Logger
-	f           *os.File
 	fd          int
 }
 
@@ -96,12 +96,12 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 	}
 
 	t := &tun{
-		f:           os.NewFile(uintptr(fd), ""),
+		ReadWriteCloser: os.NewFile(uintptr(fd), ""),
-		fd:          fd,
+		fd:              fd,
-		Device:      deviceName,
+		Device:          deviceName,
-		vpnNetworks: vpnNetworks,
+		vpnNetworks:     vpnNetworks,
-		MTU:         c.GetInt("tun.mtu", DefaultMTU),
+		MTU:             c.GetInt("tun.mtu", DefaultMTU),
-		l:           l,
+		l:               l,
 	}
 
 	err = t.reload(c, true)
@@ -120,12 +120,12 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 }
 
 func (t *tun) Close() error {
-	if t.f != nil {
+	if t.ReadWriteCloser != nil {
-		if err := t.f.Close(); err != nil {
+		if err := t.ReadWriteCloser.Close(); err != nil {
 			return fmt.Errorf("error closing tun file: %w", err)
 		}
 
-		// t.f.Close should have handled it for us but let's be extra sure
+		// Close on the os.File should have handled the fd for us but let's be extra sure
 		_ = unix.Close(t.fd)
 
 		s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_DGRAM, syscall.IPPROTO_IP)
@@ -141,99 +141,6 @@ func (t *tun) Close() error {
 	return nil
 }
 
-func (t *tun) Read(to []byte) (int, error) {
-	rc, err := t.f.SyscallConn()
-	if err != nil {
-		return 0, fmt.Errorf("failed to get syscall conn for tun: %w", err)
-	}
-
-	var errno syscall.Errno
-	var n uintptr
-	err = rc.Read(func(fd uintptr) bool {
-		// first 4 bytes is protocol family, in network byte order
-		head := [4]byte{}
-		iovecs := []syscall.Iovec{
-			{&head[0], 4},
-			{&to[0], uint64(len(to))},
-		}
-
-		n, _, errno = syscall.Syscall(syscall.SYS_READV, fd, uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
-		if errno.Temporary() {
-			// We got an EAGAIN, EINTR, or EWOULDBLOCK, go again
-			return false
-		}
-		return true
-	})
-	if err != nil {
-		if err == syscall.EBADF || err.Error() == "use of closed file" {
-			// Go doesn't export poll.ErrFileClosing but happily reports it to us so here we are
-			// https://github.com/golang/go/blob/master/src/internal/poll/fd_poll_runtime.go#L121
-			return 0, os.ErrClosed
-		}
-		return 0, fmt.Errorf("failed to make read call for tun: %w", err)
-	}
-
-	if errno != 0 {
-		return 0, fmt.Errorf("failed to make inner read call for tun: %w", errno)
-	}
-
-	// fix bytes read number to exclude header
-	bytesRead := int(n)
-	if bytesRead < 0 {
-		return bytesRead, nil
-	} else if bytesRead < 4 {
-		return 0, nil
-	} else {
-		return bytesRead - 4, nil
-	}
-}
-
-// Write is only valid for single threaded use
-func (t *tun) Write(from []byte) (int, error) {
-	if len(from) <= 1 {
-		return 0, syscall.EIO
-	}
-
-	ipVer := from[0] >> 4
-	var head [4]byte
-	// first 4 bytes is protocol family, in network byte order
-	if ipVer == 4 {
-		head[3] = syscall.AF_INET
-	} else if ipVer == 6 {
-		head[3] = syscall.AF_INET6
-	} else {
-		return 0, fmt.Errorf("unable to determine IP version from packet")
-	}
-
-	rc, err := t.f.SyscallConn()
-	if err != nil {
-		return 0, err
-	}
-
-	var errno syscall.Errno
-	var n uintptr
-	err = rc.Write(func(fd uintptr) bool {
-		iovecs := []syscall.Iovec{
-			{&head[0], 4},
-			{&from[0], uint64(len(from))},
-		}
-
-		n, _, errno = syscall.Syscall(syscall.SYS_WRITEV, fd, uintptr(unsafe.Pointer(&iovecs[0])), uintptr(2))
-		// According to NetBSD documentation for TUN, writes will only return errors in which
-		// this packet will never be delivered so just go on living life.
-		return true
-	})
-	if err != nil {
-		return 0, err
-	}
-
-	if errno != 0 {
-		return 0, errno
-	}
-
-	return int(n) - 4, err
-}
-
 func (t *tun) addIp(cidr netip.Prefix) error {
 	if cidr.Addr().Is4() {
 		var req ifreqAlias4
@@ -551,3 +458,7 @@ func delRoute(prefix netip.Prefix, gateways []netip.Prefix) error {
 
 	return nil
 }
+
+// TunPrefixLen reports the 4-byte BSD AF_INET / AF_INET6 protocol-family
+// marker the kernel prepends on read and expects on write.
+func (t *tun) TunPrefixLen() int { return 4 }