Merge remote-tracking branch 'origin/master' into multiport

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -8,13 +8,13 @@ body:
     attributes:
       value: |
         ### Thank you for taking the time to file a bug report!
-        
+
         Please fill out this form as completely as possible.
 
   - type: input
     id: version
     attributes:
-      label: What version of `nebula` are you using?
+      label: What version of `nebula` are you using? (`nebula -version`)
       placeholder: 0.0.0
     validations:
       required: true
@@ -41,10 +41,17 @@ body:
     attributes:
       label: Logs from affected hosts
       description: |
-        Provide logs from all affected hosts during the time of the issue.
+        Please provide logs from ALL affected hosts during the time of the issue. If you do not provide logs we will be unable to assist you!
+
+        [Learn how to find Nebula logs here.](https://nebula.defined.net/docs/guides/viewing-nebula-logs/)
+
         Improve formatting by using <code>```</code> at the beginning and end of each log block.
+      value: |
+        ```
+
+        ```
     validations:
-      required: false
+      required: true
 
   - type: textarea
     id: configs
@@ -52,6 +59,11 @@ body:
       label: Config files from affected hosts
       description: |
         Provide config files for all affected hosts.
+
         Improve formatting by using <code>```</code> at the beginning and end of each config file.
+      value: |
+        ```
+
+        ```
     validations:
-      required: false
+      required: true

.github/workflows/gofmt.yml

@@ -18,7 +18,7 @@ jobs:
 
     - uses: actions/setup-go@v5
       with:
-        go-version-file: 'go.mod'
+        go-version: '1.22'
         check-latest: true
 
     - name: Install goimports

.github/workflows/release.yml

@@ -14,7 +14,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version-file: 'go.mod'
+          go-version: '1.22'
           check-latest: true
 
       - name: Build
@@ -24,7 +24,7 @@ jobs:
           mv build/*.tar.gz release
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: linux-latest
           path: release
@@ -37,7 +37,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version-file: 'go.mod'
+          go-version: '1.22'
           check-latest: true
 
       - name: Build
@@ -55,7 +55,7 @@ jobs:
           mv dist\windows\wintun build\dist\windows\
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: windows-latest
           path: build
@@ -70,7 +70,7 @@ jobs:
 
       - uses: actions/setup-go@v5
         with:
-          go-version-file: 'go.mod'
+          go-version: '1.22'
           check-latest: true
 
       - name: Import certificates
@@ -104,11 +104,57 @@ jobs:
           fi
 
       - name: Upload artifacts
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: darwin-latest
           path: ./release/*
 
+  build-docker:
+    name: Create and Upload Docker Images
+    # Technically we only need build-linux to succeed, but if any platforms fail we'll
+    # want to investigate and restart the build
+    needs: [build-linux, build-darwin, build-windows]
+    runs-on: ubuntu-latest
+    env:
+      HAS_DOCKER_CREDS: ${{ vars.DOCKERHUB_USERNAME != '' && secrets.DOCKERHUB_TOKEN != '' }}
+    # XXX It's not possible to write a conditional here, so instead we do it on every step
+    #if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
+    steps:
+      # Be sure to checkout the code before downloading artifacts, or they will
+      # be overwritten
+      - name: Checkout code
+        if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
+        uses: actions/checkout@v4
+
+      - name: Download artifacts
+        if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
+        uses: actions/download-artifact@v4
+        with:
+          name: linux-latest
+          path: artifacts
+
+      - name: Login to Docker Hub
+        if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push images
+        if: ${{ env.HAS_DOCKER_CREDS == 'true' }}
+        env:
+          DOCKER_IMAGE_REPO: ${{ vars.DOCKER_IMAGE_REPO || 'nebulaoss/nebula' }}
+          DOCKER_IMAGE_TAG: ${{ vars.DOCKER_IMAGE_TAG || 'latest' }}
+        run: |
+          mkdir -p build/linux-{amd64,arm64}
+          tar -zxvf artifacts/nebula-linux-amd64.tar.gz -C build/linux-amd64/
+          tar -zxvf artifacts/nebula-linux-arm64.tar.gz -C build/linux-arm64/
+          docker buildx build . --push -f docker/Dockerfile --platform linux/amd64,linux/arm64 --tag "${DOCKER_IMAGE_REPO}:${DOCKER_IMAGE_TAG}" --tag "${DOCKER_IMAGE_REPO}:${GITHUB_REF#refs/tags/v}"
+
   release:
     name: Create and Upload Release
     needs: [build-linux, build-darwin, build-windows]
@@ -117,7 +163,7 @@ jobs:
       - uses: actions/checkout@v4
 
       - name: Download artifacts
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: artifacts
 

.github/workflows/smoke-extra.yml

@@ -0,0 +1,48 @@
+name: smoke-extra
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    types: [opened, synchronize, labeled, reopened]
+    paths:
+      - '.github/workflows/smoke**'
+      - '**Makefile'
+      - '**.go'
+      - '**.proto'
+      - 'go.mod'
+      - 'go.sum'
+jobs:
+
+  smoke-extra:
+    if: github.ref == 'refs/heads/master' || contains(github.event.pull_request.labels.*.name, 'smoke-test-extra')
+    name: Run extra smoke tests
+    runs-on: ubuntu-latest
+    steps:
+
+    - uses: actions/checkout@v4
+
+    - uses: actions/setup-go@v5
+      with:
+        go-version-file: 'go.mod'
+        check-latest: true
+
+    - name: install vagrant
+      run: sudo apt-get update && sudo apt-get install -y vagrant virtualbox
+
+    - name: freebsd-amd64
+      run: make smoke-vagrant/freebsd-amd64
+
+    - name: openbsd-amd64
+      run: make smoke-vagrant/openbsd-amd64
+
+    - name: netbsd-amd64
+      run: make smoke-vagrant/netbsd-amd64
+
+    - name: linux-386
+      run: make smoke-vagrant/linux-386
+
+    - name: linux-amd64-ipv6disable
+      run: make smoke-vagrant/linux-amd64-ipv6disable
+
+    timeout-minutes: 30

.github/workflows/smoke.yml

@@ -22,7 +22,7 @@ jobs:
 
     - uses: actions/setup-go@v5
       with:
-        go-version-file: 'go.mod'
+        go-version: '1.22'
         check-latest: true
 
     - name: build

.github/workflows/smoke/build.sh

@@ -11,6 +11,11 @@ mkdir ./build
     cp ../../../../build/linux-amd64/nebula .
     cp ../../../../build/linux-amd64/nebula-cert .
 
+    if [ "$1" ]
+    then
+        cp "../../../../build/$1/nebula" "$1-nebula"
+    fi
+
     HOST="lighthouse1" \
         AM_LIGHTHOUSE=true \
         ../genconfig.sh >lighthouse1.yml

.github/workflows/smoke/genconfig.sh

@@ -47,7 +47,7 @@ listen:
   port: ${LISTEN_PORT:-4242}
 
 tun:
-  dev: ${TUN_DEV:-nebula1}
+  dev: ${TUN_DEV:-tun0}
   multiport:
     tx_enabled: ${MULTIPORT_TX:-false}
     rx_enabled: ${MULTIPORT_RX:-false}

.github/workflows/smoke/smoke-relay.sh

@@ -76,7 +76,7 @@ docker exec host4 sh -c 'kill 1'
 docker exec host3 sh -c 'kill 1'
 docker exec host2 sh -c 'kill 1'
 docker exec lighthouse1 sh -c 'kill 1'
-sleep 1
+sleep 5
 
 if [ "$(jobs -r)" ]
 then

.github/workflows/smoke/smoke-vagrant.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+set -e -x
+
+set -o pipefail
+
+export VAGRANT_CWD="$PWD/vagrant-$1"
+
+mkdir -p logs
+
+cleanup() {
+    echo
+    echo " *** cleanup"
+    echo
+
+    set +e
+    if [ "$(jobs -r)" ]
+    then
+        docker kill lighthouse1 host2
+    fi
+    vagrant destroy -f
+}
+
+trap cleanup EXIT
+
+CONTAINER="nebula:${NAME:-smoke}"
+
+docker run --name lighthouse1 --rm "$CONTAINER" -config lighthouse1.yml -test
+docker run --name host2 --rm "$CONTAINER" -config host2.yml -test
+
+vagrant up
+vagrant ssh -c "cd /nebula && /nebula/$1-nebula -config host3.yml -test"
+
+docker run --name lighthouse1 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config lighthouse1.yml 2>&1 | tee logs/lighthouse1 | sed -u 's/^/  [lighthouse1]  /' &
+sleep 1
+docker run --name host2 --device /dev/net/tun:/dev/net/tun --cap-add NET_ADMIN --rm "$CONTAINER" -config host2.yml 2>&1 | tee logs/host2 | sed -u 's/^/  [host2]  /' &
+sleep 1
+vagrant ssh -c "cd /nebula && sudo sh -c 'echo \$\$ >/nebula/pid && exec /nebula/$1-nebula -config host3.yml'" &
+sleep 15
+
+# grab tcpdump pcaps for debugging
+docker exec lighthouse1 tcpdump -i nebula1 -q -w - -U 2>logs/lighthouse1.inside.log >logs/lighthouse1.inside.pcap &
+docker exec lighthouse1 tcpdump -i eth0 -q -w - -U 2>logs/lighthouse1.outside.log >logs/lighthouse1.outside.pcap &
+docker exec host2 tcpdump -i nebula1 -q -w - -U 2>logs/host2.inside.log >logs/host2.inside.pcap &
+docker exec host2 tcpdump -i eth0 -q -w - -U 2>logs/host2.outside.log >logs/host2.outside.pcap &
+# vagrant ssh -c "tcpdump -i nebula1 -q -w - -U" 2>logs/host3.inside.log >logs/host3.inside.pcap &
+# vagrant ssh -c "tcpdump -i eth0 -q -w - -U" 2>logs/host3.outside.log >logs/host3.outside.pcap &
+
+docker exec host2 ncat -nklv 0.0.0.0 2000 &
+vagrant ssh -c "ncat -nklv 0.0.0.0 2000" &
+#docker exec host2 ncat -e '/usr/bin/echo host2' -nkluv 0.0.0.0 3000 &
+#vagrant ssh -c "ncat -e '/usr/bin/echo host3' -nkluv 0.0.0.0 3000" &
+
+set +x
+echo
+echo " *** Testing ping from lighthouse1"
+echo
+set -x
+docker exec lighthouse1 ping -c1 192.168.100.2
+docker exec lighthouse1 ping -c1 192.168.100.3
+
+set +x
+echo
+echo " *** Testing ping from host2"
+echo
+set -x
+docker exec host2 ping -c1 192.168.100.1
+# Should fail because not allowed by host3 inbound firewall
+! docker exec host2 ping -c1 192.168.100.3 -w5 || exit 1
+
+set +x
+echo
+echo " *** Testing ncat from host2"
+echo
+set -x
+# Should fail because not allowed by host3 inbound firewall
+#! docker exec host2 ncat -nzv -w5 192.168.100.3 2000 || exit 1
+#! docker exec host2 ncat -nzuv -w5 192.168.100.3 3000 | grep -q host3 || exit 1
+
+set +x
+echo
+echo " *** Testing ping from host3"
+echo
+set -x
+vagrant ssh -c "ping -c1 192.168.100.1"
+vagrant ssh -c "ping -c1 192.168.100.2"
+
+set +x
+echo
+echo " *** Testing ncat from host3"
+echo
+set -x
+#vagrant ssh -c "ncat -nzv -w5 192.168.100.2 2000"
+#vagrant ssh -c "ncat -nzuv -w5 192.168.100.2 3000" | grep -q host2
+
+vagrant ssh -c "sudo xargs kill </nebula/pid"
+docker exec host2 sh -c 'kill 1'
+docker exec lighthouse1 sh -c 'kill 1'
+sleep 1
+
+if [ "$(jobs -r)" ]
+then
+    echo "nebula still running after SIGTERM sent" >&2
+    exit 1
+fi

.github/workflows/smoke/smoke.sh

@@ -129,7 +129,7 @@ docker exec host4 sh -c 'kill 1'
 docker exec host3 sh -c 'kill 1'
 docker exec host2 sh -c 'kill 1'
 docker exec lighthouse1 sh -c 'kill 1'
-sleep 1
+sleep 5
 
 if [ "$(jobs -r)" ]
 then

.github/workflows/smoke/vagrant-freebsd-amd64/Vagrantfile

@@ -0,0 +1,7 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/freebsd14"
+
+  config.vm.synced_folder "../build", "/nebula", type: "rsync"
+end

.github/workflows/smoke/vagrant-linux-386/Vagrantfile

@@ -0,0 +1,7 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+Vagrant.configure("2") do |config|
+  config.vm.box = "ubuntu/xenial32"
+
+  config.vm.synced_folder "../build", "/nebula"
+end

.github/workflows/smoke/vagrant-linux-amd64-ipv6disable/Vagrantfile

@@ -0,0 +1,16 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+Vagrant.configure("2") do |config|
+  config.vm.box = "ubuntu/jammy64"
+
+  config.vm.synced_folder "../build", "/nebula"
+
+  config.vm.provision :shell do |shell|
+    shell.inline = <<-EOF
+      sed -i 's/GRUB_CMDLINE_LINUX=""/GRUB_CMDLINE_LINUX="ipv6.disable=1"/' /etc/default/grub
+      update-grub
+    EOF
+    shell.privileged = true
+    shell.reboot = true
+  end
+end

.github/workflows/smoke/vagrant-netbsd-amd64/Vagrantfile

@@ -0,0 +1,7 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/netbsd9"
+
+  config.vm.synced_folder "../build", "/nebula", type: "rsync"
+end

.github/workflows/smoke/vagrant-openbsd-amd64/Vagrantfile

@@ -0,0 +1,7 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/openbsd7"
+
+  config.vm.synced_folder "../build", "/nebula", type: "rsync"
+end

.github/workflows/test.yml

@@ -22,7 +22,7 @@ jobs:
 
     - uses: actions/setup-go@v5
       with:
-        go-version-file: 'go.mod'
+        go-version: '1.22'
         check-latest: true
 
     - name: Build
@@ -40,10 +40,10 @@ jobs:
     - name: Build test mobile
       run: make build-test-mobile
 
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
-        name: e2e packet flow
+        name: e2e packet flow linux-latest
-        path: e2e/mermaid/
+        path: e2e/mermaid/linux-latest
         if-no-files-found: warn
 
   test-linux-boringcrypto:
@@ -55,7 +55,7 @@ jobs:
 
     - uses: actions/setup-go@v5
       with:
-        go-version-file: 'go.mod'
+        go-version: '1.22'
         check-latest: true
 
     - name: Build
@@ -79,7 +79,7 @@ jobs:
 
     - uses: actions/setup-go@v5
       with:
-        go-version-file: 'go.mod'
+        go-version: '1.22'
         check-latest: true
 
     - name: Build nebula
@@ -97,8 +97,8 @@ jobs:
     - name: End 2 end
       run: make e2evv
 
-    - uses: actions/upload-artifact@v3
+    - uses: actions/upload-artifact@v4
       with:
-        name: e2e packet flow
+        name: e2e packet flow ${{ matrix.os }}
-        path: e2e/mermaid/
+        path: e2e/mermaid/${{ matrix.os }}
         if-no-files-found: warn

CHANGELOG.md

@@ -7,6 +7,74 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.9.0] - 2024-05-07
+
+### Deprecated
+
+- This release adds a new setting `default_local_cidr_any` that defaults to
+  true to match previous behavior, but will default to false in the next
+  release (1.10). When set to false, `local_cidr` is matched correctly for
+  firewall rules on hosts acting as unsafe routers, and should be set for any
+  firewall rules you want to allow unsafe route hosts to access. See the issue
+  and example config for more details. (#1071, #1099)
+
+### Added
+
+- Nebula now has an official Docker image `nebulaoss/nebula` that is
+  distroless and contains just the `nebula` and `nebula-cert` binaries. You
+  can find it here: https://hub.docker.com/r/nebulaoss/nebula (#1037)
+
+- Experimental binaries for `loong64` are now provided. (#1003)
+
+- Added example service script for OpenRC. (#711)
+
+- The SSH daemon now supports inlined host keys. (#1054)
+
+- The SSH daemon now supports certificates with `sshd.trusted_cas`. (#1098)
+
+### Changed
+
+- Config setting `tun.unsafe_routes` is now reloadable. (#1083)
+
+- Small documentation and internal improvements. (#1065, #1067, #1069, #1108,
+  #1109, #1111, #1135)
+
+- Various dependency updates. (#1139, #1138, #1134, #1133, #1126, #1123, #1110,
+  #1094, #1092, #1087, #1086, #1085, #1072, #1063, #1059, #1055, #1053, #1047,
+  #1046, #1034, #1022)
+
+### Removed
+
+- Support for the deprecated `local_range` option has been removed. Please
+  change to `preferred_ranges` (which is also now reloadable). (#1043)
+
+- We are now building with go1.22, which means that for Windows you need at
+  least Windows 10 or Windows Server 2016. This is because support for earlier
+  versions was removed in Go 1.21. See https://go.dev/doc/go1.21#windows (#981)
+
+- Removed vagrant example, as it was unmaintained. (#1129)
+
+- Removed Fedora and Arch nebula.service files, as they are maintained in the
+  upstream repos. (#1128, #1132)
+
+- Remove the TCP round trip tracking metrics, as they never had correct data
+  and were an experiment to begin with. (#1114)
+
+### Fixed
+
+- Fixed a potential deadlock introduced in 1.8.1. (#1112)
+
+- Fixed support for Linux when IPv6 has been disabled at the OS level. (#787)
+
+- DNS will return NXDOMAIN now when there are no results. (#845)
+
+- Allow `::` in `lighthouse.dns.host`. (#1115)
+
+- Capitalization of `NotAfter` fixed in DNS TXT response. (#1127)
+
+- Don't log invalid certificates. It is untrusted data and can cause a large
+  volume of logs. (#1116)
+
 ## [1.8.2] - 2024-01-08
 
 ### Fixed
@@ -558,7 +626,8 @@ created.)
 
 - Initial public release.
 
-[Unreleased]: https://github.com/slackhq/nebula/compare/v1.8.2...HEAD
+[Unreleased]: https://github.com/slackhq/nebula/compare/v1.9.0...HEAD
+[1.9.0]: https://github.com/slackhq/nebula/releases/tag/v1.9.0
 [1.8.2]: https://github.com/slackhq/nebula/releases/tag/v1.8.2
 [1.8.1]: https://github.com/slackhq/nebula/releases/tag/v1.8.1
 [1.8.0]: https://github.com/slackhq/nebula/releases/tag/v1.8.0

LOGGING.md

@@ -33,6 +33,5 @@ l.WithError(err).
     WithField("vpnIp", IntIp(hostinfo.hostId)).
     WithField("udpAddr", addr).
     WithField("handshake", m{"stage": 1, "style": "ix"}).
-    WithField("cert", remoteCert).
     Info("Invalid certificate from host")
 ```

Makefile

@@ -1,22 +1,14 @@
-GOMINVERSION = 1.20
 NEBULA_CMD_PATH = "./cmd/nebula"
-GO111MODULE = on
-export GO111MODULE
 CGO_ENABLED = 0
 export CGO_ENABLED
 
 # Set up OS specific bits
 ifeq ($(OS),Windows_NT)
-	#TODO: we should be able to ditch awk as well
-	GOVERSION := $(shell go version | awk "{print substr($$3, 3)}")
-	GOISMIN := $(shell IF "$(GOVERSION)" GEQ "$(GOMINVERSION)" ECHO 1)
 	NEBULA_CMD_SUFFIX = .exe
 	NULL_FILE = nul
 	# RIO on windows does pointer stuff that makes go vet angry
 	VET_FLAGS = -unsafeptr=false
 else
-	GOVERSION := $(shell go version | awk '{print substr($$3, 3)}')
-	GOISMIN := $(shell expr "$(GOVERSION)" ">=" "$(GOMINVERSION)")
 	NEBULA_CMD_SUFFIX =
 	NULL_FILE = /dev/null
 endif
@@ -30,6 +22,9 @@ ifndef BUILD_NUMBER
 	endif
 endif
 
+DOCKER_IMAGE_REPO ?= nebulaoss/nebula
+DOCKER_IMAGE_TAG ?= latest
+
 LDFLAGS = -X main.Build=$(BUILD_NUMBER)
 
 ALL_LINUX = linux-amd64 \
@@ -44,7 +39,8 @@ ALL_LINUX = linux-amd64 \
 	linux-mips64 \
 	linux-mips64le \
 	linux-mips-softfloat \
-	linux-riscv64
+	linux-riscv64 \
+        linux-loong64
 
 ALL_FREEBSD = freebsd-amd64 \
 	freebsd-arm64
@@ -82,8 +78,12 @@ e2evvvv: e2ev
 e2e-bench: TEST_FLAGS = -bench=. -benchmem -run=^$
 e2e-bench: e2e
 
+DOCKER_BIN = build/linux-amd64/nebula build/linux-amd64/nebula-cert
+
 all: $(ALL:%=build/%/nebula) $(ALL:%=build/%/nebula-cert)
 
+docker: docker/linux-$(shell go env GOARCH)
+
 release: $(ALL:%=build/nebula-%.tar.gz)
 
 release-linux: $(ALL_LINUX:%=build/nebula-%.tar.gz)
@@ -156,6 +156,9 @@ build/nebula-%.tar.gz: build/%/nebula build/%/nebula-cert
 build/nebula-%.zip: build/%/nebula.exe build/%/nebula-cert.exe
 	cd build/$* && zip ../nebula-$*.zip nebula.exe nebula-cert.exe
 
+docker/%: build/%/nebula build/%/nebula-cert
+	docker build . $(DOCKER_BUILD_ARGS) -f docker/Dockerfile --platform "$(subst -,/,$*)" --tag "${DOCKER_IMAGE_REPO}:${DOCKER_IMAGE_TAG}" --tag "${DOCKER_IMAGE_REPO}:$(BUILD_NUMBER)"
+
 vet:
 	go vet $(VET_FLAGS) -v ./...
 
@@ -223,6 +226,10 @@ smoke-docker-race: BUILD_ARGS = -race
 smoke-docker-race: CGO_ENABLED = 1
 smoke-docker-race: smoke-docker
 
+smoke-vagrant/%: bin-docker build/%/nebula
+	cd .github/workflows/smoke/ && ./build.sh $*
+	cd .github/workflows/smoke/ && ./smoke-vagrant.sh $*
+
 .FORCE:
-.PHONY: bench bench-cpu bench-cpu-long bin build-test-mobile e2e e2ev e2evv e2evvv e2evvvv proto release service smoke-docker smoke-docker-race test test-cov-html
+.PHONY: bench bench-cpu bench-cpu-long bin build-test-mobile e2e e2ev e2evv e2evvv e2evvvv proto release service smoke-docker smoke-docker-race test test-cov-html smoke-vagrant/%
 .DEFAULT_GOAL := bin

README.md

@@ -52,6 +52,11 @@ Check the [releases](https://github.com/slackhq/nebula/releases/latest) page for
     $ brew install nebula
     ```
 
+- [Docker](https://hub.docker.com/r/nebulaoss/nebula)
+    ```
+    $ docker pull nebulaoss/nebula
+    ```
+
 #### Mobile
 
 - [iOS](https://apps.apple.com/us/app/mobile-nebula/id1509587936?itsct=apps_box&itscg=30200)

cert/cert.go

@@ -324,7 +324,7 @@ func UnmarshalEd25519PrivateKey(b []byte) (ed25519.PrivateKey, []byte, error) {
 	return k.Bytes, r, nil
 }
 
-// UnmarshalNebulaCertificate will unmarshal a protobuf byte representation of a nebula cert into its
+// UnmarshalNebulaEncryptedData will unmarshal a protobuf byte representation of a nebula cert into its
 // protobuf-generated struct.
 func UnmarshalNebulaEncryptedData(b []byte) (*NebulaEncryptedData, error) {
 	if len(b) == 0 {

cidr/tree4.go

@@ -142,15 +142,22 @@ func (tree *Tree4[T]) MostSpecificContains(ip iputil.VpnIp) (ok bool, value T) {
 	return ok, value
 }
 
-// Match finds the most specific match
+type eachFunc[T any] func(T) bool
-// TODO this is exact match
+
-func (tree *Tree4[T]) Match(ip iputil.VpnIp) (ok bool, value T) {
+// EachContains will call a function, passing the value, for each entry until the function returns true or the search is complete
+// The final return value will be true if the provided function returned true
+func (tree *Tree4[T]) EachContains(ip iputil.VpnIp, each eachFunc[T]) bool {
 	bit := startbit
 	node := tree.root
-	lastNode := node
 
 	for node != nil {
-		lastNode = node
+		if node.hasValue {
+			// If the each func returns true then we can exit the loop
+			if each(node.value) {
+				return true
+			}
+		}
+
 		if ip&bit != 0 {
 			node = node.right
 		} else {
@@ -160,10 +167,33 @@ func (tree *Tree4[T]) Match(ip iputil.VpnIp) (ok bool, value T) {
 		bit >>= 1
 	}
 
-	if bit == 0 && lastNode != nil {
+	return false
-		value = lastNode.value
+}
-		ok = true
+
+// GetCIDR returns the entry added by the most recent matching AddCIDR call
+func (tree *Tree4[T]) GetCIDR(cidr *net.IPNet) (ok bool, value T) {
+	bit := startbit
+	node := tree.root
+
+	ip := iputil.Ip2VpnIp(cidr.IP)
+	mask := iputil.Ip2VpnIp(cidr.Mask)
+
+	// Find our last ancestor in the tree
+	for node != nil && bit&mask != 0 {
+		if ip&bit != 0 {
+			node = node.right
+		} else {
+			node = node.left
+		}
+
+		bit = bit >> 1
 	}
+
+	if bit&mask == 0 && node != nil {
+		value = node.value
+		ok = node.hasValue
+	}
+
 	return ok, value
 }
 

cidr/tree4_test.go

@@ -115,35 +115,36 @@ func TestCIDRTree_MostSpecificContains(t *testing.T) {
 	assert.Equal(t, "cool", r)
 }
 
-func TestCIDRTree_Match(t *testing.T) {
+func TestTree4_GetCIDR(t *testing.T) {
 	tree := NewTree4[string]()
-	tree.AddCIDR(Parse("4.1.1.0/32"), "1a")
+	tree.AddCIDR(Parse("1.0.0.0/8"), "1")
-	tree.AddCIDR(Parse("4.1.1.1/32"), "1b")
+	tree.AddCIDR(Parse("2.1.0.0/16"), "2")
+	tree.AddCIDR(Parse("3.1.1.0/24"), "3")
+	tree.AddCIDR(Parse("4.1.1.0/24"), "4a")
+	tree.AddCIDR(Parse("4.1.1.1/32"), "4b")
+	tree.AddCIDR(Parse("4.1.2.1/32"), "4c")
+	tree.AddCIDR(Parse("254.0.0.0/4"), "5")
 
 	tests := []struct {
 		Found  bool
 		Result interface{}
-		IP     string
+		IPNet  *net.IPNet
 	}{
-		{true, "1a", "4.1.1.0"},
+		{true, "1", Parse("1.0.0.0/8")},
-		{true, "1b", "4.1.1.1"},
+		{true, "2", Parse("2.1.0.0/16")},
+		{true, "3", Parse("3.1.1.0/24")},
+		{true, "4a", Parse("4.1.1.0/24")},
+		{true, "4b", Parse("4.1.1.1/32")},
+		{true, "4c", Parse("4.1.2.1/32")},
+		{true, "5", Parse("254.0.0.0/4")},
+		{false, "", Parse("2.0.0.0/8")},
 	}
 
 	for _, tt := range tests {
-		ok, r := tree.Match(iputil.Ip2VpnIp(net.ParseIP(tt.IP)))
+		ok, r := tree.GetCIDR(tt.IPNet)
 		assert.Equal(t, tt.Found, ok)
 		assert.Equal(t, tt.Result, r)
 	}
-
-	tree = NewTree4[string]()
-	tree.AddCIDR(Parse("1.1.1.1/0"), "cool")
-	ok, r := tree.Contains(iputil.Ip2VpnIp(net.ParseIP("0.0.0.0")))
-	assert.True(t, ok)
-	assert.Equal(t, "cool", r)
-
-	ok, r = tree.Contains(iputil.Ip2VpnIp(net.ParseIP("255.255.255.255")))
-	assert.True(t, ok)
-	assert.Equal(t, "cool", r)
 }
 
 func BenchmarkCIDRTree_Contains(b *testing.B) {
@@ -167,25 +168,3 @@ func BenchmarkCIDRTree_Contains(b *testing.B) {
 		}
 	})
 }
-
-func BenchmarkCIDRTree_Match(b *testing.B) {
-	tree := NewTree4[string]()
-	tree.AddCIDR(Parse("1.1.0.0/16"), "1")
-	tree.AddCIDR(Parse("1.2.1.1/32"), "1")
-	tree.AddCIDR(Parse("192.2.1.1/32"), "1")
-	tree.AddCIDR(Parse("172.2.1.1/32"), "1")
-
-	ip := iputil.Ip2VpnIp(net.ParseIP("1.2.1.1"))
-	b.Run("found", func(b *testing.B) {
-		for i := 0; i < b.N; i++ {
-			tree.Match(ip)
-		}
-	})
-
-	ip = iputil.Ip2VpnIp(net.ParseIP("1.2.1.255"))
-	b.Run("not found", func(b *testing.B) {
-		for i := 0; i < b.N; i++ {
-			tree.Match(ip)
-		}
-	})
-}

cmd/nebula-cert/ca.go

@@ -180,9 +180,15 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error
 		if err != nil {
 			return fmt.Errorf("error while generating ecdsa keys: %s", err)
 		}
-		// ref: https://github.com/golang/go/blob/go1.19/src/crypto/x509/sec1.go#L60
+
-		rawPriv = key.D.FillBytes(make([]byte, 32))
+		// ecdh.PrivateKey lets us get at the encoded bytes, even though
-		pub = elliptic.Marshal(elliptic.P256(), key.X, key.Y)
+		// we aren't using ECDH here.
+		eKey, err := key.ECDH()
+		if err != nil {
+			return fmt.Errorf("error while converting ecdsa key: %s", err)
+		}
+		rawPriv = eKey.Bytes()
+		pub = eKey.PublicKey().Bytes()
 	}
 
 	nc := cert.NebulaCertificate{

connection_manager.go

@@ -457,7 +457,7 @@ func (n *connectionManager) sendPunch(hostinfo *HostInfo) {
 	}
 
 	if n.punchy.GetTargetEverything() {
-		hostinfo.remotes.ForEach(n.hostMap.preferredRanges, func(addr *udp.Addr, preferred bool) {
+		hostinfo.remotes.ForEach(n.hostMap.GetPreferredRanges(), func(addr *udp.Addr, preferred bool) {
 			n.metricsTxPunchy.Inc(1)
 			n.intf.outside.WriteTo([]byte{1}, addr)
 		})

connection_manager_test.go

@@ -43,7 +43,9 @@ func Test_NewConnectionManagerTest(t *testing.T) {
 	preferredRanges := []*net.IPNet{localrange}
 
 	// Very incomplete mock objects
-	hostMap := NewHostMap(l, vpncidr, preferredRanges)
+	hostMap := newHostMap(l, vpncidr)
+	hostMap.preferredRanges.Store(&preferredRanges)
+
 	cs := &CertState{
 		RawCertificate:      []byte{},
 		PrivateKey:          []byte{},
@@ -123,7 +125,9 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
 	preferredRanges := []*net.IPNet{localrange}
 
 	// Very incomplete mock objects
-	hostMap := NewHostMap(l, vpncidr, preferredRanges)
+	hostMap := newHostMap(l, vpncidr)
+	hostMap.preferredRanges.Store(&preferredRanges)
+
 	cs := &CertState{
 		RawCertificate:      []byte{},
 		PrivateKey:          []byte{},
@@ -210,7 +214,8 @@ func Test_NewConnectionManagerTest_DisconnectInvalid(t *testing.T) {
 	_, vpncidr, _ := net.ParseCIDR("172.1.1.1/24")
 	_, localrange, _ := net.ParseCIDR("10.1.1.1/24")
 	preferredRanges := []*net.IPNet{localrange}
-	hostMap := NewHostMap(l, vpncidr, preferredRanges)
+	hostMap := newHostMap(l, vpncidr)
+	hostMap.preferredRanges.Store(&preferredRanges)
 
 	// Generate keys for CA and peer's cert.
 	pubCA, privCA, _ := ed25519.GenerateKey(rand.Reader)

control.go

@@ -145,7 +145,7 @@ func (c *Control) GetHostInfoByVpnIp(vpnIp iputil.VpnIp, pending bool) *ControlH
 		return nil
 	}
 
-	ch := copyHostInfo(h, c.f.hostMap.preferredRanges)
+	ch := copyHostInfo(h, c.f.hostMap.GetPreferredRanges())
 	return &ch
 }
 
@@ -157,7 +157,7 @@ func (c *Control) SetRemoteForTunnel(vpnIp iputil.VpnIp, addr udp.Addr) *Control
 	}
 
 	hostInfo.SetRemote(addr.Copy())
-	ch := copyHostInfo(hostInfo, c.f.hostMap.preferredRanges)
+	ch := copyHostInfo(hostInfo, c.f.hostMap.GetPreferredRanges())
 	return &ch
 }
 

control_test.go

@@ -18,7 +18,9 @@ func TestControl_GetHostInfoByVpnIp(t *testing.T) {
 	l := test.NewLogger()
 	// Special care must be taken to re-use all objects provided to the hostmap and certificate in the expectedInfo object
 	// To properly ensure we are not exposing core memory to the caller
-	hm := NewHostMap(l, &net.IPNet{}, make([]*net.IPNet, 0))
+	hm := newHostMap(l, &net.IPNet{})
+	hm.preferredRanges.Store(&[]*net.IPNet{})
+
 	remote1 := udp.NewAddr(net.ParseIP("0.0.0.100"), 4444)
 	remote2 := udp.NewAddr(net.ParseIP("1:2:3:4:5:6:7:8"), 4444)
 	ipNet := net.IPNet{

dist/arch/nebula.service

@@ -1,15 +0,0 @@
-[Unit]
-Description=Nebula overlay networking tool
-Wants=basic.target network-online.target nss-lookup.target time-sync.target
-After=basic.target network.target network-online.target
-
-[Service]
-Type=notify
-NotifyAccess=main
-SyslogIdentifier=nebula
-ExecReload=/bin/kill -HUP $MAINPID
-ExecStart=/usr/bin/nebula -config /etc/nebula/config.yml
-Restart=always
-
-[Install]
-WantedBy=multi-user.target

dist/fedora/nebula.service

@@ -1,16 +0,0 @@
-[Unit]
-Description=Nebula overlay networking tool
-Wants=basic.target network-online.target nss-lookup.target time-sync.target
-After=basic.target network.target network-online.target
-Before=sshd.service
-
-[Service]
-Type=notify
-NotifyAccess=main
-SyslogIdentifier=nebula
-ExecReload=/bin/kill -HUP $MAINPID
-ExecStart=/usr/bin/nebula -config /etc/nebula/config.yml
-Restart=always
-
-[Install]
-WantedBy=multi-user.target

dns_server.go

@@ -56,7 +56,7 @@ func (d *dnsRecords) QueryCert(data string) string {
 		return ""
 	}
 	cert := q.Details
-	c := fmt.Sprintf("\"Name: %s\" \"Ips: %s\" \"Subnets %s\" \"Groups %s\" \"NotBefore %s\" \"NotAFter %s\" \"PublicKey %x\" \"IsCA %t\" \"Issuer %s\"", cert.Name, cert.Ips, cert.Subnets, cert.Groups, cert.NotBefore, cert.NotAfter, cert.PublicKey, cert.IsCA, cert.Issuer)
+	c := fmt.Sprintf("\"Name: %s\" \"Ips: %s\" \"Subnets %s\" \"Groups %s\" \"NotBefore %s\" \"NotAfter %s\" \"PublicKey %x\" \"IsCA %t\" \"Issuer %s\"", cert.Name, cert.Ips, cert.Subnets, cert.Groups, cert.NotBefore, cert.NotAfter, cert.PublicKey, cert.IsCA, cert.Issuer)
 	return c
 }
 
@@ -96,6 +96,10 @@ func parseQuery(l *logrus.Logger, m *dns.Msg, w dns.ResponseWriter) {
 			}
 		}
 	}
+
+	if len(m.Answer) == 0 {
+		m.Rcode = dns.RcodeNameError
+	}
 }
 
 func handleDnsRequest(l *logrus.Logger, w dns.ResponseWriter, r *dns.Msg) {
@@ -129,7 +133,12 @@ func dnsMain(l *logrus.Logger, hostMap *HostMap, c *config.C) func() {
 }
 
 func getDnsServerAddr(c *config.C) string {
-	return c.GetString("lighthouse.dns.host", "") + ":" + strconv.Itoa(c.GetInt("lighthouse.dns.port", 53))
+	dnsHost := strings.TrimSpace(c.GetString("lighthouse.dns.host", ""))
+	// Old guidance was to provide the literal `[::]` in `lighthouse.dns.host` but that won't resolve.
+	if dnsHost == "[::]" {
+		dnsHost = "::"
+	}
+	return net.JoinHostPort(dnsHost, strconv.Itoa(c.GetInt("lighthouse.dns.port", 53)))
 }
 
 func startDns(l *logrus.Logger, c *config.C) {

dns_server_test.go

@@ -4,6 +4,8 @@ import (
 	"testing"
 
 	"github.com/miekg/dns"
+	"github.com/slackhq/nebula/config"
+	"github.com/stretchr/testify/assert"
 )
 
 func TestParsequery(t *testing.T) {
@@ -17,3 +19,40 @@ func TestParsequery(t *testing.T) {
 
 	//parseQuery(m)
 }
+
+func Test_getDnsServerAddr(t *testing.T) {
+	c := config.NewC(nil)
+
+	c.Settings["lighthouse"] = map[interface{}]interface{}{
+		"dns": map[interface{}]interface{}{
+			"host": "0.0.0.0",
+			"port": "1",
+		},
+	}
+	assert.Equal(t, "0.0.0.0:1", getDnsServerAddr(c))
+
+	c.Settings["lighthouse"] = map[interface{}]interface{}{
+		"dns": map[interface{}]interface{}{
+			"host": "::",
+			"port": "1",
+		},
+	}
+	assert.Equal(t, "[::]:1", getDnsServerAddr(c))
+
+	c.Settings["lighthouse"] = map[interface{}]interface{}{
+		"dns": map[interface{}]interface{}{
+			"host": "[::]",
+			"port": "1",
+		},
+	}
+	assert.Equal(t, "[::]:1", getDnsServerAddr(c))
+
+	// Make sure whitespace doesn't mess us up
+	c.Settings["lighthouse"] = map[interface{}]interface{}{
+		"dns": map[interface{}]interface{}{
+			"host": "[::] ",
+			"port": "1",
+		},
+	}
+	assert.Equal(t, "[::]:1", getDnsServerAddr(c))
+}

docker/Dockerfile

@@ -0,0 +1,11 @@
+FROM gcr.io/distroless/static:latest
+
+ARG TARGETOS TARGETARCH
+COPY build/$TARGETOS-$TARGETARCH/nebula /nebula
+COPY build/$TARGETOS-$TARGETARCH/nebula-cert /nebula-cert
+
+VOLUME ["/config"]
+
+ENTRYPOINT ["/nebula"]
+# Allow users to override the args passed to nebula
+CMD ["-config", "/config/config.yml"]

docker/README.md

@@ -0,0 +1,24 @@
+# NebulaOSS/nebula Docker Image
+
+## Building
+
+From the root of the repository, run `make docker`.
+
+## Running
+
+To run the built image, use the following command:
+
+```
+docker run \
+    --name nebula \
+    --network host \
+    --cap-add NET_ADMIN \
+    --volume ./config:/config \
+    --rm \
+    nebulaoss/nebula
+```
+
+A few notes:
+
+- The `NET_ADMIN` capability is necessary to create the tun adapter on the host (this is unnecessary if the tun device is disabled.)
+- `--volume ./config:/config` should point to a directory that contains your `config.yml` and any other necessary files.

examples/config.yml

@@ -167,8 +167,7 @@ punchy:
 
 # Preferred ranges is used to define a hint about the local network ranges, which speeds up discovering the fastest
 # path to a network adjacent nebula node.
-# NOTE: the previous option "local_range" only allowed definition of a single range
+# This setting is reloadable.
-# and has been deprecated for "preferred_ranges"
 #preferred_ranges: ["172.16.0.0/24"]
 
 # sshd can expose informational and administrative functions via ssh. This can expose informational and administrative
@@ -181,12 +180,15 @@ punchy:
   # A file containing the ssh host private key to use
   # A decent way to generate one: ssh-keygen -t ed25519 -f ssh_host_ed25519_key -N "" < /dev/null
   #host_key: ./ssh_host_ed25519_key
-  # A file containing a list of authorized public keys
+  # Authorized users and their public keys
   #authorized_users:
     #- user: steeeeve
       # keys can be an array of strings or single string
       #keys:
         #- "ssh public key string"
+  # Trusted SSH CA public keys. These are the public keys of the CAs that are allowed to sign SSH keys for access.
+  #trusted_cas:
+    #- "ssh public key string"
 
 # EXPERIMENTAL: relay support for networks that can't establish direct connections.
 relay:
@@ -230,6 +232,7 @@ tun:
   # `mtu`: will default to tun mtu if this option is not specified
   # `metric`: will default to 0 if this option is not specified
   # `install`: will default to true, controls whether this route is installed in the systems routing table.
+  # This setting is reloadable.
   unsafe_routes:
     #- route: 172.16.1.0/24
     #  via: 192.168.100.99
@@ -285,7 +288,10 @@ tun:
 # TODO
 # Configure logging level
 logging:
-  # panic, fatal, error, warning, info, or debug. Default is info
+  # panic, fatal, error, warning, info, or debug. Default is info and is reloadable.
+  #NOTE: Debug mode can log remotely controlled/untrusted data which can quickly fill a disk in some
+  # scenarios. Debug logging is also CPU intensive and will decrease performance overall.
+  # Only enable debug logging while actively investigating an issue.
   level: info
   # json or text formats currently available. Default is text
   format: text
@@ -350,6 +356,13 @@ firewall:
   outbound_action: drop
   inbound_action: drop
 
+  # Controls the default value for local_cidr. Default is true, will be deprecated after v1.9 and defaulted to false.
+  # This setting only affects nebula hosts with subnets encoded in their certificate. A nebula host acting as an
+  # unsafe router with `default_local_cidr_any: true` will expose their unsafe routes to every inbound rule regardless
+  # of the actual destination for the packet. Setting this to false requires each inbound rule to contain a `local_cidr`
+  # if the intention is to allow traffic to flow to an unsafe route.
+  #default_local_cidr_any: false
+
   conntrack:
     tcp_timeout: 12m
     udp_timeout: 3m
@@ -357,7 +370,7 @@ firewall:
 
   # The firewall is default deny. There is no way to write a deny rule.
   # Rules are comprised of a protocol, port, and one or more of host, group, or CIDR
-  # Logical evaluation is roughly: port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr)
+  # Logical evaluation is roughly: port AND proto AND (ca_sha OR ca_name) AND (host OR group OR groups OR cidr) AND (local cidr)
   # - port: Takes `0` or `any` as any, a single number `80`, a range `200-901`, or `fragment` to match second and further fragments of fragmented packets (since there is no port available).
   #   code: same as port but makes more sense when talking about ICMP, TODO: this is not currently implemented in a way that works, use `any`
   #   proto: `any`, `tcp`, `udp`, or `icmp`
@@ -366,6 +379,8 @@ firewall:
   #   groups: Same as group but accepts a list of values. Multiple values are AND'd together and a certificate would have to contain all groups to pass
   #   cidr: a remote CIDR, `0.0.0.0/0` is any.
   #   local_cidr: a local CIDR, `0.0.0.0/0` is any. This could be used to filter destinations when using unsafe_routes.
+  #      Default is `any` unless the certificate contains subnets and then the default is the ip issued in the certificate
+  #      if `default_local_cidr_any` is false, otherwise its `any`.
   #   ca_name: An issuing CA name
   #   ca_sha: An issuing CA shasum
 
@@ -387,3 +402,10 @@ firewall:
       groups:
         - laptop
         - home
+
+    # Expose a subnet (unsafe route) to hosts with the group remote_client
+    # This example assume you have a subnet of 192.168.100.1/24 or larger encoded in the certificate
+    - port: 8080
+      proto: tcp
+      group: remote_client
+      local_cidr: 192.168.100.1/24

examples/quickstart-vagrant/README.md

@@ -1,138 +0,0 @@
-# Quickstart Guide
-
-This guide is intended to bring up a vagrant environment with 1 lighthouse and 2 generic hosts running nebula.
-
-## Creating the virtualenv for ansible
-
-Within the `quickstart/` directory, do the following
-
-```
-# make a virtual environment
-virtualenv venv
-
-# get into the virtualenv
-source venv/bin/activate
-
-# install ansible
-pip install -r requirements.yml
-```
-
-## Bringing up the vagrant environment
-
-A plugin that is used for the Vagrant environment is `vagrant-hostmanager`
-
-To install, run
-
-```
-vagrant plugin install vagrant-hostmanager
-```
-
-All hosts within the Vagrantfile are brought up with
-
-`vagrant up` 
-
-Once the boxes are up, go into the `ansible/` directory and deploy the playbook by running
-
-`ansible-playbook playbook.yml -i inventory -u vagrant`
-
-## Testing within the vagrant env
-
-Once the ansible run is done, hop onto a vagrant box 
-
-`vagrant ssh generic1.vagrant`
-
-or specifically
-
-`ssh vagrant@<ip-address-in-vagrant-file` (password for the vagrant user on the boxes is `vagrant`)
-
-Some quick tests once the vagrant boxes are up are to ping from `generic1.vagrant` to `generic2.vagrant` using 
-their respective nebula ip address. 
-
-```
-vagrant@generic1:~$ ping 10.168.91.220
-PING 10.168.91.220 (10.168.91.220) 56(84) bytes of data.
-64 bytes from 10.168.91.220: icmp_seq=1 ttl=64 time=241 ms
-64 bytes from 10.168.91.220: icmp_seq=2 ttl=64 time=0.704 ms
-```
-
-You can further verify that the allowed nebula firewall rules work by ssh'ing from 1 generic box to the other.
-
-`ssh vagrant@<nebula-ip-address>`  (password for the vagrant user on the boxes is `vagrant`)
-
-See `/etc/nebula/config.yml` on a box for firewall rules.
-
-To see full handshakes and hostmaps, change the logging config of `/etc/nebula/config.yml` on the vagrant boxes from 
-info to debug.
-
-You can watch nebula logs by running
-
-```
-sudo journalctl -fu nebula
-```
-
-Refer to the nebula src code directory's README for further instructions on configuring nebula.
-
-## Troubleshooting
-
-### Is nebula up and running?
-
-Run and verify that 
-
-```
-ifconfig
-``` 
-
-shows you an interface with the name `nebula1` being up.
-
-```
-vagrant@generic1:~$ ifconfig nebula1
-nebula1: flags=4305<UP,POINTOPOINT,RUNNING,NOARP,MULTICAST>  mtu 1300
-        inet 10.168.91.210  netmask 255.128.0.0  destination 10.168.91.210
-        inet6 fe80::aeaf:b105:e6dc:936c  prefixlen 64  scopeid 0x20<link>
-        unspec 00-00-00-00-00-00-00-00-00-00-00-00-00-00-00-00  txqueuelen 500  (UNSPEC)
-        RX packets 2  bytes 168 (168.0 B)
-        RX errors 0  dropped 0  overruns 0  frame 0
-        TX packets 11  bytes 600 (600.0 B)
-        TX errors 0  dropped 0 overruns 0  carrier 0  collisions 0
-```
-
-### Connectivity
-
-Are you able to ping other boxes on the private nebula network?
-
-The following are the private nebula ip addresses of the vagrant env 
-
-```
-generic1.vagrant [nebula_ip] 10.168.91.210
-generic2.vagrant [nebula_ip] 10.168.91.220 
-lighthouse1.vagrant [nebula_ip] 10.168.91.230
-```
-
-Try pinging generic1.vagrant to and from any other box using its nebula ip above.
-
-Double check the nebula firewall rules under /etc/nebula/config.yml to make sure that connectivity is allowed for your use-case if on a specific port.
-
-```
-vagrant@lighthouse1:~$ grep -A21 firewall /etc/nebula/config.yml 
-firewall:
-  conntrack:
-    tcp_timeout: 12m
-    udp_timeout: 3m
-    default_timeout: 10m
-
-  inbound:
-    - proto: icmp
-      port: any
-      host: any
-    - proto: any
-      port: 22
-      host: any
-    - proto: any
-      port: 53
-      host: any
-
-  outbound:
-    - proto: any
-      port: any
-      host: any
-```

examples/quickstart-vagrant/Vagrantfile

@@ -1,40 +0,0 @@
-Vagrant.require_version ">= 2.2.6"
-
-nodes = [
-  { :hostname => 'generic1.vagrant', :ip => '172.11.91.210', :box => 'bento/ubuntu-18.04', :ram => '512', :cpus => 1},
-  { :hostname => 'generic2.vagrant', :ip => '172.11.91.220', :box => 'bento/ubuntu-18.04', :ram => '512', :cpus => 1},
-  { :hostname => 'lighthouse1.vagrant', :ip => '172.11.91.230', :box => 'bento/ubuntu-18.04', :ram => '512', :cpus => 1},
-]
-
-Vagrant.configure("2") do |config|
-
-  config.ssh.insert_key = false
-
-  if Vagrant.has_plugin?('vagrant-cachier')
-    config.cache.enable :apt
-  else
-    printf("** Install vagrant-cachier plugin to speedup deploy: `vagrant plugin install vagrant-cachier`.**\n")
-  end
-
-  if Vagrant.has_plugin?('vagrant-hostmanager')
-    config.hostmanager.enabled = true
-    config.hostmanager.manage_host = true
-    config.hostmanager.include_offline = true
-  else
-    config.vagrant.plugins = "vagrant-hostmanager"
-  end
-
- nodes.each do |node|
-    config.vm.define node[:hostname] do |node_config|
-      node_config.vm.box = node[:box]
-      node_config.vm.hostname = node[:hostname]
-      node_config.vm.network :private_network, ip: node[:ip]
-      node_config.vm.provider :virtualbox do |vb|
-        vb.memory = node[:ram]
-        vb.cpus = node[:cpus]
-        vb.customize ["modifyvm", :id, "--natdnshostresolver1", "on"]
-        vb.customize ['guestproperty', 'set', :id, '/VirtualBox/GuestAdd/VBoxService/--timesync-set-threshold', 10000]
-      end
-    end
-  end
-end

examples/quickstart-vagrant/ansible/ansible.cfg

@@ -1,4 +0,0 @@
-[defaults]
-host_key_checking = False
-private_key_file = ~/.vagrant.d/insecure_private_key
-become = yes

examples/quickstart-vagrant/ansible/filter_plugins/to_nebula_ip.py

@@ -1,21 +0,0 @@
-#!/usr/bin/python
-
-
-class FilterModule(object):
-    def filters(self):
-        return {
-            'to_nebula_ip': self.to_nebula_ip,
-            'map_to_nebula_ips': self.map_to_nebula_ips,
-        }
-
-    def to_nebula_ip(self, ip_str):
-        ip_list = list(map(int, ip_str.split(".")))
-        ip_list[0] = 10
-        ip_list[1] = 168
-        ip = '.'.join(map(str, ip_list))
-        return ip
-
-    def map_to_nebula_ips(self, ip_strs):
-        ip_list = [ self.to_nebula_ip(ip_str) for ip_str in ip_strs ]
-        ips = ', '.join(ip_list)
-        return ips

examples/quickstart-vagrant/ansible/inventory

@@ -1,11 +0,0 @@
-[all]
-generic1.vagrant
-generic2.vagrant
-lighthouse1.vagrant
-
-[generic]
-generic1.vagrant
-generic2.vagrant
-
-[lighthouse]
-lighthouse1.vagrant 

examples/quickstart-vagrant/ansible/playbook.yml

@@ -1,23 +0,0 @@
----
-- name: test connection to vagrant boxes
-  hosts: all
-  tasks:
-    - debug: msg=ok
-
-- name: build nebula binaries locally 
-  connection: local
-  hosts: localhost
-  tasks:
-    - command: chdir=../../../ make build/linux-amd64/"{{ item }}"
-      with_items:
-        - nebula
-        - nebula-cert
-  tags:
-    - build-nebula
-   
-- name: install nebula on all vagrant hosts
-  hosts: all
-  become: yes
-  gather_facts: yes
-  roles:
-    - nebula

examples/quickstart-vagrant/ansible/roles/nebula/defaults/main.yml

@@ -1,3 +0,0 @@
----
-# defaults file for nebula
-nebula_config_directory: "/etc/nebula/"

examples/quickstart-vagrant/ansible/roles/nebula/files/systemd.nebula.service

@@ -1,14 +0,0 @@
-[Unit]
-Description=Nebula overlay networking tool
-Wants=basic.target network-online.target nss-lookup.target time-sync.target
-After=basic.target network.target network-online.target
-Before=sshd.service
-
-[Service]
-SyslogIdentifier=nebula
-ExecReload=/bin/kill -HUP $MAINPID
-ExecStart=/usr/local/bin/nebula -config /etc/nebula/config.yml
-Restart=always
-
-[Install]
-WantedBy=multi-user.target

examples/quickstart-vagrant/ansible/roles/nebula/files/vagrant-test-ca.crt

@@ -1,5 +0,0 @@
------BEGIN NEBULA CERTIFICATE-----
-CkAKDm5lYnVsYSB0ZXN0IENBKNXC1NYFMNXIhO0GOiCmVYeZ9tkB4WEnawmkrca+
-hsAg9otUFhpAowZeJ33KVEABEkAORybHQUUyVFbKYzw0JHfVzAQOHA4kwB1yP9IV
-KpiTw9+ADz+wA+R5tn9B+L8+7+Apc+9dem4BQULjA5mRaoYN
------END NEBULA CERTIFICATE-----

examples/quickstart-vagrant/ansible/roles/nebula/files/vagrant-test-ca.key

@@ -1,4 +0,0 @@
------BEGIN NEBULA ED25519 PRIVATE KEY-----
-FEXZKMSmg8CgIODR0ymUeNT3nbnVpMi7nD79UgkCRHWmVYeZ9tkB4WEnawmkrca+
-hsAg9otUFhpAowZeJ33KVA==
------END NEBULA ED25519 PRIVATE KEY-----

examples/quickstart-vagrant/ansible/roles/nebula/handlers/main.yml

@@ -1,5 +0,0 @@
----
-# handlers file for nebula
-
-- name: restart nebula
-  service: name=nebula state=restarted

examples/quickstart-vagrant/ansible/roles/nebula/tasks/main.yml

@@ -1,62 +0,0 @@
----
-# tasks file for nebula
-
-- name: get the vagrant network interface and set fact
-  set_fact:
-    vagrant_ifce: "ansible_{{ ansible_interfaces | difference(['lo',ansible_default_ipv4.alias]) | sort | first }}"
-  tags:
-    - nebula-conf
-  
-- name: install built nebula binary
-  copy: src="../../../../../build/linux-amd64/{{ item }}" dest="/usr/local/bin" mode=0755
-  with_items:
-    - nebula
-    - nebula-cert
-
-- name: create nebula config directory
-  file: path="{{ nebula_config_directory }}" state=directory mode=0755
- 
-- name:  temporarily copy over root.crt and root.key to sign
-  copy: src={{ item }} dest=/opt/{{ item }}
-  with_items:
-    - vagrant-test-ca.key
-    - vagrant-test-ca.crt
-
-- name: remove previously signed host certificate
-  file: dest=/etc/nebula/{{ item }} state=absent
-  with_items:
-    - host.crt
-    - host.key
-
-- name: sign using the root key
-  command: nebula-cert sign -ca-crt /opt/vagrant-test-ca.crt -ca-key /opt/vagrant-test-ca.key -duration 4320h -groups vagrant -ip {{ hostvars[inventory_hostname][vagrant_ifce]['ipv4']['address'] | to_nebula_ip }}/9 -name {{ ansible_hostname }}.nebula -out-crt /etc/nebula/host.crt -out-key /etc/nebula/host.key
-
-- name: remove root.key used to sign
-  file: dest=/opt/{{ item }} state=absent
-  with_items:
-    - vagrant-test-ca.key
-
-- name: write the content of the trusted ca certificate
-  copy: src="vagrant-test-ca.crt" dest="/etc/nebula/vagrant-test-ca.crt"
-  notify: restart nebula
-
-- name: Create config directory
-  file: path="{{ nebula_config_directory }}" owner=root group=root mode=0755 state=directory
-
-- name: nebula config
-  template: src=config.yml.j2 dest="/etc/nebula/config.yml" mode=0644 owner=root group=root
-  notify: restart nebula
-  tags:
-    - nebula-conf
-
-- name: nebula systemd
-  copy: src=systemd.nebula.service dest="/etc/systemd/system/nebula.service" mode=0644 owner=root group=root
-  register: addconf
-  notify: restart nebula
-
-- name: maybe reload systemd
-  shell: systemctl daemon-reload
-  when: addconf.changed
-
-- name: nebula running
-  service: name="nebula" state=started enabled=yes

examples/quickstart-vagrant/ansible/roles/nebula/templates/config.yml.j2

@@ -1,85 +0,0 @@
-pki:
-  ca: /etc/nebula/vagrant-test-ca.crt
-  cert: /etc/nebula/host.crt
-  key: /etc/nebula/host.key
-
-# Port Nebula will be listening on
-listen:
-  host: 0.0.0.0
-  port: 4242
-
-# sshd can expose informational and administrative functions via ssh
-sshd:
-  # Toggles the feature
-  enabled: true
-  # Host and port to listen on
-  listen: 127.0.0.1:2222
-  # A file containing the ssh host private key to use
-  host_key: /etc/ssh/ssh_host_ed25519_key
-  # A file containing a list of authorized public keys
-  authorized_users:
-{% for user in nebula_users %}
-    - user: {{ user.name }}
-      keys:
-{% for key in user.ssh_auth_keys %}
-        - "{{ key }}"
-{% endfor %}
-{% endfor %}
-
-local_range: 10.168.0.0/16
-
-static_host_map:
-# lighthouse
-  {{ hostvars[groups['lighthouse'][0]][vagrant_ifce]['ipv4']['address'] | to_nebula_ip }}: ["{{ hostvars[groups['lighthouse'][0]][vagrant_ifce]['ipv4']['address']}}:4242"]
-
-default_route: "0.0.0.0"
-
-lighthouse:
-{% if 'lighthouse' in group_names %}
-  am_lighthouse: true
-  serve_dns: true
-{% else %}
-  am_lighthouse: false
-{% endif %}
-  interval: 60
-{% if 'generic' in group_names %}
-  hosts:
-    - {{ hostvars[groups['lighthouse'][0]][vagrant_ifce]['ipv4']['address'] | to_nebula_ip }}
-{% endif %}
-  
-# Configure the private interface
-tun:
-  dev: nebula1
-  # Sets MTU of the tun dev.
-  # MTU of the tun must be smaller than the MTU of the eth0 interface
-  mtu: 1300
-
-# TODO
-# Configure logging level
-logging:
-  level: info
-  format: json
-
-firewall:
-  conntrack:
-    tcp_timeout: 12m
-    udp_timeout: 3m
-    default_timeout: 10m
-
-  inbound:
-    - proto: icmp
-      port: any
-      host: any
-    - proto: any
-      port: 22
-      host: any
-{% if "lighthouse" in groups %}
-    - proto: any
-      port: 53
-      host: any
-{% endif %}
-
-  outbound:
-    - proto: any
-      port: any
-      host: any

examples/quickstart-vagrant/ansible/roles/nebula/vars/main.yml

@@ -1,7 +0,0 @@
----
-# vars file for nebula
-
-nebula_users:
-  - name: user1
-    ssh_auth_keys: 
-      - "ed25519 place-your-ssh-public-key-here"

examples/quickstart-vagrant/requirements.yml

@@ -1 +0,0 @@
-ansible

examples/service_scripts/nebula.open-rc

@@ -0,0 +1,35 @@
+#!/sbin/openrc-run
+#
+# nebula service for open-rc systems
+
+extra_commands="checkconfig"
+
+: ${NEBULA_CONFDIR:=${RC_PREFIX%/}/etc/nebula}
+: ${NEBULA_CONFIG:=${NEBULA_CONFDIR}/config.yml}
+: ${NEBULA_BINARY:=${NEBULA_BINARY}${RC_PREFIX%/}/usr/local/sbin/nebula}
+
+command="${NEBULA_BINARY}"
+command_args="${NEBULA_OPTS} -config ${NEBULA_CONFIG}"
+
+supervisor="supervise-daemon"
+
+description="A scalable overlay networking tool with a focus on performance, simplicity and security"
+
+required_dirs="${NEBULA_CONFDIR}"
+required_files="${NEBULA_CONFIG}"
+
+checkconfig() {
+        "${command}" -test ${command_args} || return 1
+}
+
+start_pre() {
+        if [ "${RC_CMD}" != "restart" ] ; then
+                checkconfig || return $?
+        fi
+}
+
+stop_pre() {
+        if [ "${RC_CMD}" = "restart" ] ; then
+                checkconfig || return $?
+        fi
+}

firewall.go

@@ -2,7 +2,6 @@ package nebula
 
 import (
 	"crypto/sha256"
-	"encoding/binary"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -22,17 +21,12 @@ import (
 	"github.com/slackhq/nebula/firewall"
 )
 
-const tcpACK = 0x10
-const tcpFIN = 0x01
-
 type FirewallInterface interface {
 	AddRule(incoming bool, proto uint8, startPort int32, endPort int32, groups []string, host string, ip *net.IPNet, localIp *net.IPNet, caName string, caSha string) error
 }
 
 type conn struct {
 	Expires time.Time // Time when this conntrack entry will expire
-	Sent    time.Time // If tcp rtt tracking is enabled this will be when Seq was last set
-	Seq     uint32    // If tcp rtt tracking is enabled this will be the seq we are looking for an ack
 
 	// record why the original connection passed the firewall, so we can re-validate
 	// after ruleset changes. Note, rulesVersion is a uint16 so that these two
@@ -58,15 +52,16 @@ type Firewall struct {
 	DefaultTimeout time.Duration //linux: 600s
 
 	// Used to ensure we don't emit local packets for ips we don't own
-	localIps *cidr.Tree4[struct{}]
+	localIps     *cidr.Tree4[struct{}]
+	assignedCIDR *net.IPNet
+	hasSubnets   bool
 
 	rules        string
 	rulesVersion uint16
 
-	trackTCPRTT     bool
+	defaultLocalCIDRAny bool
-	metricTCPRTT    metrics.Histogram
+	incomingMetrics     firewallMetrics
-	incomingMetrics firewallMetrics
+	outgoingMetrics     firewallMetrics
-	outgoingMetrics firewallMetrics
 
 	l *logrus.Logger
 }
@@ -84,6 +79,8 @@ type FirewallConntrack struct {
 	TimerWheel *TimerWheel[firewall.Packet]
 }
 
+// FirewallTable is the entry point for a rule, the evaluation order is:
+// Proto AND port AND (CA SHA or CA name) AND local CIDR AND (group OR groups OR name OR remote CIDR)
 type FirewallTable struct {
 	TCP      firewallPort
 	UDP      firewallPort
@@ -107,18 +104,27 @@ type FirewallCA struct {
 }
 
 type FirewallRule struct {
-	// Any makes Hosts, Groups, CIDR and LocalCIDR irrelevant
+	// Any makes Hosts, Groups, and CIDR irrelevant
-	Any       bool
+	Any    *firewallLocalCIDR
-	Hosts     map[string]struct{}
+	Hosts  map[string]*firewallLocalCIDR
-	Groups    [][]string
+	Groups []*firewallGroups
-	CIDR      *cidr.Tree4[struct{}]
+	CIDR   *cidr.Tree4[*firewallLocalCIDR]
-	LocalCIDR *cidr.Tree4[struct{}]
+}
+
+type firewallGroups struct {
+	Groups    []string
+	LocalCIDR *firewallLocalCIDR
 }
 
 // Even though ports are uint16, int32 maps are faster for lookup
 // Plus we can use `-1` for fragment rules
 type firewallPort map[int32]*FirewallCA
 
+type firewallLocalCIDR struct {
+	Any       bool
+	LocalCIDR *cidr.Tree4[struct{}]
+}
+
 // NewFirewall creates a new Firewall object. A TimerWheel is created for you from the provided timeouts.
 func NewFirewall(l *logrus.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.Duration, c *cert.NebulaCertificate) *Firewall {
 	//TODO: error on 0 duration
@@ -139,8 +145,15 @@ func NewFirewall(l *logrus.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.D
 	}
 
 	localIps := cidr.NewTree4[struct{}]()
+	var assignedCIDR *net.IPNet
 	for _, ip := range c.Details.Ips {
-		localIps.AddCIDR(&net.IPNet{IP: ip.IP, Mask: net.IPMask{255, 255, 255, 255}}, struct{}{})
+		ipNet := &net.IPNet{IP: ip.IP, Mask: net.IPMask{255, 255, 255, 255}}
+		localIps.AddCIDR(ipNet, struct{}{})
+
+		if assignedCIDR == nil {
+			// Only grabbing the first one in the cert since any more than that currently has undefined behavior
+			assignedCIDR = ipNet
+		}
 	}
 
 	for _, n := range c.Details.Subnets {
@@ -158,9 +171,10 @@ func NewFirewall(l *logrus.Logger, tcpTimeout, UDPTimeout, defaultTimeout time.D
 		UDPTimeout:     UDPTimeout,
 		DefaultTimeout: defaultTimeout,
 		localIps:       localIps,
+		assignedCIDR:   assignedCIDR,
+		hasSubnets:     len(c.Details.Subnets) > 0,
 		l:              l,
 
-		metricTCPRTT: metrics.GetOrRegisterHistogram("network.tcp.rtt", nil, metrics.NewExpDecaySample(1028, 0.015)),
 		incomingMetrics: firewallMetrics{
 			droppedLocalIP:  metrics.GetOrRegisterCounter("firewall.incoming.dropped.local_ip", nil),
 			droppedRemoteIP: metrics.GetOrRegisterCounter("firewall.incoming.dropped.remote_ip", nil),
@@ -184,6 +198,9 @@ func NewFirewallFromConfig(l *logrus.Logger, nc *cert.NebulaCertificate, c *conf
 		//TODO: max_connections
 	)
 
+	//TODO: Flip to false after v1.9 release
+	fw.defaultLocalCIDRAny = c.GetBool("firewall.default_local_cidr_any", true)
+
 	inboundAction := c.GetString("firewall.inbound_action", "drop")
 	switch inboundAction {
 	case "reject":
@@ -270,7 +287,7 @@ func (f *Firewall) AddRule(incoming bool, proto uint8, startPort int32, endPort
 		return fmt.Errorf("unknown protocol %v", proto)
 	}
 
-	return fp.addRule(startPort, endPort, groups, host, ip, localIp, caName, caSha)
+	return fp.addRule(f, startPort, endPort, groups, host, ip, localIp, caName, caSha)
 }
 
 // GetRuleHash returns a hash representation of all inbound and outbound rules
@@ -396,9 +413,9 @@ var ErrNoMatchingRule = errors.New("no matching rule in firewall table")
 
 // Drop returns an error if the packet should be dropped, explaining why. It
 // returns nil if the packet should not be dropped.
-func (f *Firewall) Drop(packet []byte, fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.NebulaCAPool, localCache firewall.ConntrackCache) error {
+func (f *Firewall) Drop(fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.NebulaCAPool, localCache firewall.ConntrackCache) error {
 	// Check if we spoke to this tuple, if we did then allow this packet
-	if f.inConns(packet, fp, incoming, h, caPool, localCache) {
+	if f.inConns(fp, h, caPool, localCache) {
 		return nil
 	}
 
@@ -436,7 +453,7 @@ func (f *Firewall) Drop(packet []byte, fp firewall.Packet, incoming bool, h *Hos
 	}
 
 	// We always want to conntrack since it is a faster operation
-	f.addConn(packet, fp, incoming)
+	f.addConn(fp, incoming)
 
 	return nil
 }
@@ -465,7 +482,7 @@ func (f *Firewall) EmitStats() {
 	metrics.GetOrRegisterGauge("firewall.rules.hash", nil).Update(int64(f.GetRuleHashFNV()))
 }
 
-func (f *Firewall) inConns(packet []byte, fp firewall.Packet, incoming bool, h *HostInfo, caPool *cert.NebulaCAPool, localCache firewall.ConntrackCache) bool {
+func (f *Firewall) inConns(fp firewall.Packet, h *HostInfo, caPool *cert.NebulaCAPool, localCache firewall.ConntrackCache) bool {
 	if localCache != nil {
 		if _, ok := localCache[fp]; ok {
 			return true
@@ -525,11 +542,6 @@ func (f *Firewall) inConns(packet []byte, fp firewall.Packet, incoming bool, h *
 	switch fp.Protocol {
 	case firewall.ProtoTCP:
 		c.Expires = time.Now().Add(f.TCPTimeout)
-		if incoming {
-			f.checkTCPRTT(c, packet)
-		} else {
-			setTCPRTTTracking(c, packet)
-		}
 	case firewall.ProtoUDP:
 		c.Expires = time.Now().Add(f.UDPTimeout)
 	default:
@@ -545,16 +557,13 @@ func (f *Firewall) inConns(packet []byte, fp firewall.Packet, incoming bool, h *
 	return true
 }
 
-func (f *Firewall) addConn(packet []byte, fp firewall.Packet, incoming bool) {
+func (f *Firewall) addConn(fp firewall.Packet, incoming bool) {
 	var timeout time.Duration
 	c := &conn{}
 
 	switch fp.Protocol {
 	case firewall.ProtoTCP:
 		timeout = f.TCPTimeout
-		if !incoming {
-			setTCPRTTTracking(c, packet)
-		}
 	case firewall.ProtoUDP:
 		timeout = f.UDPTimeout
 	default:
@@ -624,7 +633,7 @@ func (ft *FirewallTable) match(p firewall.Packet, incoming bool, c *cert.NebulaC
 	return false
 }
 
-func (fp firewallPort) addRule(startPort int32, endPort int32, groups []string, host string, ip *net.IPNet, localIp *net.IPNet, caName string, caSha string) error {
+func (fp firewallPort) addRule(f *Firewall, startPort int32, endPort int32, groups []string, host string, ip *net.IPNet, localIp *net.IPNet, caName string, caSha string) error {
 	if startPort > endPort {
 		return fmt.Errorf("start port was lower than end port")
 	}
@@ -637,7 +646,7 @@ func (fp firewallPort) addRule(startPort int32, endPort int32, groups []string,
 			}
 		}
 
-		if err := fp[i].addRule(groups, host, ip, localIp, caName, caSha); err != nil {
+		if err := fp[i].addRule(f, groups, host, ip, localIp, caName, caSha); err != nil {
 			return err
 		}
 	}
@@ -668,13 +677,12 @@ func (fp firewallPort) match(p firewall.Packet, incoming bool, c *cert.NebulaCer
 	return fp[firewall.PortAny].match(p, c, caPool)
 }
 
-func (fc *FirewallCA) addRule(groups []string, host string, ip, localIp *net.IPNet, caName, caSha string) error {
+func (fc *FirewallCA) addRule(f *Firewall, groups []string, host string, ip, localIp *net.IPNet, caName, caSha string) error {
 	fr := func() *FirewallRule {
 		return &FirewallRule{
-			Hosts:     make(map[string]struct{}),
+			Hosts:  make(map[string]*firewallLocalCIDR),
-			Groups:    make([][]string, 0),
+			Groups: make([]*firewallGroups, 0),
-			CIDR:      cidr.NewTree4[struct{}](),
+			CIDR:   cidr.NewTree4[*firewallLocalCIDR](),
-			LocalCIDR: cidr.NewTree4[struct{}](),
 		}
 	}
 
@@ -683,14 +691,14 @@ func (fc *FirewallCA) addRule(groups []string, host string, ip, localIp *net.IPN
 			fc.Any = fr()
 		}
 
-		return fc.Any.addRule(groups, host, ip, localIp)
+		return fc.Any.addRule(f, groups, host, ip, localIp)
 	}
 
 	if caSha != "" {
 		if _, ok := fc.CAShas[caSha]; !ok {
 			fc.CAShas[caSha] = fr()
 		}
-		err := fc.CAShas[caSha].addRule(groups, host, ip, localIp)
+		err := fc.CAShas[caSha].addRule(f, groups, host, ip, localIp)
 		if err != nil {
 			return err
 		}
@@ -700,7 +708,7 @@ func (fc *FirewallCA) addRule(groups []string, host string, ip, localIp *net.IPN
 		if _, ok := fc.CANames[caName]; !ok {
 			fc.CANames[caName] = fr()
 		}
-		err := fc.CANames[caName].addRule(groups, host, ip, localIp)
+		err := fc.CANames[caName].addRule(f, groups, host, ip, localIp)
 		if err != nil {
 			return err
 		}
@@ -732,41 +740,63 @@ func (fc *FirewallCA) match(p firewall.Packet, c *cert.NebulaCertificate, caPool
 	return fc.CANames[s.Details.Name].match(p, c)
 }
 
-func (fr *FirewallRule) addRule(groups []string, host string, ip *net.IPNet, localIp *net.IPNet) error {
+func (fr *FirewallRule) addRule(f *Firewall, groups []string, host string, ip *net.IPNet, localCIDR *net.IPNet) error {
-	if fr.Any {
+	flc := func() *firewallLocalCIDR {
-		return nil
+		return &firewallLocalCIDR{
+			LocalCIDR: cidr.NewTree4[struct{}](),
+		}
 	}
 
-	if fr.isAny(groups, host, ip, localIp) {
+	if fr.isAny(groups, host, ip) {
-		fr.Any = true
+		if fr.Any == nil {
-		// If it's any we need to wipe out any pre-existing rules to save on memory
+			fr.Any = flc()
-		fr.Groups = make([][]string, 0)
-		fr.Hosts = make(map[string]struct{})
-		fr.CIDR = cidr.NewTree4[struct{}]()
-		fr.LocalCIDR = cidr.NewTree4[struct{}]()
-	} else {
-		if len(groups) > 0 {
-			fr.Groups = append(fr.Groups, groups)
 		}
 
-		if host != "" {
+		return fr.Any.addRule(f, localCIDR)
-			fr.Hosts[host] = struct{}{}
+	}
+
+	if len(groups) > 0 {
+		nlc := flc()
+		err := nlc.addRule(f, localCIDR)
+		if err != nil {
+			return err
 		}
 
-		if ip != nil {
+		fr.Groups = append(fr.Groups, &firewallGroups{
-			fr.CIDR.AddCIDR(ip, struct{}{})
+			Groups:    groups,
-		}
+			LocalCIDR: nlc,
+		})
+	}
 
-		if localIp != nil {
+	if host != "" {
-			fr.LocalCIDR.AddCIDR(localIp, struct{}{})
+		nlc := fr.Hosts[host]
+		if nlc == nil {
+			nlc = flc()
 		}
+		err := nlc.addRule(f, localCIDR)
+		if err != nil {
+			return err
+		}
+		fr.Hosts[host] = nlc
+	}
+
+	if ip != nil {
+		_, nlc := fr.CIDR.GetCIDR(ip)
+		if nlc == nil {
+			nlc = flc()
+		}
+		err := nlc.addRule(f, localCIDR)
+		if err != nil {
+			return err
+		}
+		fr.CIDR.AddCIDR(ip, nlc)
 	}
 
 	return nil
 }
 
-func (fr *FirewallRule) isAny(groups []string, host string, ip, localIp *net.IPNet) bool {
+func (fr *FirewallRule) isAny(groups []string, host string, ip *net.IPNet) bool {
-	if len(groups) == 0 && host == "" && ip == nil && localIp == nil {
+	if len(groups) == 0 && host == "" && ip == nil {
 		return true
 	}
 
@@ -784,10 +814,6 @@ func (fr *FirewallRule) isAny(groups []string, host string, ip, localIp *net.IPN
 		return true
 	}
 
-	if localIp != nil && localIp.Contains(net.IPv4(0, 0, 0, 0)) {
-		return true
-	}
-
 	return false
 }
 
@@ -797,7 +823,7 @@ func (fr *FirewallRule) match(p firewall.Packet, c *cert.NebulaCertificate) bool
 	}
 
 	// Shortcut path for if groups, hosts, or cidr contained an `any`
-	if fr.Any {
+	if fr.Any.match(p, c) {
 		return true
 	}
 
@@ -805,7 +831,7 @@ func (fr *FirewallRule) match(p firewall.Packet, c *cert.NebulaCertificate) bool
 	for _, sg := range fr.Groups {
 		found := false
 
-		for _, g := range sg {
+		for _, g := range sg.Groups {
 			if _, ok := c.Details.InvertedGroups[g]; !ok {
 				found = false
 				break
@@ -814,33 +840,51 @@ func (fr *FirewallRule) match(p firewall.Packet, c *cert.NebulaCertificate) bool
 			found = true
 		}
 
-		if found {
+		if found && sg.LocalCIDR.match(p, c) {
 			return true
 		}
 	}
 
 	if fr.Hosts != nil {
-		if _, ok := fr.Hosts[c.Details.Name]; ok {
+		if flc, ok := fr.Hosts[c.Details.Name]; ok {
-			return true
+			if flc.match(p, c) {
+				return true
+			}
 		}
 	}
 
-	if fr.CIDR != nil {
+	return fr.CIDR.EachContains(p.RemoteIP, func(flc *firewallLocalCIDR) bool {
-		ok, _ := fr.CIDR.Contains(p.RemoteIP)
+		return flc.match(p, c)
-		if ok {
+	})
-			return true
+}
+
+func (flc *firewallLocalCIDR) addRule(f *Firewall, localIp *net.IPNet) error {
+	if localIp == nil {
+		if !f.hasSubnets || f.defaultLocalCIDRAny {
+			flc.Any = true
+			return nil
 		}
+
+		localIp = f.assignedCIDR
+	} else if localIp.Contains(net.IPv4(0, 0, 0, 0)) {
+		flc.Any = true
 	}
 
-	if fr.LocalCIDR != nil {
+	flc.LocalCIDR.AddCIDR(localIp, struct{}{})
-		ok, _ := fr.LocalCIDR.Contains(p.LocalIP)
+	return nil
-		if ok {
+}
-			return true
+
-		}
+func (flc *firewallLocalCIDR) match(p firewall.Packet, c *cert.NebulaCertificate) bool {
+	if flc == nil {
+		return false
 	}
 
-	// No host, group, or cidr matched, bye bye
+	if flc.Any {
-	return false
+		return true
+	}
+
+	ok, _ := flc.LocalCIDR.Contains(p.LocalIP)
+	return ok
 }
 
 type rule struct {
@@ -956,42 +1000,3 @@ func parsePort(s string) (startPort, endPort int32, err error) {
 
 	return
 }
-
-// TODO: write tests for these
-func setTCPRTTTracking(c *conn, p []byte) {
-	if c.Seq != 0 {
-		return
-	}
-
-	ihl := int(p[0]&0x0f) << 2
-
-	// Don't track FIN packets
-	if p[ihl+13]&tcpFIN != 0 {
-		return
-	}
-
-	c.Seq = binary.BigEndian.Uint32(p[ihl+4 : ihl+8])
-	c.Sent = time.Now()
-}
-
-func (f *Firewall) checkTCPRTT(c *conn, p []byte) bool {
-	if c.Seq == 0 {
-		return false
-	}
-
-	ihl := int(p[0]&0x0f) << 2
-	if p[ihl+13]&tcpACK == 0 {
-		return false
-	}
-
-	// Deal with wrap around, signed int cuts the ack window in half
-	// 0 is a bad ack, no data acknowledged
-	// positive number is a bad ack, ack is over half the window away
-	if int32(c.Seq-binary.BigEndian.Uint32(p[ihl+8:ihl+12])) >= 0 {
-		return false
-	}
-
-	f.metricTCPRTT.Update(time.Since(c.Sent).Nanoseconds())
-	c.Seq = 0
-	return true
-}

firewall_test.go

@@ -2,14 +2,12 @@ package nebula
 
 import (
 	"bytes"
-	"encoding/binary"
 	"errors"
 	"math"
 	"net"
 	"testing"
 	"time"
 
-	"github.com/rcrowley/go-metrics"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/firewall"
@@ -71,36 +69,32 @@ func TestFirewall_AddRule(t *testing.T) {
 
 	assert.Nil(t, fw.AddRule(true, firewall.ProtoTCP, 1, 1, []string{}, "", nil, nil, "", ""))
 	// An empty rule is any
-	assert.True(t, fw.InRules.TCP[1].Any.Any)
+	assert.True(t, fw.InRules.TCP[1].Any.Any.Any)
 	assert.Empty(t, fw.InRules.TCP[1].Any.Groups)
 	assert.Empty(t, fw.InRules.TCP[1].Any.Hosts)
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
 	assert.Nil(t, fw.AddRule(true, firewall.ProtoUDP, 1, 1, []string{"g1"}, "", nil, nil, "", ""))
-	assert.False(t, fw.InRules.UDP[1].Any.Any)
+	assert.Nil(t, fw.InRules.UDP[1].Any.Any)
-	assert.Contains(t, fw.InRules.UDP[1].Any.Groups[0], "g1")
+	assert.Contains(t, fw.InRules.UDP[1].Any.Groups[0].Groups, "g1")
 	assert.Empty(t, fw.InRules.UDP[1].Any.Hosts)
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
 	assert.Nil(t, fw.AddRule(true, firewall.ProtoICMP, 1, 1, []string{}, "h1", nil, nil, "", ""))
-	assert.False(t, fw.InRules.ICMP[1].Any.Any)
+	assert.Nil(t, fw.InRules.ICMP[1].Any.Any)
 	assert.Empty(t, fw.InRules.ICMP[1].Any.Groups)
 	assert.Contains(t, fw.InRules.ICMP[1].Any.Hosts, "h1")
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
 	assert.Nil(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", ti, nil, "", ""))
-	assert.False(t, fw.OutRules.AnyProto[1].Any.Any)
+	assert.Nil(t, fw.OutRules.AnyProto[1].Any.Any)
-	assert.Empty(t, fw.OutRules.AnyProto[1].Any.Groups)
+	ok, _ := fw.OutRules.AnyProto[1].Any.CIDR.GetCIDR(ti)
-	assert.Empty(t, fw.OutRules.AnyProto[1].Any.Hosts)
-	ok, _ := fw.OutRules.AnyProto[1].Any.CIDR.Match(iputil.Ip2VpnIp(ti.IP))
 	assert.True(t, ok)
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
 	assert.Nil(t, fw.AddRule(false, firewall.ProtoAny, 1, 1, []string{}, "", nil, ti, "", ""))
-	assert.False(t, fw.OutRules.AnyProto[1].Any.Any)
+	assert.NotNil(t, fw.OutRules.AnyProto[1].Any.Any)
-	assert.Empty(t, fw.OutRules.AnyProto[1].Any.Groups)
+	ok, _ = fw.OutRules.AnyProto[1].Any.Any.LocalCIDR.GetCIDR(ti)
-	assert.Empty(t, fw.OutRules.AnyProto[1].Any.Hosts)
-	ok, _ = fw.OutRules.AnyProto[1].Any.LocalCIDR.Match(iputil.Ip2VpnIp(ti.IP))
 	assert.True(t, ok)
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
@@ -111,32 +105,14 @@ func TestFirewall_AddRule(t *testing.T) {
 	assert.Nil(t, fw.AddRule(true, firewall.ProtoUDP, 1, 1, []string{"g1"}, "", nil, nil, "", "ca-sha"))
 	assert.Contains(t, fw.InRules.UDP[1].CAShas, "ca-sha")
 
-	// Set any and clear fields
-	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
-	assert.Nil(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"g1", "g2"}, "h1", ti, ti, "", ""))
-	assert.Equal(t, []string{"g1", "g2"}, fw.OutRules.AnyProto[0].Any.Groups[0])
-	assert.Contains(t, fw.OutRules.AnyProto[0].Any.Hosts, "h1")
-	ok, _ = fw.OutRules.AnyProto[0].Any.CIDR.Match(iputil.Ip2VpnIp(ti.IP))
-	assert.True(t, ok)
-	ok, _ = fw.OutRules.AnyProto[0].Any.LocalCIDR.Match(iputil.Ip2VpnIp(ti.IP))
-	assert.True(t, ok)
-
-	// run twice just to make sure
-	//TODO: these ANY rules should clear the CA firewall portion
-	assert.Nil(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{"any"}, "", nil, nil, "", ""))
-	assert.Nil(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "any", nil, nil, "", ""))
-	assert.True(t, fw.OutRules.AnyProto[0].Any.Any)
-	assert.Empty(t, fw.OutRules.AnyProto[0].Any.Groups)
-	assert.Empty(t, fw.OutRules.AnyProto[0].Any.Hosts)
-
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
 	assert.Nil(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "any", nil, nil, "", ""))
-	assert.True(t, fw.OutRules.AnyProto[0].Any.Any)
+	assert.True(t, fw.OutRules.AnyProto[0].Any.Any.Any)
 
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
 	_, anyIp, _ := net.ParseCIDR("0.0.0.0/0")
 	assert.Nil(t, fw.AddRule(false, firewall.ProtoAny, 0, 0, []string{}, "", anyIp, nil, "", ""))
-	assert.True(t, fw.OutRules.AnyProto[0].Any.Any)
+	assert.True(t, fw.OutRules.AnyProto[0].Any.Any.Any)
 
 	// Test error conditions
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, c)
@@ -185,74 +161,84 @@ func TestFirewall_Drop(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// Drop outbound
-	assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule)
 	// Allow inbound
 	resetConntrack(fw)
-	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil))
+	assert.NoError(t, fw.Drop(p, true, &h, cp, nil))
 	// Allow outbound because conntrack
-	assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp, nil))
+	assert.NoError(t, fw.Drop(p, false, &h, cp, nil))
 
 	// test remote mismatch
 	oldRemote := p.RemoteIP
 	p.RemoteIP = iputil.Ip2VpnIp(net.IPv4(1, 2, 3, 10))
-	assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrInvalidRemoteIP)
+	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrInvalidRemoteIP)
 	p.RemoteIP = oldRemote
 
 	// ensure signer doesn't get in the way of group checks
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", nil, nil, "", "signer-shasum"))
 	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", nil, nil, "", "signer-shasum-bad"))
-	assert.Equal(t, fw.Drop([]byte{}, p, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caSha doesn't drop on match
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", nil, nil, "", "signer-shasum-bad"))
 	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", nil, nil, "", "signer-shasum"))
-	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil))
+	assert.NoError(t, fw.Drop(p, true, &h, cp, nil))
 
 	// ensure ca name doesn't get in the way of group checks
 	cp.CAs["signer-shasum"] = &cert.NebulaCertificate{Details: cert.NebulaCertificateDetails{Name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", nil, nil, "ca-good", ""))
 	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", nil, nil, "ca-good-bad", ""))
-	assert.Equal(t, fw.Drop([]byte{}, p, true, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, true, &h, cp, nil), ErrNoMatchingRule)
 
 	// test caName doesn't drop on match
 	cp.CAs["signer-shasum"] = &cert.NebulaCertificate{Details: cert.NebulaCertificateDetails{Name: "ca-good"}}
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
 	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"nope"}, "", nil, nil, "ca-good-bad", ""))
 	assert.Nil(t, fw.AddRule(true, firewall.ProtoAny, 0, 0, []string{"default-group"}, "", nil, nil, "ca-good", ""))
-	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil))
+	assert.NoError(t, fw.Drop(p, true, &h, cp, nil))
 }
 
 func BenchmarkFirewallTable_match(b *testing.B) {
+	f := &Firewall{}
 	ft := FirewallTable{
 		TCP: firewallPort{},
 	}
 
 	_, n, _ := net.ParseCIDR("172.1.1.1/32")
-	_ = ft.TCP.addRule(10, 10, []string{"good-group"}, "good-host", n, n, "", "")
+	goodLocalCIDRIP := iputil.Ip2VpnIp(n.IP)
-	_ = ft.TCP.addRule(10, 10, []string{"good-group2"}, "good-host", n, n, "", "")
+	_ = ft.TCP.addRule(f, 10, 10, []string{"good-group"}, "good-host", n, nil, "", "")
-	_ = ft.TCP.addRule(10, 10, []string{"good-group3"}, "good-host", n, n, "", "")
+	_ = ft.TCP.addRule(f, 100, 100, []string{"good-group"}, "good-host", nil, n, "", "")
-	_ = ft.TCP.addRule(10, 10, []string{"good-group4"}, "good-host", n, n, "", "")
-	_ = ft.TCP.addRule(10, 10, []string{"good-group, good-group1"}, "good-host", n, n, "", "")
 	cp := cert.NewCAPool()
 
 	b.Run("fail on proto", func(b *testing.B) {
+		// This benchmark is showing us the cost of failing to match the protocol
 		c := &cert.NebulaCertificate{}
 		for n := 0; n < b.N; n++ {
-			ft.match(firewall.Packet{Protocol: firewall.ProtoUDP}, true, c, cp)
+			assert.False(b, ft.match(firewall.Packet{Protocol: firewall.ProtoUDP}, true, c, cp))
 		}
 	})
 
-	b.Run("fail on port", func(b *testing.B) {
+	b.Run("pass proto, fail on port", func(b *testing.B) {
+		// This benchmark is showing us the cost of matching a specific protocol but failing to match the port
 		c := &cert.NebulaCertificate{}
 		for n := 0; n < b.N; n++ {
-			ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 1}, true, c, cp)
+			assert.False(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 1}, true, c, cp))
 		}
 	})
 
-	b.Run("fail all group, name, and cidr", func(b *testing.B) {
+	b.Run("pass proto, port, fail on local CIDR", func(b *testing.B) {
+		c := &cert.NebulaCertificate{}
+		ip, _, _ := net.ParseCIDR("9.254.254.254/32")
+		lip := iputil.Ip2VpnIp(ip)
+		for n := 0; n < b.N; n++ {
+			assert.False(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, LocalIP: lip}, true, c, cp))
+		}
+	})
+
+	b.Run("pass proto, port, any local CIDR, fail all group, name, and cidr", func(b *testing.B) {
 		_, ip, _ := net.ParseCIDR("9.254.254.254/32")
 		c := &cert.NebulaCertificate{
 			Details: cert.NebulaCertificateDetails{
@@ -262,11 +248,25 @@ func BenchmarkFirewallTable_match(b *testing.B) {
 			},
 		}
 		for n := 0; n < b.N; n++ {
-			ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10}, true, c, cp)
+			assert.False(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10}, true, c, cp))
 		}
 	})
 
-	b.Run("pass on group", func(b *testing.B) {
+	b.Run("pass proto, port, specific local CIDR, fail all group, name, and cidr", func(b *testing.B) {
+		_, ip, _ := net.ParseCIDR("9.254.254.254/32")
+		c := &cert.NebulaCertificate{
+			Details: cert.NebulaCertificateDetails{
+				InvertedGroups: map[string]struct{}{"nope": {}},
+				Name:           "nope",
+				Ips:            []*net.IPNet{ip},
+			},
+		}
+		for n := 0; n < b.N; n++ {
+			assert.False(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, LocalIP: goodLocalCIDRIP}, true, c, cp))
+		}
+	})
+
+	b.Run("pass on group on any local cidr", func(b *testing.B) {
 		c := &cert.NebulaCertificate{
 			Details: cert.NebulaCertificateDetails{
 				InvertedGroups: map[string]struct{}{"good-group": {}},
@@ -274,7 +274,19 @@ func BenchmarkFirewallTable_match(b *testing.B) {
 			},
 		}
 		for n := 0; n < b.N; n++ {
-			ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10}, true, c, cp)
+			assert.True(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10}, true, c, cp))
+		}
+	})
+
+	b.Run("pass on group on specific local cidr", func(b *testing.B) {
+		c := &cert.NebulaCertificate{
+			Details: cert.NebulaCertificateDetails{
+				InvertedGroups: map[string]struct{}{"good-group": {}},
+				Name:           "nope",
+			},
+		}
+		for n := 0; n < b.N; n++ {
+			assert.True(b, ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, LocalIP: goodLocalCIDRIP}, true, c, cp))
 		}
 	})
 
@@ -289,60 +301,60 @@ func BenchmarkFirewallTable_match(b *testing.B) {
 			ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10}, true, c, cp)
 		}
 	})
-
+	//
-	b.Run("pass on ip", func(b *testing.B) {
+	//b.Run("pass on ip", func(b *testing.B) {
-		ip := iputil.Ip2VpnIp(net.IPv4(172, 1, 1, 1))
+	//	ip := iputil.Ip2VpnIp(net.IPv4(172, 1, 1, 1))
-		c := &cert.NebulaCertificate{
+	//	c := &cert.NebulaCertificate{
-			Details: cert.NebulaCertificateDetails{
+	//		Details: cert.NebulaCertificateDetails{
-				InvertedGroups: map[string]struct{}{"nope": {}},
+	//			InvertedGroups: map[string]struct{}{"nope": {}},
-				Name:           "good-host",
+	//			Name:           "good-host",
-			},
+	//		},
-		}
+	//	}
-		for n := 0; n < b.N; n++ {
+	//	for n := 0; n < b.N; n++ {
-			ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10, RemoteIP: ip}, true, c, cp)
+	//		ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10, RemoteIP: ip}, true, c, cp)
-		}
+	//	}
-	})
+	//})
-
+	//
-	b.Run("pass on local ip", func(b *testing.B) {
+	//b.Run("pass on local ip", func(b *testing.B) {
-		ip := iputil.Ip2VpnIp(net.IPv4(172, 1, 1, 1))
+	//	ip := iputil.Ip2VpnIp(net.IPv4(172, 1, 1, 1))
-		c := &cert.NebulaCertificate{
+	//	c := &cert.NebulaCertificate{
-			Details: cert.NebulaCertificateDetails{
+	//		Details: cert.NebulaCertificateDetails{
-				InvertedGroups: map[string]struct{}{"nope": {}},
+	//			InvertedGroups: map[string]struct{}{"nope": {}},
-				Name:           "good-host",
+	//			Name:           "good-host",
-			},
+	//		},
-		}
+	//	}
-		for n := 0; n < b.N; n++ {
+	//	for n := 0; n < b.N; n++ {
-			ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10, LocalIP: ip}, true, c, cp)
+	//		ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 10, LocalIP: ip}, true, c, cp)
-		}
+	//	}
-	})
+	//})
-
+	//
-	_ = ft.TCP.addRule(0, 0, []string{"good-group"}, "good-host", n, n, "", "")
+	//_ = ft.TCP.addRule(0, 0, []string{"good-group"}, "good-host", n, n, "", "")
-
+	//
-	b.Run("pass on ip with any port", func(b *testing.B) {
+	//b.Run("pass on ip with any port", func(b *testing.B) {
-		ip := iputil.Ip2VpnIp(net.IPv4(172, 1, 1, 1))
+	//	ip := iputil.Ip2VpnIp(net.IPv4(172, 1, 1, 1))
-		c := &cert.NebulaCertificate{
+	//	c := &cert.NebulaCertificate{
-			Details: cert.NebulaCertificateDetails{
+	//		Details: cert.NebulaCertificateDetails{
-				InvertedGroups: map[string]struct{}{"nope": {}},
+	//			InvertedGroups: map[string]struct{}{"nope": {}},
-				Name:           "good-host",
+	//			Name:           "good-host",
-			},
+	//		},
-		}
+	//	}
-		for n := 0; n < b.N; n++ {
+	//	for n := 0; n < b.N; n++ {
-			ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, RemoteIP: ip}, true, c, cp)
+	//		ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, RemoteIP: ip}, true, c, cp)
-		}
+	//	}
-	})
+	//})
-
+	//
-	b.Run("pass on local ip with any port", func(b *testing.B) {
+	//b.Run("pass on local ip with any port", func(b *testing.B) {
-		ip := iputil.Ip2VpnIp(net.IPv4(172, 1, 1, 1))
+	//	ip := iputil.Ip2VpnIp(net.IPv4(172, 1, 1, 1))
-		c := &cert.NebulaCertificate{
+	//	c := &cert.NebulaCertificate{
-			Details: cert.NebulaCertificateDetails{
+	//		Details: cert.NebulaCertificateDetails{
-				InvertedGroups: map[string]struct{}{"nope": {}},
+	//			InvertedGroups: map[string]struct{}{"nope": {}},
-				Name:           "good-host",
+	//			Name:           "good-host",
-			},
+	//		},
-		}
+	//	}
-		for n := 0; n < b.N; n++ {
+	//	for n := 0; n < b.N; n++ {
-			ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, LocalIP: ip}, true, c, cp)
+	//		ft.match(firewall.Packet{Protocol: firewall.ProtoTCP, LocalPort: 100, LocalIP: ip}, true, c, cp)
-		}
+	//	}
-	})
+	//})
 }
 
 func TestFirewall_Drop2(t *testing.T) {
@@ -398,10 +410,10 @@ func TestFirewall_Drop2(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// h1/c1 lacks the proper groups
-	assert.Error(t, fw.Drop([]byte{}, p, true, &h1, cp, nil), ErrNoMatchingRule)
+	assert.Error(t, fw.Drop(p, true, &h1, cp, nil), ErrNoMatchingRule)
 	// c has the proper groups
 	resetConntrack(fw)
-	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil))
+	assert.NoError(t, fw.Drop(p, true, &h, cp, nil))
 }
 
 func TestFirewall_Drop3(t *testing.T) {
@@ -481,13 +493,13 @@ func TestFirewall_Drop3(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// c1 should pass because host match
-	assert.NoError(t, fw.Drop([]byte{}, p, true, &h1, cp, nil))
+	assert.NoError(t, fw.Drop(p, true, &h1, cp, nil))
 	// c2 should pass because ca sha match
 	resetConntrack(fw)
-	assert.NoError(t, fw.Drop([]byte{}, p, true, &h2, cp, nil))
+	assert.NoError(t, fw.Drop(p, true, &h2, cp, nil))
 	// c3 should fail because no match
 	resetConntrack(fw)
-	assert.Equal(t, fw.Drop([]byte{}, p, true, &h3, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, true, &h3, cp, nil), ErrNoMatchingRule)
 }
 
 func TestFirewall_DropConntrackReload(t *testing.T) {
@@ -531,12 +543,12 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	cp := cert.NewCAPool()
 
 	// Drop outbound
-	assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule)
 	// Allow inbound
 	resetConntrack(fw)
-	assert.NoError(t, fw.Drop([]byte{}, p, true, &h, cp, nil))
+	assert.NoError(t, fw.Drop(p, true, &h, cp, nil))
 	// Allow outbound because conntrack
-	assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp, nil))
+	assert.NoError(t, fw.Drop(p, false, &h, cp, nil))
 
 	oldFw := fw
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
@@ -545,7 +557,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	fw.rulesVersion = oldFw.rulesVersion + 1
 
 	// Allow outbound because conntrack and new rules allow port 10
-	assert.NoError(t, fw.Drop([]byte{}, p, false, &h, cp, nil))
+	assert.NoError(t, fw.Drop(p, false, &h, cp, nil))
 
 	oldFw = fw
 	fw = NewFirewall(l, time.Second, time.Minute, time.Hour, &c)
@@ -554,7 +566,7 @@ func TestFirewall_DropConntrackReload(t *testing.T) {
 	fw.rulesVersion = oldFw.rulesVersion + 1
 
 	// Drop outbound because conntrack doesn't match new ruleset
-	assert.Equal(t, fw.Drop([]byte{}, p, false, &h, cp, nil), ErrNoMatchingRule)
+	assert.Equal(t, fw.Drop(p, false, &h, cp, nil), ErrNoMatchingRule)
 }
 
 func BenchmarkLookup(b *testing.B) {
@@ -816,97 +828,6 @@ func TestAddFirewallRulesFromConfig(t *testing.T) {
 	assert.EqualError(t, AddFirewallRulesFromConfig(l, true, conf, mf), "firewall.inbound rule #0; `test error`")
 }
 
-func TestTCPRTTTracking(t *testing.T) {
-	b := make([]byte, 200)
-
-	// Max ip IHL (60 bytes) and tcp IHL (60 bytes)
-	b[0] = 15
-	b[60+12] = 15 << 4
-	f := Firewall{
-		metricTCPRTT: metrics.GetOrRegisterHistogram("nope", nil, metrics.NewExpDecaySample(1028, 0.015)),
-	}
-
-	// Set SEQ to 1
-	binary.BigEndian.PutUint32(b[60+4:60+8], 1)
-
-	c := &conn{}
-	setTCPRTTTracking(c, b)
-	assert.Equal(t, uint32(1), c.Seq)
-
-	// Bad ack - no ack flag
-	binary.BigEndian.PutUint32(b[60+8:60+12], 80)
-	assert.False(t, f.checkTCPRTT(c, b))
-
-	// Bad ack, number is too low
-	binary.BigEndian.PutUint32(b[60+8:60+12], 0)
-	b[60+13] = uint8(0x10)
-	assert.False(t, f.checkTCPRTT(c, b))
-
-	// Good ack
-	binary.BigEndian.PutUint32(b[60+8:60+12], 80)
-	assert.True(t, f.checkTCPRTT(c, b))
-	assert.Equal(t, uint32(0), c.Seq)
-
-	// Set SEQ to 1
-	binary.BigEndian.PutUint32(b[60+4:60+8], 1)
-	c = &conn{}
-	setTCPRTTTracking(c, b)
-	assert.Equal(t, uint32(1), c.Seq)
-
-	// Good acks
-	binary.BigEndian.PutUint32(b[60+8:60+12], 81)
-	assert.True(t, f.checkTCPRTT(c, b))
-	assert.Equal(t, uint32(0), c.Seq)
-
-	// Set SEQ to max uint32 - 20
-	binary.BigEndian.PutUint32(b[60+4:60+8], ^uint32(0)-20)
-	c = &conn{}
-	setTCPRTTTracking(c, b)
-	assert.Equal(t, ^uint32(0)-20, c.Seq)
-
-	// Good acks
-	binary.BigEndian.PutUint32(b[60+8:60+12], 81)
-	assert.True(t, f.checkTCPRTT(c, b))
-	assert.Equal(t, uint32(0), c.Seq)
-
-	// Set SEQ to max uint32 / 2
-	binary.BigEndian.PutUint32(b[60+4:60+8], ^uint32(0)/2)
-	c = &conn{}
-	setTCPRTTTracking(c, b)
-	assert.Equal(t, ^uint32(0)/2, c.Seq)
-
-	// Below
-	binary.BigEndian.PutUint32(b[60+8:60+12], ^uint32(0)/2-1)
-	assert.False(t, f.checkTCPRTT(c, b))
-	assert.Equal(t, ^uint32(0)/2, c.Seq)
-
-	// Halfway below
-	binary.BigEndian.PutUint32(b[60+8:60+12], uint32(0))
-	assert.False(t, f.checkTCPRTT(c, b))
-	assert.Equal(t, ^uint32(0)/2, c.Seq)
-
-	// Halfway above is ok
-	binary.BigEndian.PutUint32(b[60+8:60+12], ^uint32(0))
-	assert.True(t, f.checkTCPRTT(c, b))
-	assert.Equal(t, uint32(0), c.Seq)
-
-	// Set SEQ to max uint32
-	binary.BigEndian.PutUint32(b[60+4:60+8], ^uint32(0))
-	c = &conn{}
-	setTCPRTTTracking(c, b)
-	assert.Equal(t, ^uint32(0), c.Seq)
-
-	// Halfway + 1 above
-	binary.BigEndian.PutUint32(b[60+8:60+12], ^uint32(0)/2+1)
-	assert.False(t, f.checkTCPRTT(c, b))
-	assert.Equal(t, ^uint32(0), c.Seq)
-
-	// Halfway above
-	binary.BigEndian.PutUint32(b[60+8:60+12], ^uint32(0)/2)
-	assert.True(t, f.checkTCPRTT(c, b))
-	assert.Equal(t, uint32(0), c.Seq)
-}
-
 func TestFirewall_convertRule(t *testing.T) {
 	l := test.NewLogger()
 	ob := &bytes.Buffer{}

go.mod

@@ -1,53 +1,53 @@
 module github.com/slackhq/nebula
 
-go 1.20
+go 1.22.0
+
+toolchain go1.22.2
 
 require (
 	dario.cat/mergo v1.0.0
 	github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
 	github.com/armon/go-radix v1.0.0
 	github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432
-	github.com/flynn/noise v1.0.1
+	github.com/flynn/noise v1.1.0
 	github.com/gogo/protobuf v1.3.2
 	github.com/google/gopacket v1.1.19
 	github.com/kardianos/service v1.2.2
-	github.com/miekg/dns v1.1.56
+	github.com/miekg/dns v1.1.59
 	github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f
-	github.com/prometheus/client_golang v1.17.0
+	github.com/prometheus/client_golang v1.19.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
 	github.com/sirupsen/logrus v1.9.3
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/songgao/water v0.0.0-20200317203138-2b4b6d7c09d8
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
-	github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54
+	github.com/vishvananda/netlink v1.2.1-beta.2
-	golang.org/x/crypto v0.17.0
+	golang.org/x/crypto v0.23.0
-	golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53
+	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
-	golang.org/x/net v0.19.0
+	golang.org/x/net v0.25.0
-	golang.org/x/sync v0.5.0
+	golang.org/x/sync v0.7.0
-	golang.org/x/sys v0.15.0
+	golang.org/x/sys v0.20.0
-	golang.org/x/term v0.15.0
+	golang.org/x/term v0.20.0
 	golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
 	golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
 	golang.zx2c4.com/wireguard/windows v0.5.3
-	google.golang.org/protobuf v1.31.0
+	google.golang.org/protobuf v1.34.1
 	gopkg.in/yaml.v2 v2.4.0
-	gvisor.dev/gvisor v0.0.0-20230504175454-7b0a1988a28f
+	gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
 )
 
 require (
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/btree v1.1.2 // indirect
-	github.com/google/btree v1.0.1 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
-	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/common v0.48.0 // indirect
-	github.com/prometheus/procfs v0.11.1 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/vishvananda/netns v0.0.4 // indirect
-	golang.org/x/mod v0.12.0 // indirect
+	golang.org/x/mod v0.16.0 // indirect
-	golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect
+	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.13.0 // indirect
+	golang.org/x/tools v0.19.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -22,8 +22,8 @@ github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432/go.
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/flynn/noise v1.0.1 h1:vPp/jdQLXC6ppsXSj/pM3W1BIJ5FEHE2TulSJBpb43Y=
+github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg=
-github.com/flynn/noise v1.0.1/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
+github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -44,17 +44,15 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
-github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
 github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
@@ -74,14 +72,13 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
-github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/miekg/dns v1.1.59 h1:C9EXc/UToRwKLhK5wKU/I4QVsBUc8kE6MkHBkeypWZs=
-github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/miekg/dns v1.1.59/go.mod h1:nZpewl5p6IvctfgrckopVx2OlSEHPRO/U4SYkRklrEk=
-github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE=
-github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@@ -99,27 +96,28 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
-github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
+github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
-github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
+github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
+github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
-github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
+github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
-github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
+github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
-github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
+github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
+github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
-github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
+github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
@@ -135,10 +133,10 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54 h1:8mhqcHPqTMhSPoslhGYihEgSfc77+7La1P6kiB6+9So=
+github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs=
-github.com/vishvananda/netlink v1.1.1-0.20211118161826-650dca95af54/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
+github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8=
 github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
@@ -149,16 +147,16 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
-golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
-golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o=
+golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
-golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
+golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc=
+golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic=
-golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -169,8 +167,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
-golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -178,8 +176,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
+golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
-golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -197,23 +195,23 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
-golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
+golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
-golang.org/x/time v0.0.0-20220210224613-90d013bbcef8/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ=
+golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw=
-golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
+golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -232,9 +230,8 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
-google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
-google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -250,5 +247,5 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gvisor.dev/gvisor v0.0.0-20230504175454-7b0a1988a28f h1:8GE2MRjGiFmfpon8dekPI08jEuNMQzSffVHgdupcO4E=
+gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe h1:fre4i6mv4iBuz5lCMOzHD1rH1ljqHWSICFmZRbbgp3g=
-gvisor.dev/gvisor v0.0.0-20230504175454-7b0a1988a28f/go.mod h1:pzr6sy8gDLfVmDAg8OYrlKvGEHw5C3PGTiBXBTCx76Q=
+gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU=

handshake_ix.go

@@ -99,9 +99,14 @@ func ixHandshakeStage1(f *Interface, addr *udp.Addr, via *ViaSender, packet []by
 
 	remoteCert, err := RecombineCertAndValidate(ci.H, hs.Details.Cert, f.pki.GetCAPool())
 	if err != nil {
-		f.l.WithError(err).WithField("udpAddr", addr).
+		e := f.l.WithError(err).WithField("udpAddr", addr).
-			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).WithField("cert", remoteCert).
+			WithField("handshake", m{"stage": 1, "style": "ix_psk0"})
-			Info("Invalid certificate from host")
+
+		if f.l.Level > logrus.DebugLevel {
+			e = e.WithField("cert", remoteCert)
+		}
+
+		e.Info("Invalid certificate from host")
 		return
 	}
 	vpnIp := iputil.Ip2VpnIp(remoteCert.Details.Ips[0].IP)
@@ -439,9 +444,14 @@ func ixHandshakeStage2(f *Interface, addr *udp.Addr, via *ViaSender, hh *Handsha
 
 	remoteCert, err := RecombineCertAndValidate(ci.H, hs.Details.Cert, f.pki.GetCAPool())
 	if err != nil {
-		f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).
+		e := f.l.WithError(err).WithField("vpnIp", hostinfo.vpnIp).WithField("udpAddr", addr).
-			WithField("cert", remoteCert).WithField("handshake", m{"stage": 2, "style": "ix_psk0"}).
+			WithField("handshake", m{"stage": 2, "style": "ix_psk0"})
-			Error("Invalid certificate from host")
+
+		if f.l.Level > logrus.DebugLevel {
+			e = e.WithField("cert", remoteCert)
+		}
+
+		e.Error("Invalid certificate from host")
 
 		// The handshake state machine is complete, if things break now there is no chance to recover. Tear down and start again
 		return true
@@ -473,7 +483,7 @@ func ixHandshakeStage2(f *Interface, addr *udp.Addr, via *ViaSender, hh *Handsha
 			hostinfo.remotes = f.lightHouse.QueryCache(vpnIp)
 
 			f.l.WithField("blockedUdpAddrs", newHH.hostinfo.remotes.CopyBlockedRemotes()).WithField("vpnIp", vpnIp).
-				WithField("remotes", newHH.hostinfo.remotes.CopyAddrs(f.hostMap.preferredRanges)).
+				WithField("remotes", newHH.hostinfo.remotes.CopyAddrs(f.hostMap.GetPreferredRanges())).
 				Info("Blocked addresses for handshakes")
 
 			// Swap the packet store to benefit the original intended recipient

handshake_manager.go

@@ -184,7 +184,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, lighthouseTrigger
 	hostinfo := hh.hostinfo
 	// If we are out of time, clean up
 	if hh.counter >= hm.config.retries {
-		hh.hostinfo.logger(hm.l).WithField("udpAddrs", hh.hostinfo.remotes.CopyAddrs(hm.mainHostMap.preferredRanges)).
+		hh.hostinfo.logger(hm.l).WithField("udpAddrs", hh.hostinfo.remotes.CopyAddrs(hm.mainHostMap.GetPreferredRanges())).
 			WithField("initiatorIndex", hh.hostinfo.localIndexId).
 			WithField("remoteIndex", hh.hostinfo.remoteIndexId).
 			WithField("handshake", m{"stage": 1, "style": "ix_psk0"}).
@@ -214,7 +214,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, lighthouseTrigger
 		hostinfo.remotes = hm.lightHouse.QueryCache(vpnIp)
 	}
 
-	remotes := hostinfo.remotes.CopyAddrs(hm.mainHostMap.preferredRanges)
+	remotes := hostinfo.remotes.CopyAddrs(hm.mainHostMap.GetPreferredRanges())
 	remotesHaveChanged := !udp.AddrSlice(remotes).Equal(hh.lastRemotes)
 
 	// We only care about a lighthouse trigger if we have new remotes to send to.
@@ -239,7 +239,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp iputil.VpnIp, lighthouseTrigger
 	// Send the handshake to all known ips, stage 2 takes care of assigning the hostinfo.remote based on the first to reply
 	var sentTo []*udp.Addr
 	var sentMultiport bool
-	hostinfo.remotes.ForEach(hm.mainHostMap.preferredRanges, func(addr *udp.Addr, _ bool) {
+	hostinfo.remotes.ForEach(hm.mainHostMap.GetPreferredRanges(), func(addr *udp.Addr, _ bool) {
 		hm.messageMetrics.Tx(header.Handshake, header.MessageSubType(hostinfo.HandshakePacket[0][1]), 1)
 		err := hm.outside.WriteTo(hostinfo.HandshakePacket[0], addr)
 		if err != nil {
@@ -388,7 +388,7 @@ func (hm *HandshakeManager) GetOrHandshake(vpnIp iputil.VpnIp, cacheCb func(*Han
 		hm.mainHostMap.RUnlock()
 		// Do not attempt promotion if you are a lighthouse
 		if !hm.lightHouse.amLighthouse {
-			h.TryPromoteBest(hm.mainHostMap.preferredRanges, hm.f)
+			h.TryPromoteBest(hm.mainHostMap.GetPreferredRanges(), hm.f)
 		}
 		return h, true
 	}
@@ -400,13 +400,13 @@ func (hm *HandshakeManager) GetOrHandshake(vpnIp iputil.VpnIp, cacheCb func(*Han
 // StartHandshake will ensure a handshake is currently being attempted for the provided vpn ip
 func (hm *HandshakeManager) StartHandshake(vpnIp iputil.VpnIp, cacheCb func(*HandshakeHostInfo)) *HostInfo {
 	hm.Lock()
-	defer hm.Unlock()
 
 	if hh, ok := hm.vpnIps[vpnIp]; ok {
 		// We are already trying to handshake with this vpn ip
 		if cacheCb != nil {
 			cacheCb(hh)
 		}
+		hm.Unlock()
 		return hh.hostinfo
 	}
 
@@ -447,6 +447,7 @@ func (hm *HandshakeManager) StartHandshake(vpnIp iputil.VpnIp, cacheCb func(*Han
 		}
 	}
 
+	hm.Unlock()
 	hm.lightHouse.QueryServer(vpnIp)
 	return hostinfo
 }
@@ -625,7 +626,7 @@ func (hm *HandshakeManager) queryIndex(index uint32) *HandshakeHostInfo {
 }
 
 func (c *HandshakeManager) GetPreferredRanges() []*net.IPNet {
-	return c.mainHostMap.preferredRanges
+	return c.mainHostMap.GetPreferredRanges()
 }
 
 func (c *HandshakeManager) ForEachVpnIp(f controlEach) {

handshake_manager_test.go

@@ -19,7 +19,9 @@ func Test_NewHandshakeManagerVpnIp(t *testing.T) {
 	_, localrange, _ := net.ParseCIDR("10.1.1.1/24")
 	ip := iputil.Ip2VpnIp(net.ParseIP("172.1.1.2"))
 	preferredRanges := []*net.IPNet{localrange}
-	mainHM := NewHostMap(l, vpncidr, preferredRanges)
+	mainHM := newHostMap(l, vpncidr)
+	mainHM.preferredRanges.Store(&preferredRanges)
+
 	lh := newTestLighthouse()
 
 	cs := &CertState{

hostmap.go

@@ -11,6 +11,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cert"
 	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/header"
 	"github.com/slackhq/nebula/iputil"
 	"github.com/slackhq/nebula/udp"
@@ -57,9 +58,8 @@ type HostMap struct {
 	Relays          map[uint32]*HostInfo // Maps a Relay IDX to a Relay HostInfo object
 	RemoteIndexes   map[uint32]*HostInfo
 	Hosts           map[iputil.VpnIp]*HostInfo
-	preferredRanges []*net.IPNet
+	preferredRanges atomic.Pointer[[]*net.IPNet]
 	vpnCIDR         *net.IPNet
-	metricsEnabled  bool
 	l               *logrus.Logger
 }
 
@@ -260,21 +260,53 @@ type cachedPacketMetrics struct {
 	dropped metrics.Counter
 }
 
-func NewHostMap(l *logrus.Logger, vpnCIDR *net.IPNet, preferredRanges []*net.IPNet) *HostMap {
+func NewHostMapFromConfig(l *logrus.Logger, vpnCIDR *net.IPNet, c *config.C) *HostMap {
-	h := map[iputil.VpnIp]*HostInfo{}
+	hm := newHostMap(l, vpnCIDR)
-	i := map[uint32]*HostInfo{}
+
-	r := map[uint32]*HostInfo{}
+	hm.reload(c, true)
-	relays := map[uint32]*HostInfo{}
+	c.RegisterReloadCallback(func(c *config.C) {
-	m := HostMap{
+		hm.reload(c, false)
-		Indexes:         i,
+	})
-		Relays:          relays,
+
-		RemoteIndexes:   r,
+	l.WithField("network", hm.vpnCIDR.String()).
-		Hosts:           h,
+		WithField("preferredRanges", hm.GetPreferredRanges()).
-		preferredRanges: preferredRanges,
+		Info("Main HostMap created")
-		vpnCIDR:         vpnCIDR,
+
-		l:               l,
+	return hm
+}
+
+func newHostMap(l *logrus.Logger, vpnCIDR *net.IPNet) *HostMap {
+	return &HostMap{
+		Indexes:       map[uint32]*HostInfo{},
+		Relays:        map[uint32]*HostInfo{},
+		RemoteIndexes: map[uint32]*HostInfo{},
+		Hosts:         map[iputil.VpnIp]*HostInfo{},
+		vpnCIDR:       vpnCIDR,
+		l:             l,
+	}
+}
+
+func (hm *HostMap) reload(c *config.C, initial bool) {
+	if initial || c.HasChanged("preferred_ranges") {
+		var preferredRanges []*net.IPNet
+		rawPreferredRanges := c.GetStringSlice("preferred_ranges", []string{})
+
+		for _, rawPreferredRange := range rawPreferredRanges {
+			_, preferredRange, err := net.ParseCIDR(rawPreferredRange)
+
+			if err != nil {
+				hm.l.WithError(err).WithField("range", rawPreferredRanges).Warn("Failed to parse preferred ranges, ignoring")
+				continue
+			}
+
+			preferredRanges = append(preferredRanges, preferredRange)
+		}
+
+		oldRanges := hm.preferredRanges.Swap(&preferredRanges)
+		if !initial {
+			hm.l.WithField("oldPreferredRanges", *oldRanges).WithField("newPreferredRanges", preferredRanges).Info("preferred_ranges changed")
+		}
 	}
-	return &m
 }
 
 // EmitStats reports host, index, and relay counts to the stats collection system
@@ -463,7 +495,7 @@ func (hm *HostMap) queryVpnIp(vpnIp iputil.VpnIp, promoteIfce *Interface) *HostI
 		hm.RUnlock()
 		// Do not attempt promotion if you are a lighthouse
 		if promoteIfce != nil && !promoteIfce.lightHouse.amLighthouse {
-			h.TryPromoteBest(hm.preferredRanges, promoteIfce)
+			h.TryPromoteBest(hm.GetPreferredRanges(), promoteIfce)
 		}
 		return h
 
@@ -510,7 +542,8 @@ func (hm *HostMap) unlockedAddHostInfo(hostinfo *HostInfo, f *Interface) {
 }
 
 func (hm *HostMap) GetPreferredRanges() []*net.IPNet {
-	return hm.preferredRanges
+	//NOTE: if preferredRanges is ever not stored before a load this will fail to dereference a nil pointer
+	return *hm.preferredRanges.Load()
 }
 
 func (hm *HostMap) ForEachVpnIp(f controlEach) {
@@ -602,7 +635,7 @@ func (i *HostInfo) SetRemoteIfPreferred(hm *HostMap, newRemote *udp.Addr) bool {
 	// NOTE: We do this loop here instead of calling `isPreferred` in
 	// remote_list.go so that we only have to loop over preferredRanges once.
 	newIsPreferred := false
-	for _, l := range hm.preferredRanges {
+	for _, l := range hm.GetPreferredRanges() {
 		// return early if we are already on a preferred remote
 		if l.Contains(currentRemote.IP) {
 			return false

hostmap_test.go

@@ -4,19 +4,19 @@ import (
 	"net"
 	"testing"
 
+	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/test"
 	"github.com/stretchr/testify/assert"
 )
 
 func TestHostMap_MakePrimary(t *testing.T) {
 	l := test.NewLogger()
-	hm := NewHostMap(
+	hm := newHostMap(
 		l,
 		&net.IPNet{
 			IP:   net.IP{10, 0, 0, 1},
 			Mask: net.IPMask{255, 255, 255, 0},
 		},
-		[]*net.IPNet{},
 	)
 
 	f := &Interface{}
@@ -91,13 +91,12 @@ func TestHostMap_MakePrimary(t *testing.T) {
 
 func TestHostMap_DeleteHostInfo(t *testing.T) {
 	l := test.NewLogger()
-	hm := NewHostMap(
+	hm := newHostMap(
 		l,
 		&net.IPNet{
 			IP:   net.IP{10, 0, 0, 1},
 			Mask: net.IPMask{255, 255, 255, 0},
 		},
-		[]*net.IPNet{},
 	)
 
 	f := &Interface{}
@@ -205,3 +204,33 @@ func TestHostMap_DeleteHostInfo(t *testing.T) {
 	prim = hm.QueryVpnIp(1)
 	assert.Nil(t, prim)
 }
+
+func TestHostMap_reload(t *testing.T) {
+	l := test.NewLogger()
+	c := config.NewC(l)
+
+	hm := NewHostMapFromConfig(
+		l,
+		&net.IPNet{
+			IP:   net.IP{10, 0, 0, 1},
+			Mask: net.IPMask{255, 255, 255, 0},
+		},
+		c,
+	)
+
+	toS := func(ipn []*net.IPNet) []string {
+		var s []string
+		for _, n := range ipn {
+			s = append(s, n.String())
+		}
+		return s
+	}
+
+	assert.Empty(t, hm.GetPreferredRanges())
+
+	c.ReloadConfigString("preferred_ranges: [1.1.1.0/24, 10.1.1.0/24]")
+	assert.EqualValues(t, []string{"1.1.1.0/24", "10.1.1.0/24"}, toS(hm.GetPreferredRanges()))
+
+	c.ReloadConfigString("preferred_ranges: [1.1.1.1/32]")
+	assert.EqualValues(t, []string{"1.1.1.1/32"}, toS(hm.GetPreferredRanges()))
+}

inside.go

@@ -62,7 +62,7 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
 		return
 	}
 
-	dropReason := f.firewall.Drop(packet, *fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
+	dropReason := f.firewall.Drop(*fwPacket, false, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason == nil {
 		f.sendNoMetrics(header.Message, 0, hostinfo.ConnectionState, hostinfo, nil, packet, nb, out, q, fwPacket)
 
@@ -142,7 +142,7 @@ func (f *Interface) sendMessageNow(t header.MessageType, st header.MessageSubTyp
 	}
 
 	// check if packet is in outbound fw rules
-	dropReason := f.firewall.Drop(p, *fp, false, hostinfo, f.pki.GetCAPool(), nil)
+	dropReason := f.firewall.Drop(*fp, false, hostinfo, f.pki.GetCAPool(), nil)
 	if dropReason != nil {
 		if f.l.Level >= logrus.DebugLevel {
 			f.l.WithField("fwPacket", fp).

main.go

@@ -183,52 +183,7 @@ func Main(c *config.C, configTest bool, buildVersion string, logger *logrus.Logg
 		}
 	}
 
-	// Set up my internal host map
+	hostMap := NewHostMapFromConfig(l, tunCidr, c)
-	var preferredRanges []*net.IPNet
-	rawPreferredRanges := c.GetStringSlice("preferred_ranges", []string{})
-	// First, check if 'preferred_ranges' is set and fallback to 'local_range'
-	if len(rawPreferredRanges) > 0 {
-		for _, rawPreferredRange := range rawPreferredRanges {
-			_, preferredRange, err := net.ParseCIDR(rawPreferredRange)
-			if err != nil {
-				return nil, util.ContextualizeIfNeeded("Failed to parse preferred ranges", err)
-			}
-			preferredRanges = append(preferredRanges, preferredRange)
-		}
-	}
-
-	// local_range was superseded by preferred_ranges. If it is still present,
-	// merge the local_range setting into preferred_ranges. We will probably
-	// deprecate local_range and remove in the future.
-	rawLocalRange := c.GetString("local_range", "")
-	if rawLocalRange != "" {
-		_, localRange, err := net.ParseCIDR(rawLocalRange)
-		if err != nil {
-			return nil, util.ContextualizeIfNeeded("Failed to parse local_range", err)
-		}
-
-		// Check if the entry for local_range was already specified in
-		// preferred_ranges. Don't put it into the slice twice if so.
-		var found bool
-		for _, r := range preferredRanges {
-			if r.String() == localRange.String() {
-				found = true
-				break
-			}
-		}
-		if !found {
-			preferredRanges = append(preferredRanges, localRange)
-		}
-	}
-
-	hostMap := NewHostMap(l, tunCidr, preferredRanges)
-	hostMap.metricsEnabled = c.GetBool("stats.message_metrics", false)
-
-	l.
-		WithField("network", hostMap.vpnCIDR.String()).
-		WithField("preferredRanges", hostMap.preferredRanges).
-		Info("Main HostMap created")
-
 	punchy := NewPunchyFromConfig(l, c)
 	lightHouse, err := NewLightHouseFromConfig(ctx, l, c, tunCidr, udpConns[0], punchy)
 	if err != nil {

noiseutil/nist.go

@@ -48,7 +48,7 @@ func (c nistCurve) DH(privkey, pubkey []byte) ([]byte, error) {
 	}
 	ecdhPrivKey, err := c.curve.NewPrivateKey(privkey)
 	if err != nil {
-		return nil, fmt.Errorf("unable to unmarshal pubkey: %w", err)
+		return nil, fmt.Errorf("unable to unmarshal private key: %w", err)
 	}
 
 	return ecdhPrivKey.ECDH(ecdhPubKey)

outside.go

@@ -417,7 +417,7 @@ func (f *Interface) decryptToTun(hostinfo *HostInfo, messageCounter uint64, out
 		return false
 	}
 
-	dropReason := f.firewall.Drop(out, *fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache)
+	dropReason := f.firewall.Drop(*fwPacket, true, hostinfo, f.pki.GetCAPool(), localCache)
 	if dropReason != nil {
 		// NOTE: We give `packet` as the `out` here since we already decrypted from it and we don't need it anymore
 		// This gives us a buffer to build the reject packet in

overlay/route.go

@@ -1,6 +1,7 @@
 package overlay
 
 import (
+	"bytes"
 	"fmt"
 	"math"
 	"net"
@@ -21,6 +22,35 @@ type Route struct {
 	Install bool
 }
 
+// Equal determines if a route that could be installed in the system route table is equal to another
+// Via is ignored since that is only consumed within nebula itself
+func (r Route) Equal(t Route) bool {
+	if !r.Cidr.IP.Equal(t.Cidr.IP) {
+		return false
+	}
+	if !bytes.Equal(r.Cidr.Mask, t.Cidr.Mask) {
+		return false
+	}
+	if r.Metric != t.Metric {
+		return false
+	}
+	if r.MTU != t.MTU {
+		return false
+	}
+	if r.Install != t.Install {
+		return false
+	}
+	return true
+}
+
+func (r Route) String() string {
+	s := r.Cidr.String()
+	if r.Metric != 0 {
+		s += fmt.Sprintf(" metric: %v", r.Metric)
+	}
+	return s
+}
+
 func makeRouteTree(l *logrus.Logger, routes []Route, allowMTU bool) (*cidr.Tree4[iputil.VpnIp], error) {
 	routeTree := cidr.NewTree4[iputil.VpnIp]()
 	for _, r := range routes {

overlay/tun.go

@@ -10,60 +10,63 @@ import (
 
 const DefaultMTU = 1300
 
+// TODO: We may be able to remove routines
 type DeviceFactory func(c *config.C, l *logrus.Logger, tunCidr *net.IPNet, routines int) (Device, error)
 
 func NewDeviceFromConfig(c *config.C, l *logrus.Logger, tunCidr *net.IPNet, routines int) (Device, error) {
-	routes, err := parseRoutes(c, tunCidr)
-	if err != nil {
-		return nil, util.NewContextualError("Could not parse tun.routes", nil, err)
-	}
-
-	unsafeRoutes, err := parseUnsafeRoutes(c, tunCidr)
-	if err != nil {
-		return nil, util.NewContextualError("Could not parse tun.unsafe_routes", nil, err)
-	}
-	routes = append(routes, unsafeRoutes...)
-
 	switch {
 	case c.GetBool("tun.disabled", false):
 		tun := newDisabledTun(tunCidr, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l)
 		return tun, nil
 
 	default:
-		return newTun(
+		return newTun(c, l, tunCidr, routines > 1)
-			l,
-			c.GetString("tun.dev", ""),
-			tunCidr,
-			c.GetInt("tun.mtu", DefaultMTU),
-			routes,
-			c.GetInt("tun.tx_queue", 500),
-			routines > 1,
-			c.GetBool("tun.use_system_route_table", false),
-		)
 	}
 }
 
 func NewFdDeviceFromConfig(fd *int) DeviceFactory {
 	return func(c *config.C, l *logrus.Logger, tunCidr *net.IPNet, routines int) (Device, error) {
-		routes, err := parseRoutes(c, tunCidr)
+		return newTunFromFd(c, l, *fd, tunCidr)
-		if err != nil {
-			return nil, util.NewContextualError("Could not parse tun.routes", nil, err)
-		}
-
-		unsafeRoutes, err := parseUnsafeRoutes(c, tunCidr)
-		if err != nil {
-			return nil, util.NewContextualError("Could not parse tun.unsafe_routes", nil, err)
-		}
-		routes = append(routes, unsafeRoutes...)
-		return newTunFromFd(
-			l,
-			*fd,
-			tunCidr,
-			c.GetInt("tun.mtu", DefaultMTU),
-			routes,
-			c.GetInt("tun.tx_queue", 500),
-			c.GetBool("tun.use_system_route_table", false),
-		)
-
 	}
 }
+
+func getAllRoutesFromConfig(c *config.C, cidr *net.IPNet, initial bool) (bool, []Route, error) {
+	if !initial && !c.HasChanged("tun.routes") && !c.HasChanged("tun.unsafe_routes") {
+		return false, nil, nil
+	}
+
+	routes, err := parseRoutes(c, cidr)
+	if err != nil {
+		return true, nil, util.NewContextualError("Could not parse tun.routes", nil, err)
+	}
+
+	unsafeRoutes, err := parseUnsafeRoutes(c, cidr)
+	if err != nil {
+		return true, nil, util.NewContextualError("Could not parse tun.unsafe_routes", nil, err)
+	}
+
+	routes = append(routes, unsafeRoutes...)
+	return true, routes, nil
+}
+
+// findRemovedRoutes will return all routes that are not present in the newRoutes list and would affect the system route table.
+// Via is not used to evaluate since it does not affect the system route table.
+func findRemovedRoutes(newRoutes, oldRoutes []Route) []Route {
+	var removed []Route
+	has := func(entry Route) bool {
+		for _, check := range newRoutes {
+			if check.Equal(entry) {
+				return true
+			}
+		}
+		return false
+	}
+
+	for _, oldEntry := range oldRoutes {
+		if !has(oldEntry) {
+			removed = append(removed, oldEntry)
+		}
+	}
+
+	return removed
+}

overlay/tun_android.go

@@ -8,45 +8,57 @@ import (
 	"io"
 	"net"
 	"os"
+	"sync/atomic"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/util"
 )
 
 type tun struct {
 	io.ReadWriteCloser
 	fd        int
 	cidr      *net.IPNet
-	routeTree *cidr.Tree4[iputil.VpnIp]
+	Routes    atomic.Pointer[[]Route]
+	routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]]
 	l         *logrus.Logger
 }
 
-func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, _ int, routes []Route, _ int, _ bool) (*tun, error) {
+func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, cidr *net.IPNet) (*tun, error) {
-	routeTree, err := makeRouteTree(l, routes, false)
-	if err != nil {
-		return nil, err
-	}
-
 	// XXX Android returns an fd in non-blocking mode which is necessary for shutdown to work properly.
 	// Be sure not to call file.Fd() as it will set the fd to blocking mode.
 	file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
 
-	return &tun{
+	t := &tun{
 		ReadWriteCloser: file,
 		fd:              deviceFd,
 		cidr:            cidr,
 		l:               l,
-		routeTree:       routeTree,
+	}
-	}, nil
+
+	err := t.reload(c, true)
+	if err != nil {
+		return nil, err
+	}
+
+	c.RegisterReloadCallback(func(c *config.C) {
+		err := t.reload(c, false)
+		if err != nil {
+			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+		}
+	})
+
+	return t, nil
 }
 
-func newTun(_ *logrus.Logger, _ string, _ *net.IPNet, _ int, _ []Route, _ int, _ bool, _ bool) (*tun, error) {
+func newTun(_ *config.C, _ *logrus.Logger, _ *net.IPNet, _ bool) (*tun, error) {
 	return nil, fmt.Errorf("newTun not supported in Android")
 }
 
 func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp {
-	_, r := t.routeTree.MostSpecificContains(ip)
+	_, r := t.routeTree.Load().MostSpecificContains(ip)
 	return r
 }
 
@@ -54,6 +66,27 @@ func (t tun) Activate() error {
 	return nil
 }
 
+func (t *tun) reload(c *config.C, initial bool) error {
+	change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial)
+	if err != nil {
+		return err
+	}
+
+	if !initial && !change {
+		return nil
+	}
+
+	routeTree, err := makeRouteTree(t.l, routes, false)
+	if err != nil {
+		return err
+	}
+
+	// Teach nebula how to handle the routes
+	t.Routes.Store(&routes)
+	t.routeTree.Store(routeTree)
+	return nil
+}
+
 func (t *tun) Cidr() *net.IPNet {
 	return t.cidr
 }

overlay/tun_darwin.go

@@ -9,12 +9,15 @@ import (
 	"io"
 	"net"
 	"os"
+	"sync/atomic"
 	"syscall"
 	"unsafe"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/util"
 	netroute "golang.org/x/net/route"
 	"golang.org/x/sys/unix"
 )
@@ -24,8 +27,9 @@ type tun struct {
 	Device     string
 	cidr       *net.IPNet
 	DefaultMTU int
-	Routes     []Route
+	Routes     atomic.Pointer[[]Route]
-	routeTree  *cidr.Tree4[iputil.VpnIp]
+	routeTree  atomic.Pointer[cidr.Tree4[iputil.VpnIp]]
+	linkAddr   *netroute.LinkAddr
 	l          *logrus.Logger
 
 	// cache out buffer since we need to prepend 4 bytes for tun metadata
@@ -69,12 +73,8 @@ type ifreqMTU struct {
 	pad  [8]byte
 }
 
-func newTun(l *logrus.Logger, name string, cidr *net.IPNet, defaultMTU int, routes []Route, _ int, _ bool, _ bool) (*tun, error) {
+func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error) {
-	routeTree, err := makeRouteTree(l, routes, false)
+	name := c.GetString("tun.dev", "")
-	if err != nil {
-		return nil, err
-	}
-
 	ifIndex := -1
 	if name != "" && name != "utun" {
 		_, err := fmt.Sscanf(name, "utun%d", &ifIndex)
@@ -142,17 +142,27 @@ func newTun(l *logrus.Logger, name string, cidr *net.IPNet, defaultMTU int, rout
 
 	file := os.NewFile(uintptr(fd), "")
 
-	tun := &tun{
+	t := &tun{
 		ReadWriteCloser: file,
 		Device:          name,
 		cidr:            cidr,
-		DefaultMTU:      defaultMTU,
+		DefaultMTU:      c.GetInt("tun.mtu", DefaultMTU),
-		Routes:          routes,
-		routeTree:       routeTree,
 		l:               l,
 	}
 
-	return tun, nil
+	err = t.reload(c, true)
+	if err != nil {
+		return nil, err
+	}
+
+	c.RegisterReloadCallback(func(c *config.C) {
+		err := t.reload(c, false)
+		if err != nil {
+			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+		}
+	})
+
+	return t, nil
 }
 
 func (t *tun) deviceBytes() (o [16]byte) {
@@ -162,7 +172,7 @@ func (t *tun) deviceBytes() (o [16]byte) {
 	return
 }
 
-func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []Route, _ int, _ bool) (*tun, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (*tun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in Darwin")
 }
 
@@ -260,6 +270,7 @@ func (t *tun) Activate() error {
 	if linkAddr == nil {
 		return fmt.Errorf("unable to discover link_addr for tun interface")
 	}
+	t.linkAddr = linkAddr
 
 	copy(routeAddr.IP[:], addr[:])
 	copy(maskAddr.IP[:], mask[:])
@@ -278,33 +289,48 @@ func (t *tun) Activate() error {
 	}
 
 	// Unsafe path routes
-	for _, r := range t.Routes {
+	return t.addRoutes(false)
-		if r.Via == nil || !r.Install {
+}
-			// We don't allow route MTUs so only install routes with a via
-			continue
-		}
 
-		copy(routeAddr.IP[:], r.Cidr.IP.To4())
+func (t *tun) reload(c *config.C, initial bool) error {
-		copy(maskAddr.IP[:], net.IP(r.Cidr.Mask).To4())
+	change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial)
+	if err != nil {
+		return err
+	}
 
-		err = addRoute(routeSock, routeAddr, maskAddr, linkAddr)
+	if !initial && !change {
+		return nil
+	}
+
+	routeTree, err := makeRouteTree(t.l, routes, false)
+	if err != nil {
+		return err
+	}
+
+	// Teach nebula how to handle the routes before establishing them in the system table
+	oldRoutes := t.Routes.Swap(&routes)
+	t.routeTree.Store(routeTree)
+
+	if !initial {
+		// Remove first, if the system removes a wanted route hopefully it will be re-added next
+		err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
 		if err != nil {
-			if errors.Is(err, unix.EEXIST) {
+			util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
-				t.l.WithField("route", r.Cidr).
-					Warnf("unable to add unsafe_route, identical route already exists")
-			} else {
-				return err
-			}
 		}
 
-		// TODO how to set metric
+		// Ensure any routes we actually want are installed
+		err = t.addRoutes(true)
+		if err != nil {
+			// Catch any stray logs
+			util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
+		}
 	}
 
 	return nil
 }
 
 func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp {
-	ok, r := t.routeTree.MostSpecificContains(ip)
+	ok, r := t.routeTree.Load().MostSpecificContains(ip)
 	if ok {
 		return r
 	}
@@ -340,6 +366,88 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
 	return nil, nil
 }
 
+func (t *tun) addRoutes(logErrors bool) error {
+	routeSock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
+	if err != nil {
+		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
+	}
+
+	defer func() {
+		unix.Shutdown(routeSock, unix.SHUT_RDWR)
+		err := unix.Close(routeSock)
+		if err != nil {
+			t.l.WithError(err).Error("failed to close AF_ROUTE socket")
+		}
+	}()
+
+	routeAddr := &netroute.Inet4Addr{}
+	maskAddr := &netroute.Inet4Addr{}
+	routes := *t.Routes.Load()
+	for _, r := range routes {
+		if r.Via == nil || !r.Install {
+			// We don't allow route MTUs so only install routes with a via
+			continue
+		}
+
+		copy(routeAddr.IP[:], r.Cidr.IP.To4())
+		copy(maskAddr.IP[:], net.IP(r.Cidr.Mask).To4())
+
+		err := addRoute(routeSock, routeAddr, maskAddr, t.linkAddr)
+		if err != nil {
+			if errors.Is(err, unix.EEXIST) {
+				t.l.WithField("route", r.Cidr).
+					Warnf("unable to add unsafe_route, identical route already exists")
+			} else {
+				retErr := util.NewContextualError("Failed to add route", map[string]interface{}{"route": r}, err)
+				if logErrors {
+					retErr.Log(t.l)
+				} else {
+					return retErr
+				}
+			}
+		} else {
+			t.l.WithField("route", r).Info("Added route")
+		}
+	}
+
+	return nil
+}
+
+func (t *tun) removeRoutes(routes []Route) error {
+	routeSock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
+	if err != nil {
+		return fmt.Errorf("unable to create AF_ROUTE socket: %v", err)
+	}
+
+	defer func() {
+		unix.Shutdown(routeSock, unix.SHUT_RDWR)
+		err := unix.Close(routeSock)
+		if err != nil {
+			t.l.WithError(err).Error("failed to close AF_ROUTE socket")
+		}
+	}()
+
+	routeAddr := &netroute.Inet4Addr{}
+	maskAddr := &netroute.Inet4Addr{}
+
+	for _, r := range routes {
+		if !r.Install {
+			continue
+		}
+
+		copy(routeAddr.IP[:], r.Cidr.IP.To4())
+		copy(maskAddr.IP[:], net.IP(r.Cidr.Mask).To4())
+
+		err := delRoute(routeSock, routeAddr, maskAddr, t.linkAddr)
+		if err != nil {
+			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
+		} else {
+			t.l.WithField("route", r).Info("Removed route")
+		}
+	}
+	return nil
+}
+
 func addRoute(sock int, addr, mask *netroute.Inet4Addr, link *netroute.LinkAddr) error {
 	r := netroute.RouteMessage{
 		Version: unix.RTM_VERSION,
@@ -365,6 +473,30 @@ func addRoute(sock int, addr, mask *netroute.Inet4Addr, link *netroute.LinkAddr)
 	return nil
 }
 
+func delRoute(sock int, addr, mask *netroute.Inet4Addr, link *netroute.LinkAddr) error {
+	r := netroute.RouteMessage{
+		Version: unix.RTM_VERSION,
+		Type:    unix.RTM_DELETE,
+		Seq:     1,
+		Addrs: []netroute.Addr{
+			unix.RTAX_DST:     addr,
+			unix.RTAX_GATEWAY: link,
+			unix.RTAX_NETMASK: mask,
+		},
+	}
+
+	data, err := r.Marshal()
+	if err != nil {
+		return fmt.Errorf("failed to create route.RouteMessage: %w", err)
+	}
+	_, err = unix.Write(sock, data[:])
+	if err != nil {
+		return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
+	}
+
+	return nil
+}
+
 func (t *tun) Read(to []byte) (int, error) {
 
 	buf := make([]byte, len(to)+4)

overlay/tun_freebsd.go

@@ -13,12 +13,15 @@ import (
 	"os"
 	"os/exec"
 	"strconv"
+	"sync/atomic"
 	"syscall"
 	"unsafe"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/util"
 )
 
 const (
@@ -47,8 +50,8 @@ type tun struct {
 	Device    string
 	cidr      *net.IPNet
 	MTU       int
-	Routes    []Route
+	Routes    atomic.Pointer[[]Route]
-	routeTree *cidr.Tree4[iputil.VpnIp]
+	routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]]
 	l         *logrus.Logger
 
 	io.ReadWriteCloser
@@ -76,14 +79,15 @@ func (t *tun) Close() error {
 	return nil
 }
 
-func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []Route, _ int, _ bool) (*tun, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (*tun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in FreeBSD")
 }
 
-func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, _ int, _ bool, _ bool) (*tun, error) {
+func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error) {
 	// Try to open existing tun device
 	var file *os.File
 	var err error
+	deviceName := c.GetString("tun.dev", "")
 	if deviceName != "" {
 		file, err = os.OpenFile("/dev/"+deviceName, os.O_RDWR, 0)
 	}
@@ -144,47 +148,85 @@ func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int
 		ioctl(fd, syscall.SIOCSIFNAME, uintptr(unsafe.Pointer(&ifrr)))
 	}
 
-	routeTree, err := makeRouteTree(l, routes, false)
+	t := &tun{
+		ReadWriteCloser: file,
+		Device:          deviceName,
+		cidr:            cidr,
+		MTU:             c.GetInt("tun.mtu", DefaultMTU),
+		l:               l,
+	}
+
+	err = t.reload(c, true)
 	if err != nil {
 		return nil, err
 	}
 
-	return &tun{
+	c.RegisterReloadCallback(func(c *config.C) {
-		ReadWriteCloser: file,
+		err := t.reload(c, false)
-		Device:          deviceName,
+		if err != nil {
-		cidr:            cidr,
+			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
-		MTU:             defaultMTU,
+		}
-		Routes:          routes,
+	})
-		routeTree:       routeTree,
+
-		l:               l,
+	return t, nil
-	}, nil
 }
 
 func (t *tun) Activate() error {
 	var err error
 	// TODO use syscalls instead of exec.Command
-	t.l.Debug("command: ifconfig", t.Device, t.cidr.String(), t.cidr.IP.String())
+	cmd := exec.Command("/sbin/ifconfig", t.Device, t.cidr.String(), t.cidr.IP.String())
-	if err = exec.Command("/sbin/ifconfig", t.Device, t.cidr.String(), t.cidr.IP.String()).Run(); err != nil {
+	t.l.Debug("command: ", cmd.String())
+	if err = cmd.Run(); err != nil {
 		return fmt.Errorf("failed to run 'ifconfig': %s", err)
 	}
-	t.l.Debug("command: route", "-n", "add", "-net", t.cidr.String(), "-interface", t.Device)
+
-	if err = exec.Command("/sbin/route", "-n", "add", "-net", t.cidr.String(), "-interface", t.Device).Run(); err != nil {
+	cmd = exec.Command("/sbin/route", "-n", "add", "-net", t.cidr.String(), "-interface", t.Device)
+	t.l.Debug("command: ", cmd.String())
+	if err = cmd.Run(); err != nil {
 		return fmt.Errorf("failed to run 'route add': %s", err)
 	}
-	t.l.Debug("command: ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU))
+
-	if err = exec.Command("/sbin/ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU)).Run(); err != nil {
+	cmd = exec.Command("/sbin/ifconfig", t.Device, "mtu", strconv.Itoa(t.MTU))
+	t.l.Debug("command: ", cmd.String())
+	if err = cmd.Run(); err != nil {
 		return fmt.Errorf("failed to run 'ifconfig': %s", err)
 	}
+
 	// Unsafe path routes
-	for _, r := range t.Routes {
+	return t.addRoutes(false)
-		if r.Via == nil || !r.Install {
+}
-			// We don't allow route MTUs so only install routes with a via
+
-			continue
+func (t *tun) reload(c *config.C, initial bool) error {
+	change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial)
+	if err != nil {
+		return err
+	}
+
+	if !initial && !change {
+		return nil
+	}
+
+	routeTree, err := makeRouteTree(t.l, routes, false)
+	if err != nil {
+		return err
+	}
+
+	// Teach nebula how to handle the routes before establishing them in the system table
+	oldRoutes := t.Routes.Swap(&routes)
+	t.routeTree.Store(routeTree)
+
+	if !initial {
+		// Remove first, if the system removes a wanted route hopefully it will be re-added next
+		err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
+		if err != nil {
+			util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
 		}
 
-		t.l.Debug("command: route", "-n", "add", "-net", r.Cidr.String(), "-interface", t.Device)
+		// Ensure any routes we actually want are installed
-		if err = exec.Command("/sbin/route", "-n", "add", "-net", r.Cidr.String(), "-interface", t.Device).Run(); err != nil {
+		err = t.addRoutes(true)
-			return fmt.Errorf("failed to run 'route add' for unsafe_route %s: %s", r.Cidr.String(), err)
+		if err != nil {
+			// Catch any stray logs
+			util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
 		}
 	}
 
@@ -192,7 +234,7 @@ func (t *tun) Activate() error {
 }
 
 func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp {
-	_, r := t.routeTree.MostSpecificContains(ip)
+	_, r := t.routeTree.Load().MostSpecificContains(ip)
 	return r
 }
 
@@ -208,6 +250,46 @@ func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for freebsd")
 }
 
+func (t *tun) addRoutes(logErrors bool) error {
+	routes := *t.Routes.Load()
+	for _, r := range routes {
+		if r.Via == nil || !r.Install {
+			// We don't allow route MTUs so only install routes with a via
+			continue
+		}
+
+		cmd := exec.Command("/sbin/route", "-n", "add", "-net", r.Cidr.String(), "-interface", t.Device)
+		t.l.Debug("command: ", cmd.String())
+		if err := cmd.Run(); err != nil {
+			retErr := util.NewContextualError("failed to run 'route add' for unsafe_route", map[string]interface{}{"route": r}, err)
+			if logErrors {
+				retErr.Log(t.l)
+			} else {
+				return retErr
+			}
+		}
+	}
+
+	return nil
+}
+
+func (t *tun) removeRoutes(routes []Route) error {
+	for _, r := range routes {
+		if !r.Install {
+			continue
+		}
+
+		cmd := exec.Command("/sbin/route", "-n", "delete", "-net", r.Cidr.String(), "-interface", t.Device)
+		t.l.Debug("command: ", cmd.String())
+		if err := cmd.Run(); err != nil {
+			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
+		} else {
+			t.l.WithField("route", r).Info("Removed route")
+		}
+	}
+	return nil
+}
+
 func (t *tun) deviceBytes() (o [16]byte) {
 	for i, c := range t.Device {
 		o[i] = byte(c)

overlay/tun_ios.go

@@ -10,43 +10,78 @@ import (
 	"net"
 	"os"
 	"sync"
+	"sync/atomic"
 	"syscall"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/util"
 )
 
 type tun struct {
 	io.ReadWriteCloser
 	cidr      *net.IPNet
-	routeTree *cidr.Tree4[iputil.VpnIp]
+	Routes    atomic.Pointer[[]Route]
+	routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]]
+	l         *logrus.Logger
 }
 
-func newTun(_ *logrus.Logger, _ string, _ *net.IPNet, _ int, _ []Route, _ int, _ bool, _ bool) (*tun, error) {
+func newTun(_ *config.C, _ *logrus.Logger, _ *net.IPNet, _ bool) (*tun, error) {
 	return nil, fmt.Errorf("newTun not supported in iOS")
 }
 
-func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, _ int, routes []Route, _ int, _ bool) (*tun, error) {
+func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, cidr *net.IPNet) (*tun, error) {
-	routeTree, err := makeRouteTree(l, routes, false)
+	file := os.NewFile(uintptr(deviceFd), "/dev/tun")
+	t := &tun{
+		cidr:            cidr,
+		ReadWriteCloser: &tunReadCloser{f: file},
+		l:               l,
+	}
+
+	err := t.reload(c, true)
 	if err != nil {
 		return nil, err
 	}
 
-	file := os.NewFile(uintptr(deviceFd), "/dev/tun")
+	c.RegisterReloadCallback(func(c *config.C) {
-	return &tun{
+		err := t.reload(c, false)
-		cidr:            cidr,
+		if err != nil {
-		ReadWriteCloser: &tunReadCloser{f: file},
+			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
-		routeTree:       routeTree,
+		}
-	}, nil
+	})
+
+	return t, nil
 }
 
 func (t *tun) Activate() error {
 	return nil
 }
 
+func (t *tun) reload(c *config.C, initial bool) error {
+	change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial)
+	if err != nil {
+		return err
+	}
+
+	if !initial && !change {
+		return nil
+	}
+
+	routeTree, err := makeRouteTree(t.l, routes, false)
+	if err != nil {
+		return err
+	}
+
+	// Teach nebula how to handle the routes
+	t.Routes.Store(&routes)
+	t.routeTree.Store(routeTree)
+	return nil
+}
+
 func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp {
-	_, r := t.routeTree.MostSpecificContains(ip)
+	_, r := t.routeTree.Load().MostSpecificContains(ip)
 	return r
 }
 

overlay/tun_linux.go

@@ -15,21 +15,25 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/util"
 	"github.com/vishvananda/netlink"
 	"golang.org/x/sys/unix"
 )
 
 type tun struct {
 	io.ReadWriteCloser
-	fd         int
+	fd          int
-	Device     string
+	Device      string
-	cidr       *net.IPNet
+	cidr        *net.IPNet
-	MaxMTU     int
+	MaxMTU      int
-	DefaultMTU int
+	DefaultMTU  int
-	TXQueueLen int
+	TXQueueLen  int
+	deviceIndex int
+	ioctlFd     uintptr
 
-	Routes          []Route
+	Routes          atomic.Pointer[[]Route]
 	routeTree       atomic.Pointer[cidr.Tree4[iputil.VpnIp]]
 	routeChan       chan struct{}
 	useSystemRoutes bool
@@ -61,33 +65,40 @@ type ifreqQLEN struct {
 	pad   [8]byte
 }
 
-func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, defaultMTU int, routes []Route, txQueueLen int, useSystemRoutes bool) (*tun, error) {
+func newTunFromFd(c *config.C, l *logrus.Logger, deviceFd int, cidr *net.IPNet) (*tun, error) {
-	routeTree, err := makeRouteTree(l, routes, true)
+	file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
+
+	t, err := newTunGeneric(c, l, file, cidr)
 	if err != nil {
 		return nil, err
 	}
 
-	file := os.NewFile(uintptr(deviceFd), "/dev/net/tun")
+	t.Device = "tun0"
 
-	t := &tun{
-		ReadWriteCloser: file,
-		fd:              int(file.Fd()),
-		Device:          "tun0",
-		cidr:            cidr,
-		DefaultMTU:      defaultMTU,
-		TXQueueLen:      txQueueLen,
-		Routes:          routes,
-		useSystemRoutes: useSystemRoutes,
-		l:               l,
-	}
-	t.routeTree.Store(routeTree)
 	return t, nil
 }
 
-func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, txQueueLen int, multiqueue bool, useSystemRoutes bool) (*tun, error) {
+func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, multiqueue bool) (*tun, error) {
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
 	if err != nil {
-		return nil, err
+		// If /dev/net/tun doesn't exist, try to create it (will happen in docker)
+		if os.IsNotExist(err) {
+			err = os.MkdirAll("/dev/net", 0755)
+			if err != nil {
+				return nil, fmt.Errorf("/dev/net/tun doesn't exist, failed to mkdir -p /dev/net: %w", err)
+			}
+			err = unix.Mknod("/dev/net/tun", unix.S_IFCHR|0600, int(unix.Mkdev(10, 200)))
+			if err != nil {
+				return nil, fmt.Errorf("failed to create /dev/net/tun: %w", err)
+			}
+
+			fd, err = unix.Open("/dev/net/tun", os.O_RDWR, 0)
+			if err != nil {
+				return nil, fmt.Errorf("created /dev/net/tun, but still failed: %w", err)
+			}
+		} else {
+			return nil, err
+		}
 	}
 
 	var req ifReq
@@ -95,46 +106,113 @@ func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int
 	if multiqueue {
 		req.Flags |= unix.IFF_MULTI_QUEUE
 	}
-	copy(req.Name[:], deviceName)
+	copy(req.Name[:], c.GetString("tun.dev", ""))
 	if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
 		return nil, err
 	}
 	name := strings.Trim(string(req.Name[:]), "\x00")
 
 	file := os.NewFile(uintptr(fd), "/dev/net/tun")
-
+	t, err := newTunGeneric(c, l, file, cidr)
-	maxMTU := defaultMTU
-	for _, r := range routes {
-		if r.MTU == 0 {
-			r.MTU = defaultMTU
-		}
-
-		if r.MTU > maxMTU {
-			maxMTU = r.MTU
-		}
-	}
-
-	routeTree, err := makeRouteTree(l, routes, true)
 	if err != nil {
 		return nil, err
 	}
 
+	t.Device = name
+
+	return t, nil
+}
+
+func newTunGeneric(c *config.C, l *logrus.Logger, file *os.File, cidr *net.IPNet) (*tun, error) {
 	t := &tun{
 		ReadWriteCloser: file,
 		fd:              int(file.Fd()),
-		Device:          name,
 		cidr:            cidr,
-		MaxMTU:          maxMTU,
+		TXQueueLen:      c.GetInt("tun.tx_queue", 500),
-		DefaultMTU:      defaultMTU,
+		useSystemRoutes: c.GetBool("tun.use_system_route_table", false),
-		TXQueueLen:      txQueueLen,
-		Routes:          routes,
-		useSystemRoutes: useSystemRoutes,
 		l:               l,
 	}
-	t.routeTree.Store(routeTree)
+
+	err := t.reload(c, true)
+	if err != nil {
+		return nil, err
+	}
+
+	c.RegisterReloadCallback(func(c *config.C) {
+		err := t.reload(c, false)
+		if err != nil {
+			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+		}
+	})
+
 	return t, nil
 }
 
+func (t *tun) reload(c *config.C, initial bool) error {
+	routeChange, routes, err := getAllRoutesFromConfig(c, t.cidr, initial)
+	if err != nil {
+		return err
+	}
+
+	if !initial && !routeChange && !c.HasChanged("tun.mtu") {
+		return nil
+	}
+
+	routeTree, err := makeRouteTree(t.l, routes, true)
+	if err != nil {
+		return err
+	}
+
+	oldDefaultMTU := t.DefaultMTU
+	oldMaxMTU := t.MaxMTU
+	newDefaultMTU := c.GetInt("tun.mtu", DefaultMTU)
+	newMaxMTU := newDefaultMTU
+	for i, r := range routes {
+		if r.MTU == 0 {
+			routes[i].MTU = newDefaultMTU
+		}
+
+		if r.MTU > t.MaxMTU {
+			newMaxMTU = r.MTU
+		}
+	}
+
+	t.MaxMTU = newMaxMTU
+	t.DefaultMTU = newDefaultMTU
+
+	// Teach nebula how to handle the routes before establishing them in the system table
+	oldRoutes := t.Routes.Swap(&routes)
+	t.routeTree.Store(routeTree)
+
+	if !initial {
+		if oldMaxMTU != newMaxMTU {
+			t.setMTU()
+			t.l.Infof("Set max MTU to %v was %v", t.MaxMTU, oldMaxMTU)
+		}
+
+		if oldDefaultMTU != newDefaultMTU {
+			err := t.setDefaultRoute()
+			if err != nil {
+				t.l.Warn(err)
+			} else {
+				t.l.Infof("Set default MTU to %v was %v", t.DefaultMTU, oldDefaultMTU)
+			}
+		}
+
+		// Remove first, if the system removes a wanted route hopefully it will be re-added next
+		t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
+
+		// Ensure any routes we actually want are installed
+		err = t.addRoutes(true)
+		if err != nil {
+			// This should never be called since addRoutes should log its own errors in a reload condition
+			util.LogWithContextIfNeeded("Failed to refresh routes", err, t.l)
+		}
+	}
+
+	return nil
+}
+
 func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
 	if err != nil {
@@ -208,7 +286,7 @@ func (t *tun) Activate() error {
 	if err != nil {
 		return err
 	}
-	fd := uintptr(s)
+	t.ioctlFd = uintptr(s)
 
 	ifra := ifreqAddr{
 		Name: devName,
@@ -219,52 +297,76 @@ func (t *tun) Activate() error {
 	}
 
 	// Set the device ip address
-	if err = ioctl(fd, unix.SIOCSIFADDR, uintptr(unsafe.Pointer(&ifra))); err != nil {
+	if err = ioctl(t.ioctlFd, unix.SIOCSIFADDR, uintptr(unsafe.Pointer(&ifra))); err != nil {
 		return fmt.Errorf("failed to set tun address: %s", err)
 	}
 
 	// Set the device network
 	ifra.Addr.Addr = mask
-	if err = ioctl(fd, unix.SIOCSIFNETMASK, uintptr(unsafe.Pointer(&ifra))); err != nil {
+	if err = ioctl(t.ioctlFd, unix.SIOCSIFNETMASK, uintptr(unsafe.Pointer(&ifra))); err != nil {
 		return fmt.Errorf("failed to set tun netmask: %s", err)
 	}
 
 	// Set the device name
 	ifrf := ifReq{Name: devName}
-	if err = ioctl(fd, unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
+	if err = ioctl(t.ioctlFd, unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
 		return fmt.Errorf("failed to set tun device name: %s", err)
 	}
 
-	// Set the MTU on the device
+	// Setup our default MTU
-	ifm := ifreqMTU{Name: devName, MTU: int32(t.MaxMTU)}
+	t.setMTU()
-	if err = ioctl(fd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil {
-		// This is currently a non fatal condition because the route table must have the MTU set appropriately as well
-		t.l.WithError(err).Error("Failed to set tun mtu")
-	}
 
 	// Set the transmit queue length
 	ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)}
-	if err = ioctl(fd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil {
+	if err = ioctl(t.ioctlFd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil {
 		// If we can't set the queue length nebula will still work but it may lead to packet loss
 		t.l.WithError(err).Error("Failed to set tun tx queue length")
 	}
 
 	// Bring up the interface
 	ifrf.Flags = ifrf.Flags | unix.IFF_UP
-	if err = ioctl(fd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
+	if err = ioctl(t.ioctlFd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
 		return fmt.Errorf("failed to bring the tun device up: %s", err)
 	}
 
-	// Set the routes
 	link, err := netlink.LinkByName(t.Device)
 	if err != nil {
 		return fmt.Errorf("failed to get tun device link: %s", err)
 	}
+	t.deviceIndex = link.Attrs().Index
 
+	if err = t.setDefaultRoute(); err != nil {
+		return err
+	}
+
+	// Set the routes
+	if err = t.addRoutes(false); err != nil {
+		return err
+	}
+
+	// Run the interface
+	ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING
+	if err = ioctl(t.ioctlFd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
+		return fmt.Errorf("failed to run tun device: %s", err)
+	}
+
+	return nil
+}
+
+func (t *tun) setMTU() {
+	// Set the MTU on the device
+	ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MaxMTU)}
+	if err := ioctl(t.ioctlFd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil {
+		// This is currently a non fatal condition because the route table must have the MTU set appropriately as well
+		t.l.WithError(err).Error("Failed to set tun mtu")
+	}
+}
+
+func (t *tun) setDefaultRoute() error {
 	// Default route
 	dr := &net.IPNet{IP: t.cidr.IP.Mask(t.cidr.Mask), Mask: t.cidr.Mask}
 	nr := netlink.Route{
-		LinkIndex: link.Attrs().Index,
+		LinkIndex: t.deviceIndex,
 		Dst:       dr,
 		MTU:       t.DefaultMTU,
 		AdvMSS:    t.advMSS(Route{}),
@@ -274,19 +376,24 @@ func (t *tun) Activate() error {
 		Table:     unix.RT_TABLE_MAIN,
 		Type:      unix.RTN_UNICAST,
 	}
-	err = netlink.RouteReplace(&nr)
+	err := netlink.RouteReplace(&nr)
 	if err != nil {
 		return fmt.Errorf("failed to set mtu %v on the default route %v; %v", t.DefaultMTU, dr, err)
 	}
 
+	return nil
+}
+
+func (t *tun) addRoutes(logErrors bool) error {
 	// Path routes
-	for _, r := range t.Routes {
+	routes := *t.Routes.Load()
+	for _, r := range routes {
 		if !r.Install {
 			continue
 		}
 
 		nr := netlink.Route{
-			LinkIndex: link.Attrs().Index,
+			LinkIndex: t.deviceIndex,
 			Dst:       r.Cidr,
 			MTU:       r.MTU,
 			AdvMSS:    t.advMSS(r),
@@ -297,21 +404,49 @@ func (t *tun) Activate() error {
 			nr.Priority = r.Metric
 		}
 
-		err = netlink.RouteAdd(&nr)
+		err := netlink.RouteReplace(&nr)
 		if err != nil {
-			return fmt.Errorf("failed to set mtu %v on route %v; %v", r.MTU, r.Cidr, err)
+			retErr := util.NewContextualError("Failed to add route", map[string]interface{}{"route": r}, err)
+			if logErrors {
+				retErr.Log(t.l)
+			} else {
+				return retErr
+			}
+		} else {
+			t.l.WithField("route", r).Info("Added route")
 		}
 	}
 
-	// Run the interface
-	ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING
-	if err = ioctl(fd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil {
-		return fmt.Errorf("failed to run tun device: %s", err)
-	}
-
 	return nil
 }
 
+func (t *tun) removeRoutes(routes []Route) {
+	for _, r := range routes {
+		if !r.Install {
+			continue
+		}
+
+		nr := netlink.Route{
+			LinkIndex: t.deviceIndex,
+			Dst:       r.Cidr,
+			MTU:       r.MTU,
+			AdvMSS:    t.advMSS(r),
+			Scope:     unix.RT_SCOPE_LINK,
+		}
+
+		if r.Metric > 0 {
+			nr.Priority = r.Metric
+		}
+
+		err := netlink.RouteDel(&nr)
+		if err != nil {
+			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
+		} else {
+			t.l.WithField("route", r).Info("Removed route")
+		}
+	}
+}
+
 func (t *tun) Cidr() *net.IPNet {
 	return t.cidr
 }
@@ -410,5 +545,9 @@ func (t *tun) Close() error {
 		t.ReadWriteCloser.Close()
 	}
 
+	if t.ioctlFd > 0 {
+		os.NewFile(t.ioctlFd, "ioctlFd").Close()
+	}
+
 	return nil
 }

overlay/tun_netbsd.go

@@ -11,12 +11,15 @@ import (
 	"os/exec"
 	"regexp"
 	"strconv"
+	"sync/atomic"
 	"syscall"
 	"unsafe"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/util"
 )
 
 type ifreqDestroy struct {
@@ -28,8 +31,8 @@ type tun struct {
 	Device    string
 	cidr      *net.IPNet
 	MTU       int
-	Routes    []Route
+	Routes    atomic.Pointer[[]Route]
-	routeTree *cidr.Tree4[iputil.VpnIp]
+	routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]]
 	l         *logrus.Logger
 
 	io.ReadWriteCloser
@@ -56,43 +59,50 @@ func (t *tun) Close() error {
 	return nil
 }
 
-func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []Route, _ int, _ bool) (*tun, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (*tun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in NetBSD")
 }
 
 var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
 
-func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, _ int, _ bool, _ bool) (*tun, error) {
+func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error) {
 	// Try to open tun device
 	var file *os.File
 	var err error
+	deviceName := c.GetString("tun.dev", "")
 	if deviceName == "" {
 		return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
 	}
 	if !deviceNameRE.MatchString(deviceName) {
 		return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified")
 	}
+
 	file, err = os.OpenFile("/dev/"+deviceName, os.O_RDWR, 0)
-
 	if err != nil {
 		return nil, err
 	}
 
-	routeTree, err := makeRouteTree(l, routes, false)
+	t := &tun{
-
-	if err != nil {
-		return nil, err
-	}
-
-	return &tun{
 		ReadWriteCloser: file,
 		Device:          deviceName,
 		cidr:            cidr,
-		MTU:             defaultMTU,
+		MTU:             c.GetInt("tun.mtu", DefaultMTU),
-		Routes:          routes,
-		routeTree:       routeTree,
 		l:               l,
-	}, nil
+	}
+
+	err = t.reload(c, true)
+	if err != nil {
+		return nil, err
+	}
+
+	c.RegisterReloadCallback(func(c *config.C) {
+		err := t.reload(c, false)
+		if err != nil {
+			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+		}
+	})
+
+	return t, nil
 }
 
 func (t *tun) Activate() error {
@@ -116,17 +126,42 @@ func (t *tun) Activate() error {
 	if err = cmd.Run(); err != nil {
 		return fmt.Errorf("failed to run 'ifconfig': %s", err)
 	}
+
 	// Unsafe path routes
-	for _, r := range t.Routes {
+	return t.addRoutes(false)
-		if r.Via == nil || !r.Install {
+}
-			// We don't allow route MTUs so only install routes with a via
+
-			continue
+func (t *tun) reload(c *config.C, initial bool) error {
+	change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial)
+	if err != nil {
+		return err
+	}
+
+	if !initial && !change {
+		return nil
+	}
+
+	routeTree, err := makeRouteTree(t.l, routes, false)
+	if err != nil {
+		return err
+	}
+
+	// Teach nebula how to handle the routes before establishing them in the system table
+	oldRoutes := t.Routes.Swap(&routes)
+	t.routeTree.Store(routeTree)
+
+	if !initial {
+		// Remove first, if the system removes a wanted route hopefully it will be re-added next
+		err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
+		if err != nil {
+			util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
 		}
 
-		cmd = exec.Command("/sbin/route", "-n", "add", "-net", r.Cidr.String(), t.cidr.IP.String())
+		// Ensure any routes we actually want are installed
-		t.l.Debug("command: ", cmd.String())
+		err = t.addRoutes(true)
-		if err = cmd.Run(); err != nil {
+		if err != nil {
-			return fmt.Errorf("failed to run 'route add' for unsafe_route %s: %s", r.Cidr.String(), err)
+			// Catch any stray logs
+			util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
 		}
 	}
 
@@ -134,7 +169,7 @@ func (t *tun) Activate() error {
 }
 
 func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp {
-	_, r := t.routeTree.MostSpecificContains(ip)
+	_, r := t.routeTree.Load().MostSpecificContains(ip)
 	return r
 }
 
@@ -150,6 +185,46 @@ func (t *tun) NewMultiQueueReader() (io.ReadWriteCloser, error) {
 	return nil, fmt.Errorf("TODO: multiqueue not implemented for netbsd")
 }
 
+func (t *tun) addRoutes(logErrors bool) error {
+	routes := *t.Routes.Load()
+	for _, r := range routes {
+		if r.Via == nil || !r.Install {
+			// We don't allow route MTUs so only install routes with a via
+			continue
+		}
+
+		cmd := exec.Command("/sbin/route", "-n", "add", "-net", r.Cidr.String(), t.cidr.IP.String())
+		t.l.Debug("command: ", cmd.String())
+		if err := cmd.Run(); err != nil {
+			retErr := util.NewContextualError("failed to run 'route add' for unsafe_route", map[string]interface{}{"route": r}, err)
+			if logErrors {
+				retErr.Log(t.l)
+			} else {
+				return retErr
+			}
+		}
+	}
+
+	return nil
+}
+
+func (t *tun) removeRoutes(routes []Route) error {
+	for _, r := range routes {
+		if !r.Install {
+			continue
+		}
+
+		cmd := exec.Command("/sbin/route", "-n", "delete", "-net", r.Cidr.String(), t.cidr.IP.String())
+		t.l.Debug("command: ", cmd.String())
+		if err := cmd.Run(); err != nil {
+			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
+		} else {
+			t.l.WithField("route", r).Info("Removed route")
+		}
+	}
+	return nil
+}
+
 func (t *tun) deviceBytes() (o [16]byte) {
 	for i, c := range t.Device {
 		o[i] = byte(c)

overlay/tun_openbsd.go

@@ -11,19 +11,22 @@ import (
 	"os/exec"
 	"regexp"
 	"strconv"
+	"sync/atomic"
 	"syscall"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/util"
 )
 
 type tun struct {
 	Device    string
 	cidr      *net.IPNet
 	MTU       int
-	Routes    []Route
+	Routes    atomic.Pointer[[]Route]
-	routeTree *cidr.Tree4[iputil.VpnIp]
+	routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]]
 	l         *logrus.Logger
 
 	io.ReadWriteCloser
@@ -40,13 +43,14 @@ func (t *tun) Close() error {
 	return nil
 }
 
-func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []Route, _ int, _ bool) (*tun, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (*tun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in OpenBSD")
 }
 
 var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`)
 
-func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, _ int, _ bool, _ bool) (*tun, error) {
+func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*tun, error) {
+	deviceName := c.GetString("tun.dev", "")
 	if deviceName == "" {
 		return nil, fmt.Errorf("a device name in the format of tunN must be specified")
 	}
@@ -60,20 +64,64 @@ func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int
 		return nil, err
 	}
 
-	routeTree, err := makeRouteTree(l, routes, false)
+	t := &tun{
+		ReadWriteCloser: file,
+		Device:          deviceName,
+		cidr:            cidr,
+		MTU:             c.GetInt("tun.mtu", DefaultMTU),
+		l:               l,
+	}
+
+	err = t.reload(c, true)
 	if err != nil {
 		return nil, err
 	}
 
-	return &tun{
+	c.RegisterReloadCallback(func(c *config.C) {
-		ReadWriteCloser: file,
+		err := t.reload(c, false)
-		Device:          deviceName,
+		if err != nil {
-		cidr:            cidr,
+			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
-		MTU:             defaultMTU,
+		}
-		Routes:          routes,
+	})
-		routeTree:       routeTree,
+
-		l:               l,
+	return t, nil
-	}, nil
+}
+
+func (t *tun) reload(c *config.C, initial bool) error {
+	change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial)
+	if err != nil {
+		return err
+	}
+
+	if !initial && !change {
+		return nil
+	}
+
+	routeTree, err := makeRouteTree(t.l, routes, false)
+	if err != nil {
+		return err
+	}
+
+	// Teach nebula how to handle the routes before establishing them in the system table
+	oldRoutes := t.Routes.Swap(&routes)
+	t.routeTree.Store(routeTree)
+
+	if !initial {
+		// Remove first, if the system removes a wanted route hopefully it will be re-added next
+		err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
+		if err != nil {
+			util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
+		}
+
+		// Ensure any routes we actually want are installed
+		err = t.addRoutes(true)
+		if err != nil {
+			// Catch any stray logs
+			util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
+		}
+	}
+
+	return nil
 }
 
 func (t *tun) Activate() error {
@@ -98,25 +146,52 @@ func (t *tun) Activate() error {
 	}
 
 	// Unsafe path routes
-	for _, r := range t.Routes {
+	return t.addRoutes(false)
+}
+
+func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp {
+	_, r := t.routeTree.Load().MostSpecificContains(ip)
+	return r
+}
+
+func (t *tun) addRoutes(logErrors bool) error {
+	routes := *t.Routes.Load()
+	for _, r := range routes {
 		if r.Via == nil || !r.Install {
 			// We don't allow route MTUs so only install routes with a via
 			continue
 		}
 
-		cmd = exec.Command("/sbin/route", "-n", "add", "-inet", r.Cidr.String(), t.cidr.IP.String())
+		cmd := exec.Command("/sbin/route", "-n", "add", "-inet", r.Cidr.String(), t.cidr.IP.String())
 		t.l.Debug("command: ", cmd.String())
-		if err = cmd.Run(); err != nil {
+		if err := cmd.Run(); err != nil {
-			return fmt.Errorf("failed to run 'route add' for unsafe_route %s: %s", r.Cidr.String(), err)
+			retErr := util.NewContextualError("failed to run 'route add' for unsafe_route", map[string]interface{}{"route": r}, err)
+			if logErrors {
+				retErr.Log(t.l)
+			} else {
+				return retErr
+			}
 		}
 	}
 
 	return nil
 }
 
-func (t *tun) RouteFor(ip iputil.VpnIp) iputil.VpnIp {
+func (t *tun) removeRoutes(routes []Route) error {
-	_, r := t.routeTree.MostSpecificContains(ip)
+	for _, r := range routes {
-	return r
+		if !r.Install {
+			continue
+		}
+
+		cmd := exec.Command("/sbin/route", "-n", "delete", "-inet", r.Cidr.String(), t.cidr.IP.String())
+		t.l.Debug("command: ", cmd.String())
+		if err := cmd.Run(); err != nil {
+			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
+		} else {
+			t.l.WithField("route", r).Info("Removed route")
+		}
+	}
+	return nil
 }
 
 func (t *tun) Cidr() *net.IPNet {

overlay/tun_tester.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/iputil"
 )
 
@@ -27,14 +28,18 @@ type TestTun struct {
 	TxPackets chan []byte // Packets transmitted outside by nebula
 }
 
-func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, _ int, routes []Route, _ int, _ bool, _ bool) (*TestTun, error) {
+func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*TestTun, error) {
+	_, routes, err := getAllRoutesFromConfig(c, cidr, true)
+	if err != nil {
+		return nil, err
+	}
 	routeTree, err := makeRouteTree(l, routes, false)
 	if err != nil {
 		return nil, err
 	}
 
 	return &TestTun{
-		Device:    deviceName,
+		Device:    c.GetString("tun.dev", ""),
 		cidr:      cidr,
 		Routes:    routes,
 		routeTree: routeTree,
@@ -44,7 +49,7 @@ func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, _ int, routes
 	}, nil
 }
 
-func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []Route, _ int, _ bool) (*TestTun, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (*TestTun, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported")
 }
 

overlay/tun_water_windows.go

@@ -6,10 +6,13 @@ import (
 	"net"
 	"os/exec"
 	"strconv"
+	"sync/atomic"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/util"
 	"github.com/songgao/water"
 )
 
@@ -17,25 +20,34 @@ type waterTun struct {
 	Device    string
 	cidr      *net.IPNet
 	MTU       int
-	Routes    []Route
+	Routes    atomic.Pointer[[]Route]
-	routeTree *cidr.Tree4[iputil.VpnIp]
+	routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]]
-
+	l         *logrus.Logger
+	f         *net.Interface
 	*water.Interface
 }
 
-func newWaterTun(l *logrus.Logger, cidr *net.IPNet, defaultMTU int, routes []Route) (*waterTun, error) {
+func newWaterTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*waterTun, error) {
-	routeTree, err := makeRouteTree(l, routes, false)
+	// NOTE: You cannot set the deviceName under Windows, so you must check tun.Device after calling .Activate()
+	t := &waterTun{
+		cidr: cidr,
+		MTU:  c.GetInt("tun.mtu", DefaultMTU),
+		l:    l,
+	}
+
+	err := t.reload(c, true)
 	if err != nil {
 		return nil, err
 	}
 
-	// NOTE: You cannot set the deviceName under Windows, so you must check tun.Device after calling .Activate()
+	c.RegisterReloadCallback(func(c *config.C) {
-	return &waterTun{
+		err := t.reload(c, false)
-		cidr:      cidr,
+		if err != nil {
-		MTU:       defaultMTU,
+			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
-		Routes:    routes,
+		}
-		routeTree: routeTree,
+	})
-	}, nil
+
+	return t, nil
 }
 
 func (t *waterTun) Activate() error {
@@ -74,30 +86,104 @@ func (t *waterTun) Activate() error {
 		return fmt.Errorf("failed to run 'netsh' to set MTU: %s", err)
 	}
 
-	iface, err := net.InterfaceByName(t.Device)
+	t.f, err = net.InterfaceByName(t.Device)
 	if err != nil {
 		return fmt.Errorf("failed to find interface named %s: %v", t.Device, err)
 	}
 
-	for _, r := range t.Routes {
+	err = t.addRoutes(false)
-		if r.Via == nil || !r.Install {
+	if err != nil {
-			// We don't allow route MTUs so only install routes with a via
+		return err
-			continue
+	}
-		}
 
-		err = exec.Command(
+	return nil
-			"C:\\Windows\\System32\\route.exe", "add", r.Cidr.String(), r.Via.String(), "IF", strconv.Itoa(iface.Index), "METRIC", strconv.Itoa(r.Metric),
+}
-		).Run()
+
+func (t *waterTun) reload(c *config.C, initial bool) error {
+	change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial)
+	if err != nil {
+		return err
+	}
+
+	if !initial && !change {
+		return nil
+	}
+
+	routeTree, err := makeRouteTree(t.l, routes, false)
+	if err != nil {
+		return err
+	}
+
+	// Teach nebula how to handle the routes before establishing them in the system table
+	oldRoutes := t.Routes.Swap(&routes)
+	t.routeTree.Store(routeTree)
+
+	if !initial {
+		// Remove first, if the system removes a wanted route hopefully it will be re-added next
+		t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
+
+		// Ensure any routes we actually want are installed
+		err = t.addRoutes(true)
 		if err != nil {
-			return fmt.Errorf("failed to add the unsafe_route %s: %v", r.Cidr.String(), err)
+			// Catch any stray logs
+			util.LogWithContextIfNeeded("Failed to set routes", err, t.l)
+		} else {
+			for _, r := range findRemovedRoutes(routes, *oldRoutes) {
+				t.l.WithField("route", r).Info("Removed route")
+			}
 		}
 	}
 
 	return nil
 }
 
+func (t *waterTun) addRoutes(logErrors bool) error {
+	// Path routes
+	routes := *t.Routes.Load()
+	for _, r := range routes {
+		if r.Via == nil || !r.Install {
+			// We don't allow route MTUs so only install routes with a via
+			continue
+		}
+
+		err := exec.Command(
+			"C:\\Windows\\System32\\route.exe", "add", r.Cidr.String(), r.Via.String(), "IF", strconv.Itoa(t.f.Index), "METRIC", strconv.Itoa(r.Metric),
+		).Run()
+
+		if err != nil {
+			retErr := util.NewContextualError("Failed to add route", map[string]interface{}{"route": r}, err)
+			if logErrors {
+				retErr.Log(t.l)
+			} else {
+				return retErr
+			}
+		} else {
+			t.l.WithField("route", r).Info("Added route")
+		}
+	}
+
+	return nil
+}
+
+func (t *waterTun) removeRoutes(routes []Route) {
+	for _, r := range routes {
+		if !r.Install {
+			continue
+		}
+
+		err := exec.Command(
+			"C:\\Windows\\System32\\route.exe", "delete", r.Cidr.String(), r.Via.String(), "IF", strconv.Itoa(t.f.Index), "METRIC", strconv.Itoa(r.Metric),
+		).Run()
+		if err != nil {
+			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
+		} else {
+			t.l.WithField("route", r).Info("Removed route")
+		}
+	}
+}
+
 func (t *waterTun) RouteFor(ip iputil.VpnIp) iputil.VpnIp {
-	_, r := t.routeTree.MostSpecificContains(ip)
+	_, r := t.routeTree.Load().MostSpecificContains(ip)
 	return r
 }
 

overlay/tun_windows.go

@@ -12,13 +12,14 @@ import (
 	"syscall"
 
 	"github.com/sirupsen/logrus"
+	"github.com/slackhq/nebula/config"
 )
 
-func newTunFromFd(_ *logrus.Logger, _ int, _ *net.IPNet, _ int, _ []Route, _ int, _ bool) (Device, error) {
+func newTunFromFd(_ *config.C, _ *logrus.Logger, _ int, _ *net.IPNet) (Device, error) {
 	return nil, fmt.Errorf("newTunFromFd not supported in Windows")
 }
 
-func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route, _ int, _ bool, _ bool) (Device, error) {
+func newTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, multiqueue bool) (Device, error) {
 	useWintun := true
 	if err := checkWinTunExists(); err != nil {
 		l.WithError(err).Warn("Check Wintun driver failed, fallback to wintap driver")
@@ -26,14 +27,14 @@ func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int
 	}
 
 	if useWintun {
-		device, err := newWinTun(l, deviceName, cidr, defaultMTU, routes)
+		device, err := newWinTun(c, l, cidr, multiqueue)
 		if err != nil {
 			return nil, fmt.Errorf("create Wintun interface failed, %w", err)
 		}
 		return device, nil
 	}
 
-	device, err := newWaterTun(l, cidr, defaultMTU, routes)
+	device, err := newWaterTun(c, l, cidr, multiqueue)
 	if err != nil {
 		return nil, fmt.Errorf("create wintap driver failed, %w", err)
 	}

overlay/tun_wintun_windows.go

@@ -6,11 +6,14 @@ import (
 	"io"
 	"net"
 	"net/netip"
+	"sync/atomic"
 	"unsafe"
 
 	"github.com/sirupsen/logrus"
 	"github.com/slackhq/nebula/cidr"
+	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/iputil"
+	"github.com/slackhq/nebula/util"
 	"github.com/slackhq/nebula/wintun"
 	"golang.org/x/sys/windows"
 	"golang.zx2c4.com/wireguard/windows/tunnel/winipcfg"
@@ -23,8 +26,9 @@ type winTun struct {
 	cidr      *net.IPNet
 	prefix    netip.Prefix
 	MTU       int
-	Routes    []Route
+	Routes    atomic.Pointer[[]Route]
-	routeTree *cidr.Tree4[iputil.VpnIp]
+	routeTree atomic.Pointer[cidr.Tree4[iputil.VpnIp]]
+	l         *logrus.Logger
 
 	tun *wintun.NativeTun
 }
@@ -48,83 +52,148 @@ func generateGUIDByDeviceName(name string) (*windows.GUID, error) {
 	return (*windows.GUID)(unsafe.Pointer(&sum[0])), nil
 }
 
-func newWinTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []Route) (*winTun, error) {
+func newWinTun(c *config.C, l *logrus.Logger, cidr *net.IPNet, _ bool) (*winTun, error) {
+	deviceName := c.GetString("tun.dev", "")
 	guid, err := generateGUIDByDeviceName(deviceName)
 	if err != nil {
 		return nil, fmt.Errorf("generate GUID failed: %w", err)
 	}
 
-	var tunDevice wintun.Device
-	tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, defaultMTU)
-	if err != nil {
-		// Windows 10 has an issue with unclean shutdowns not fully cleaning up the wintun device.
-		// Trying a second time resolves the issue.
-		l.WithError(err).Debug("Failed to create wintun device, retrying")
-		tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, defaultMTU)
-		if err != nil {
-			return nil, fmt.Errorf("create TUN device failed: %w", err)
-		}
-	}
-
-	routeTree, err := makeRouteTree(l, routes, false)
-	if err != nil {
-		return nil, err
-	}
-
 	prefix, err := iputil.ToNetIpPrefix(*cidr)
 	if err != nil {
 		return nil, err
 	}
 
-	return &winTun{
+	t := &winTun{
-		Device:    deviceName,
+		Device: deviceName,
-		cidr:      cidr,
+		cidr:   cidr,
-		prefix:    prefix,
+		prefix: prefix,
-		MTU:       defaultMTU,
+		MTU:    c.GetInt("tun.mtu", DefaultMTU),
-		Routes:    routes,
+		l:      l,
-		routeTree: routeTree,
+	}
 
-		tun: tunDevice.(*wintun.NativeTun),
+	err = t.reload(c, true)
-	}, nil
+	if err != nil {
+		return nil, err
+	}
+
+	var tunDevice wintun.Device
+	tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU)
+	if err != nil {
+		// Windows 10 has an issue with unclean shutdowns not fully cleaning up the wintun device.
+		// Trying a second time resolves the issue.
+		l.WithError(err).Debug("Failed to create wintun device, retrying")
+		tunDevice, err = wintun.CreateTUNWithRequestedGUID(deviceName, guid, t.MTU)
+		if err != nil {
+			return nil, fmt.Errorf("create TUN device failed: %w", err)
+		}
+	}
+	t.tun = tunDevice.(*wintun.NativeTun)
+
+	c.RegisterReloadCallback(func(c *config.C) {
+		err := t.reload(c, false)
+		if err != nil {
+			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+		}
+	})
+
+	return t, nil
+}
+
+func (t *winTun) reload(c *config.C, initial bool) error {
+	change, routes, err := getAllRoutesFromConfig(c, t.cidr, initial)
+	if err != nil {
+		return err
+	}
+
+	if !initial && !change {
+		return nil
+	}
+
+	routeTree, err := makeRouteTree(t.l, routes, false)
+	if err != nil {
+		return err
+	}
+
+	// Teach nebula how to handle the routes before establishing them in the system table
+	oldRoutes := t.Routes.Swap(&routes)
+	t.routeTree.Store(routeTree)
+
+	if !initial {
+		// Remove first, if the system removes a wanted route hopefully it will be re-added next
+		err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes))
+		if err != nil {
+			util.LogWithContextIfNeeded("Failed to remove routes", err, t.l)
+		}
+
+		// Ensure any routes we actually want are installed
+		err = t.addRoutes(true)
+		if err != nil {
+			// Catch any stray logs
+			util.LogWithContextIfNeeded("Failed to add routes", err, t.l)
+		}
+	}
+
+	return nil
 }
 
 func (t *winTun) Activate() error {
 	luid := winipcfg.LUID(t.tun.LUID())
 
-	if err := luid.SetIPAddresses([]netip.Prefix{t.prefix}); err != nil {
+	err := luid.SetIPAddresses([]netip.Prefix{t.prefix})
+	if err != nil {
 		return fmt.Errorf("failed to set address: %w", err)
 	}
 
-	foundDefault4 := false
+	err = t.addRoutes(false)
-	routes := make([]*winipcfg.RouteData, 0, len(t.Routes)+1)
+	if err != nil {
+		return err
+	}
 
-	for _, r := range t.Routes {
+	return nil
+}
+
+func (t *winTun) addRoutes(logErrors bool) error {
+	luid := winipcfg.LUID(t.tun.LUID())
+	routes := *t.Routes.Load()
+	foundDefault4 := false
+
+	for _, r := range routes {
 		if r.Via == nil || !r.Install {
 			// We don't allow route MTUs so only install routes with a via
 			continue
 		}
 
+		prefix, err := iputil.ToNetIpPrefix(*r.Cidr)
+		if err != nil {
+			retErr := util.NewContextualError("Failed to parse cidr to netip prefix, ignoring route", map[string]interface{}{"route": r}, err)
+			if logErrors {
+				retErr.Log(t.l)
+				continue
+			} else {
+				return retErr
+			}
+		}
+
+		// Add our unsafe route
+		err = luid.AddRoute(prefix, r.Via.ToNetIpAddr(), uint32(r.Metric))
+		if err != nil {
+			retErr := util.NewContextualError("Failed to add route", map[string]interface{}{"route": r}, err)
+			if logErrors {
+				retErr.Log(t.l)
+				continue
+			} else {
+				return retErr
+			}
+		} else {
+			t.l.WithField("route", r).Info("Added route")
+		}
+
 		if !foundDefault4 {
 			if ones, bits := r.Cidr.Mask.Size(); ones == 0 && bits != 0 {
 				foundDefault4 = true
 			}
 		}
-
-		prefix, err := iputil.ToNetIpPrefix(*r.Cidr)
-		if err != nil {
-			return err
-		}
-
-		// Add our unsafe route
-		routes = append(routes, &winipcfg.RouteData{
-			Destination: prefix,
-			NextHop:     r.Via.ToNetIpAddr(),
-			Metric:      uint32(r.Metric),
-		})
-	}
-
-	if err := luid.AddRoutes(routes); err != nil {
-		return fmt.Errorf("failed to add routes: %w", err)
 	}
 
 	ipif, err := luid.IPInterface(windows.AF_INET)
@@ -141,12 +210,35 @@ func (t *winTun) Activate() error {
 	if err := ipif.Set(); err != nil {
 		return fmt.Errorf("failed to set ip interface: %w", err)
 	}
+	return nil
+}
 
+func (t *winTun) removeRoutes(routes []Route) error {
+	luid := winipcfg.LUID(t.tun.LUID())
+
+	for _, r := range routes {
+		if !r.Install {
+			continue
+		}
+
+		prefix, err := iputil.ToNetIpPrefix(*r.Cidr)
+		if err != nil {
+			t.l.WithError(err).WithField("route", r).Info("Failed to convert cidr to netip prefix")
+			continue
+		}
+
+		err = luid.DeleteRoute(prefix, r.Via.ToNetIpAddr())
+		if err != nil {
+			t.l.WithError(err).WithField("route", r).Error("Failed to remove route")
+		} else {
+			t.l.WithField("route", r).Info("Removed route")
+		}
+	}
 	return nil
 }
 
 func (t *winTun) RouteFor(ip iputil.VpnIp) iputil.VpnIp {
-	_, r := t.routeTree.MostSpecificContains(ip)
+	_, r := t.routeTree.Load().MostSpecificContains(ip)
 	return r
 }
 

service/service.go

@@ -17,7 +17,7 @@ import (
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/overlay"
 	"golang.org/x/sync/errgroup"
-	"gvisor.dev/gvisor/pkg/bufferv2"
+	"gvisor.dev/gvisor/pkg/buffer"
 	"gvisor.dev/gvisor/pkg/tcpip"
 	"gvisor.dev/gvisor/pkg/tcpip/adapters/gonet"
 	"gvisor.dev/gvisor/pkg/tcpip/header"
@@ -81,7 +81,7 @@ func New(config *config.C) (*Service, error) {
 	if tcpipProblem := s.ipstack.CreateNIC(nicID, linkEP); tcpipProblem != nil {
 		return nil, fmt.Errorf("could not create netstack NIC: %v", tcpipProblem)
 	}
-	ipv4Subnet, _ := tcpip.NewSubnet(tcpip.Address(strings.Repeat("\x00", 4)), tcpip.AddressMask(strings.Repeat("\x00", 4)))
+	ipv4Subnet, _ := tcpip.NewSubnet(tcpip.AddrFrom4([4]byte{0x00, 0x00, 0x00, 0x00}), tcpip.MaskFrom(strings.Repeat("\x00", 4)))
 	s.ipstack.SetRouteTable([]tcpip.Route{
 		{
 			Destination: ipv4Subnet,
@@ -91,7 +91,7 @@ func New(config *config.C) (*Service, error) {
 
 	ipNet := device.Cidr()
 	pa := tcpip.ProtocolAddress{
-		AddressWithPrefix: tcpip.Address(ipNet.IP).WithPrefix(),
+		AddressWithPrefix: tcpip.AddrFromSlice(ipNet.IP).WithPrefix(),
 		Protocol:          ipv4.ProtocolNumber,
 	}
 	if err := s.ipstack.AddProtocolAddress(nicID, pa, stack.AddressProperties{
@@ -124,7 +124,7 @@ func New(config *config.C) (*Service, error) {
 				return err
 			}
 			packetBuf := stack.NewPacketBuffer(stack.PacketBufferOptions{
-				Payload: bufferv2.MakeWithData(bytes.Clone(buf[:n])),
+				Payload: buffer.MakeWithData(bytes.Clone(buf[:n])),
 			})
 			linkEP.InjectInbound(header.IPv4ProtocolNumber, packetBuf)
 
@@ -136,7 +136,7 @@ func New(config *config.C) (*Service, error) {
 	eg.Go(func() error {
 		for {
 			packet := linkEP.ReadContext(ctx)
-			if packet.IsNil() {
+			if packet == nil {
 				if err := ctx.Err(); err != nil {
 					return err
 				}
@@ -166,7 +166,7 @@ func (s *Service) DialContext(ctx context.Context, network, address string) (net
 
 	fullAddr := tcpip.FullAddress{
 		NIC:  nicID,
-		Addr: tcpip.Address(addr.IP),
+		Addr: tcpip.AddrFromSlice(addr.IP),
 		Port: uint16(addr.Port),
 	}
 

ssh.go

@@ -51,6 +51,11 @@ type sshCreateTunnelFlags struct {
 	Address string
 }
 
+type sshDeviceInfoFlags struct {
+	Json   bool
+	Pretty bool
+}
+
 func wireSSHReload(l *logrus.Logger, ssh *sshd.SSHServer, c *config.C) {
 	c.RegisterReloadCallback(func(c *config.C) {
 		if c.GetBool("sshd.enabled", false) {
@@ -90,14 +95,19 @@ func configSSH(l *logrus.Logger, ssh *sshd.SSHServer, c *config.C) (func(), erro
 	}
 
 	//TODO: no good way to reload this right now
-	hostKeyFile := c.GetString("sshd.host_key", "")
+	hostKeyPathOrKey := c.GetString("sshd.host_key", "")
-	if hostKeyFile == "" {
+	if hostKeyPathOrKey == "" {
 		return nil, fmt.Errorf("sshd.host_key must be provided")
 	}
 
-	hostKeyBytes, err := os.ReadFile(hostKeyFile)
+	var hostKeyBytes []byte
-	if err != nil {
+	if strings.Contains(hostKeyPathOrKey, "-----BEGIN") {
-		return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err)
+		hostKeyBytes = []byte(hostKeyPathOrKey)
+	} else {
+		hostKeyBytes, err = os.ReadFile(hostKeyPathOrKey)
+		if err != nil {
+			return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err)
+		}
 	}
 
 	err = ssh.SetHostKey(hostKeyBytes)
@@ -105,6 +115,19 @@ func configSSH(l *logrus.Logger, ssh *sshd.SSHServer, c *config.C) (func(), erro
 		return nil, fmt.Errorf("error while adding sshd.host_key: %s", err)
 	}
 
+	// Clear existing trusted CAs and authorized keys
+	ssh.ClearTrustedCAs()
+	ssh.ClearAuthorizedKeys()
+
+	rawCAs := c.GetStringSlice("sshd.trusted_cas", []string{})
+	for _, caAuthorizedKey := range rawCAs {
+		err := ssh.AddTrustedCA(caAuthorizedKey)
+		if err != nil {
+			l.WithError(err).WithField("sshCA", caAuthorizedKey).Warn("SSH CA had an error, ignoring")
+			continue
+		}
+	}
+
 	rawKeys := c.Get("sshd.authorized_users")
 	keys, ok := rawKeys.([]interface{})
 	if ok {
@@ -226,7 +249,7 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter
 
 	ssh.RegisterCommand(&sshd.Command{
 		Name:             "start-cpu-profile",
-		ShortDescription: "Starts a cpu profile and write output to the provided file",
+		ShortDescription: "Starts a cpu profile and write output to the provided file, ex: `cpu-profile.pb.gz`",
 		Callback:         sshStartCpuProfile,
 	})
 
@@ -241,7 +264,7 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter
 
 	ssh.RegisterCommand(&sshd.Command{
 		Name:             "save-heap-profile",
-		ShortDescription: "Saves a heap profile to the provided path",
+		ShortDescription: "Saves a heap profile to the provided path, ex: `heap-profile.pb.gz`",
 		Callback:         sshGetHeapProfile,
 	})
 
@@ -253,7 +276,7 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter
 
 	ssh.RegisterCommand(&sshd.Command{
 		Name:             "save-mutex-profile",
-		ShortDescription: "Saves a mutex profile to the provided path",
+		ShortDescription: "Saves a mutex profile to the provided path, ex: `mutex-profile.pb.gz`",
 		Callback:         sshGetMutexProfile,
 	})
 
@@ -281,6 +304,21 @@ func attachCommands(l *logrus.Logger, c *config.C, ssh *sshd.SSHServer, f *Inter
 		},
 	})
 
+	ssh.RegisterCommand(&sshd.Command{
+		Name:             "device-info",
+		ShortDescription: "Prints information about the network device.",
+		Flags: func() (*flag.FlagSet, interface{}) {
+			fl := flag.NewFlagSet("", flag.ContinueOnError)
+			s := sshDeviceInfoFlags{}
+			fl.BoolVar(&s.Json, "json", false, "outputs as json with more information")
+			fl.BoolVar(&s.Pretty, "pretty", false, "pretty prints json, assumes -json")
+			return fl, &s
+		},
+		Callback: func(fs interface{}, a []string, w sshd.StringWriter) error {
+			return sshDeviceInfo(f, fs, w)
+		},
+	})
+
 	ssh.RegisterCommand(&sshd.Command{
 		Name:             "print-cert",
 		ShortDescription: "Prints the current certificate being used or the certificate for the provided vpn ip",
@@ -934,7 +972,34 @@ func sshPrintTunnel(ifce *Interface, fs interface{}, a []string, w sshd.StringWr
 		enc.SetIndent("", "    ")
 	}
 
-	return enc.Encode(copyHostInfo(hostInfo, ifce.hostMap.preferredRanges))
+	return enc.Encode(copyHostInfo(hostInfo, ifce.hostMap.GetPreferredRanges()))
+}
+
+func sshDeviceInfo(ifce *Interface, fs interface{}, w sshd.StringWriter) error {
+
+	data := struct {
+		Name string `json:"name"`
+		Cidr string `json:"cidr"`
+	}{
+		Name: ifce.inside.Name(),
+		Cidr: ifce.inside.Cidr().String(),
+	}
+
+	flags, ok := fs.(*sshDeviceInfoFlags)
+	if !ok {
+		return fmt.Errorf("internal error: expected flags to be sshDeviceInfoFlags but was %+v", fs)
+	}
+
+	if flags.Json || flags.Pretty {
+		js := json.NewEncoder(w.GetWriter())
+		if flags.Pretty {
+			js.SetIndent("", "    ")
+		}
+
+		return js.Encode(data)
+	} else {
+		return w.WriteLine(fmt.Sprintf("name=%v cidr=%v", data.Name, data.Cidr))
+	}
 }
 
 func sshReload(c *config.C, w sshd.StringWriter) error {

sshd/server.go

@@ -1,6 +1,7 @@
 package sshd
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"net"
@@ -15,8 +16,11 @@ type SSHServer struct {
 	config *ssh.ServerConfig
 	l      *logrus.Entry
 
+	certChecker *ssh.CertChecker
+
 	// Map of user -> authorized keys
 	trustedKeys map[string]map[string]bool
+	trustedCAs  []ssh.PublicKey
 
 	// List of available commands
 	helpCommand *Command
@@ -31,6 +35,7 @@ type SSHServer struct {
 
 // NewSSHServer creates a new ssh server rigged with default commands and prepares to listen
 func NewSSHServer(l *logrus.Entry) (*SSHServer, error) {
+
 	s := &SSHServer{
 		trustedKeys: make(map[string]map[string]bool),
 		l:           l,
@@ -38,8 +43,43 @@ func NewSSHServer(l *logrus.Entry) (*SSHServer, error) {
 		conns:       make(map[int]*session),
 	}
 
+	cc := ssh.CertChecker{
+		IsUserAuthority: func(auth ssh.PublicKey) bool {
+			for _, ca := range s.trustedCAs {
+				if bytes.Equal(ca.Marshal(), auth.Marshal()) {
+					return true
+				}
+			}
+
+			return false
+		},
+		UserKeyFallback: func(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
+			pk := string(pubKey.Marshal())
+			fp := ssh.FingerprintSHA256(pubKey)
+
+			tk, ok := s.trustedKeys[c.User()]
+			if !ok {
+				return nil, fmt.Errorf("unknown user %s", c.User())
+			}
+
+			_, ok = tk[pk]
+			if !ok {
+				return nil, fmt.Errorf("unknown public key for %s (%s)", c.User(), fp)
+			}
+
+			return &ssh.Permissions{
+				// Record the public key used for authentication.
+				Extensions: map[string]string{
+					"fp":   fp,
+					"user": c.User(),
+				},
+			}, nil
+
+		},
+	}
+
 	s.config = &ssh.ServerConfig{
-		PublicKeyCallback: s.matchPubKey,
+		PublicKeyCallback: cc.Authenticate,
 		//TODO: AuthLogCallback: s.authAttempt,
 		//TODO: version string
 		ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"),
@@ -66,10 +106,26 @@ func (s *SSHServer) SetHostKey(hostPrivateKey []byte) error {
 	return nil
 }
 
+func (s *SSHServer) ClearTrustedCAs() {
+	s.trustedCAs = []ssh.PublicKey{}
+}
+
 func (s *SSHServer) ClearAuthorizedKeys() {
 	s.trustedKeys = make(map[string]map[string]bool)
 }
 
+// AddTrustedCA adds a trusted CA for user certificates
+func (s *SSHServer) AddTrustedCA(pubKey string) error {
+	pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(pubKey))
+	if err != nil {
+		return err
+	}
+
+	s.trustedCAs = append(s.trustedCAs, pk)
+	s.l.WithField("sshKey", pubKey).Info("Trusted CA key")
+	return nil
+}
+
 // AddAuthorizedKey adds an ssh public key for a user
 func (s *SSHServer) AddAuthorizedKey(user, pubKey string) error {
 	pk, _, _, _, err := ssh.ParseAuthorizedKey([]byte(pubKey))
@@ -178,26 +234,3 @@ func (s *SSHServer) closeSessions() {
 	}
 	s.connsLock.Unlock()
 }
-
-func (s *SSHServer) matchPubKey(c ssh.ConnMetadata, pubKey ssh.PublicKey) (*ssh.Permissions, error) {
-	pk := string(pubKey.Marshal())
-	fp := ssh.FingerprintSHA256(pubKey)
-
-	tk, ok := s.trustedKeys[c.User()]
-	if !ok {
-		return nil, fmt.Errorf("unknown user %s", c.User())
-	}
-
-	_, ok = tk[pk]
-	if !ok {
-		return nil, fmt.Errorf("unknown public key for %s (%s)", c.User(), fp)
-	}
-
-	return &ssh.Permissions{
-		// Record the public key used for authentication.
-		Extensions: map[string]string{
-			"fp":   fp,
-			"user": c.User(),
-		},
-	}, nil
-}

udp/udp_linux.go

@@ -22,6 +22,7 @@ import (
 
 type StdConn struct {
 	sysFd int
+	isV4  bool
 	l     *logrus.Logger
 	batch int
 }
@@ -45,9 +46,22 @@ const (
 
 type _SK_MEMINFO [_SK_MEMINFO_VARS]uint32
 
+func maybeIPV4(ip net.IP) (net.IP, bool) {
+	ip4 := ip.To4()
+	if ip4 != nil {
+		return ip4, true
+	}
+	return ip, false
+}
+
 func NewListener(l *logrus.Logger, ip net.IP, port int, multi bool, batch int) (Conn, error) {
+	ipV4, isV4 := maybeIPV4(ip)
+	af := unix.AF_INET6
+	if isV4 {
+		af = unix.AF_INET
+	}
 	syscall.ForkLock.RLock()
-	fd, err := unix.Socket(unix.AF_INET6, unix.SOCK_DGRAM, unix.IPPROTO_UDP)
+	fd, err := unix.Socket(af, unix.SOCK_DGRAM, unix.IPPROTO_UDP)
 	if err == nil {
 		unix.CloseOnExec(fd)
 	}
@@ -58,9 +72,6 @@ func NewListener(l *logrus.Logger, ip net.IP, port int, multi bool, batch int) (
 		return nil, fmt.Errorf("unable to open socket: %s", err)
 	}
 
-	var lip [16]byte
-	copy(lip[:], ip.To16())
-
 	if multi {
 		if err = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil {
 			return nil, fmt.Errorf("unable to set SO_REUSEPORT: %s", err)
@@ -68,7 +79,17 @@ func NewListener(l *logrus.Logger, ip net.IP, port int, multi bool, batch int) (
 	}
 
 	//TODO: support multiple listening IPs (for limiting ipv6)
-	if err = unix.Bind(fd, &unix.SockaddrInet6{Addr: lip, Port: port}); err != nil {
+	var sa unix.Sockaddr
+	if isV4 {
+		sa4 := &unix.SockaddrInet4{Port: port}
+		copy(sa4.Addr[:], ipV4)
+		sa = sa4
+	} else {
+		sa6 := &unix.SockaddrInet6{Port: port}
+		copy(sa6.Addr[:], ip.To16())
+		sa = sa6
+	}
+	if err = unix.Bind(fd, sa); err != nil {
 		return nil, fmt.Errorf("unable to bind to socket: %s", err)
 	}
 
@@ -77,7 +98,7 @@ func NewListener(l *logrus.Logger, ip net.IP, port int, multi bool, batch int) (
 	//v, err := unix.GetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_INCOMING_CPU)
 	//l.Println(v, err)
 
-	return &StdConn{sysFd: fd, l: l, batch: batch}, err
+	return &StdConn{sysFd: fd, isV4: isV4, l: l, batch: batch}, err
 }
 
 func (u *StdConn) Rebind() error {
@@ -143,7 +164,11 @@ func (u *StdConn) ListenOut(r EncReader, lhf LightHouseHandlerFunc, cache *firew
 
 		//metric.Update(int64(n))
 		for i := 0; i < n; i++ {
-			udpAddr.IP = names[i][8:24]
+			if u.isV4 {
+				udpAddr.IP = names[i][4:8]
+			} else {
+				udpAddr.IP = names[i][8:24]
+			}
 			udpAddr.Port = binary.BigEndian.Uint16(names[i][2:4])
 			r(udpAddr, plaintext[:0], buffers[i][:msgs[i].Len], h, fwPacket, lhf, nb, q, cache.Get(u.l))
 		}
@@ -192,13 +217,18 @@ func (u *StdConn) ReadMulti(msgs []rawMessage) (int, error) {
 }
 
 func (u *StdConn) WriteTo(b []byte, addr *Addr) error {
+	if u.isV4 {
+		return u.writeTo4(b, addr)
+	}
+	return u.writeTo6(b, addr)
+}
 
+func (u *StdConn) writeTo6(b []byte, addr *Addr) error {
 	var rsa unix.RawSockaddrInet6
 	rsa.Family = unix.AF_INET6
-	p := (*[2]byte)(unsafe.Pointer(&rsa.Port))
+	// Little Endian -> Network Endian
-	p[0] = byte(addr.Port >> 8)
+	rsa.Port = (addr.Port >> 8) | ((addr.Port & 0xff) << 8)
-	p[1] = byte(addr.Port)
+	copy(rsa.Addr[:], addr.IP.To16())
-	copy(rsa.Addr[:], addr.IP)
 
 	for {
 		_, _, err := unix.Syscall6(
@@ -221,6 +251,39 @@ func (u *StdConn) WriteTo(b []byte, addr *Addr) error {
 	}
 }
 
+func (u *StdConn) writeTo4(b []byte, addr *Addr) error {
+	addrV4, isAddrV4 := maybeIPV4(addr.IP)
+	if !isAddrV4 {
+		return fmt.Errorf("Listener is IPv4, but writing to IPv6 remote")
+	}
+
+	var rsa unix.RawSockaddrInet4
+	rsa.Family = unix.AF_INET
+	// Little Endian -> Network Endian
+	rsa.Port = (addr.Port >> 8) | ((addr.Port & 0xff) << 8)
+	copy(rsa.Addr[:], addrV4)
+
+	for {
+		_, _, err := unix.Syscall6(
+			unix.SYS_SENDTO,
+			uintptr(u.sysFd),
+			uintptr(unsafe.Pointer(&b[0])),
+			uintptr(len(b)),
+			uintptr(0),
+			uintptr(unsafe.Pointer(&rsa)),
+			uintptr(unix.SizeofSockaddrInet4),
+		)
+
+		if err != 0 {
+			return &net.OpError{Op: "sendto", Err: err}
+		}
+
+		//TODO: handle incomplete writes
+
+		return nil
+	}
+}
+
 func (u *StdConn) ReloadConfig(c *config.C) {
 	b := c.GetInt("listen.read_buffer", 0)
 	if b > 0 {

udp/udp_linux_64.go

@@ -1,6 +1,6 @@
-//go:build linux && (amd64 || arm64 || ppc64 || ppc64le || mips64 || mips64le || s390x || riscv64) && !android && !e2e_testing
+//go:build linux && (amd64 || arm64 || ppc64 || ppc64le || mips64 || mips64le || s390x || riscv64 || loong64) && !android && !e2e_testing
 // +build linux
-// +build amd64 arm64 ppc64 ppc64le mips64 mips64le s390x riscv64
+// +build amd64 arm64 ppc64 ppc64le mips64 mips64le s390x riscv64 loong64
 // +build !android
 // +build !e2e_testing
 

util/error.go

@@ -2,6 +2,7 @@ package util
 
 import (
 	"errors"
+	"fmt"
 
 	"github.com/sirupsen/logrus"
 )
@@ -40,7 +41,7 @@ func (ce *ContextualError) Error() string {
 	if ce.RealError == nil {
 		return ce.Context
 	}
-	return ce.RealError.Error()
+	return fmt.Errorf("%s (%v): %w", ce.Context, ce.Fields, ce.RealError).Error()
 }
 
 func (ce *ContextualError) Unwrap() error {