From 227134a1d5bf6b622d42481a783fd9c72bb07b70 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:01:25 -0400 Subject: [PATCH 01/31] Bump golang.org/x/net in the golang-x-dependencies group (#1876) Bumps the golang-x-dependencies group with 1 update: [golang.org/x/net](https://github.com/golang/net). Updates `golang.org/x/net` from 0.58.0 to 0.59.0 - [Commits](https://github.com/golang/net/compare/v0.58.0...v0.59.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-version: 0.59.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: golang-x-dependencies ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f8bac0e1..b2705dd8 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( go.uber.org/goleak v1.3.0 go.yaml.in/yaml/v3 v3.0.5 golang.org/x/crypto v0.57.0 - golang.org/x/net v0.58.0 + golang.org/x/net v0.59.0 golang.org/x/sync v0.23.0 golang.org/x/sys v0.48.0 golang.org/x/term v0.46.0 diff --git a/go.sum b/go.sum index 53372f19..a20fd30a 100644 --- a/go.sum +++ b/go.sum @@ -171,8 +171,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= -golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues= +golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From abff1a45558377c3cc79ed96ae0340c912d62251 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 14 Sep 2026 10:11:27 -0400 Subject: [PATCH 02/31] Bump Vampire/setup-wsl from 3 to 7 (#1720) Bumps [Vampire/setup-wsl](https://github.com/vampire/setup-wsl) from 3 to 7. - [Release notes](https://github.com/vampire/setup-wsl/releases) - [Commits](https://github.com/vampire/setup-wsl/compare/v3...v7) --- updated-dependencies: - dependency-name: Vampire/setup-wsl dependency-version: '7' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/smoke-extra.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/smoke-extra.yml b/.github/workflows/smoke-extra.yml index 445b83ed..8a388f9f 100644 --- a/.github/workflows/smoke-extra.yml +++ b/.github/workflows/smoke-extra.yml @@ -98,7 +98,7 @@ jobs: # WSL2 + Ubuntu so the smoke can run a real linux peer with its own # netns. iputils-ping is needed for the in-WSL ping check. WSL1 has no # real kernel and would lack /dev/net/tun, so we have to force WSL2. - - uses: Vampire/setup-wsl@v3 + - uses: Vampire/setup-wsl@v7 with: distribution: Ubuntu-24.04 additional-packages: iputils-ping iproute2 From 3fc627e08d020c23f5f924a87d29cbc25a973e9f Mon Sep 17 00:00:00 2001 From: Wade Simmons Date: Mon, 14 Sep 2026 16:14:43 -0400 Subject: [PATCH 03/31] Add script to backport a merged PR to a release branch (#1851) * Add script to backport a merged PR to a release branch .github/backport.sh cherry-picks a merged PR's commit onto a release branch and opens a backport PR mirroring the shape of #1842. - Infers the target release from the PR's milestone (v1.12.0 -> release-1.11); an explicit target-version argument overrides it. - Requires a clean working tree (untracked files allowed) before starting. - Prompts with the exact push and gh pr create commands before running them. - Sets the new PR's milestone to the highest open v.* milestone. - Removes the needs-backport label from the original PR once the backport opens. - Supports --continue to resume after resolving cherry-pick conflicts. * cleanup * just fetch all of origin We need to ensure we have MERGE_SHA available too --- .github/backport.sh | 142 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) create mode 100755 .github/backport.sh diff --git a/.github/backport.sh b/.github/backport.sh new file mode 100755 index 00000000..ab188cf5 --- /dev/null +++ b/.github/backport.sh @@ -0,0 +1,142 @@ +#!/usr/bin/env bash +# +# Backport a merged PR to a release branch. +# +# Cherry-picks the merge commit of a merged PR onto release-1.11 and opens a +# backport PR, mirroring the shape of #1842. +# +# Usage: ./backport.sh [target-version] [--continue] +# pr-number The merged PR to backport +# target-version Release version to target (default: 1.11) +# --continue Resume after resolving cherry-pick conflicts: assumes the +# fixes are staged, runs `git cherry-pick --continue`, and +# proceeds to push and open the backport PR. +# +# Requires: gh (authenticated), git, jq. + +set -euo pipefail + +PR_NUMBER=${1:?usage: backport.sh [target-version] [--continue]} +shift +CONTINUE=0 +TARGET_VERSION="" +for arg in "$@"; do + case "$arg" in + --continue) CONTINUE=1 ;; + *) TARGET_VERSION=$arg ;; + esac +done + +for cmd in gh git jq; do + command -v "$cmd" >/dev/null || { echo "error: $cmd is required" >&2; exit 1; } +done + +# Pull the source PR's metadata. Refuse to backport a PR that never merged. +pr_json=$(gh pr view "$PR_NUMBER" --json title,body,mergedAt,mergeCommit,milestone) +PR_TITLE=$(jq -r '.title' <<<"$pr_json") +PR_BODY=$(jq -r '.body' <<<"$pr_json") +MERGE_SHA=$(jq -r '.mergeCommit.oid // empty' <<<"$pr_json") +PR_MILESTONE=$(jq -r '.milestone.title // empty' <<<"$pr_json") + +if [ "$(jq -r '.mergedAt // empty' <<<"$pr_json")" = "" ] || [ -z "$MERGE_SHA" ]; then + echo "error: PR #${PR_NUMBER} is not merged (no merge commit to cherry-pick)" >&2 + exit 1 +fi + +# Default the target to one minor below the PR's milestone: a PR landing in +# v1.12.0 backports to release-1.11. An explicit target-version arg overrides. +if [ -z "$TARGET_VERSION" ]; then + if [[ $PR_MILESTONE =~ ^v?([0-9]+)\.([0-9]+) ]]; then + TARGET_VERSION="${BASH_REMATCH[1]}.$(( BASH_REMATCH[2] - 1 ))" + else + echo "error: PR #${PR_NUMBER} has no v..* milestone to infer the target from." >&2 + echo "Pass the target version explicitly, e.g. $0 ${PR_NUMBER} 1.11" >&2 + exit 1 + fi +fi +TARGET_BRANCH="release-${TARGET_VERSION}" + +# Slug from the PR title: lower-case, non-alphanumerics to dashes, trimmed. +slug=$(printf '%s' "$PR_TITLE" \ + | tr '[:upper:]' '[:lower:]' \ + | sed -E 's/[^a-z0-9]+/-/g; s/^-+//; s/-+$//' \ + | cut -c1-50 \ + | sed -E 's/-+$//') +BRANCH="backport-${TARGET_VERSION//./-}-${slug}" + +if [ "$CONTINUE" -eq 1 ]; then + # Resuming: the branch already exists and a cherry-pick is mid-conflict. + current=$(git rev-parse --abbrev-ref HEAD) + if [ "$current" != "$BRANCH" ]; then + echo "error: --continue expects to be on ${BRANCH}, but HEAD is ${current}" >&2 + exit 1 + fi + echo "Resuming backport of #${PR_NUMBER} on ${BRANCH}" + # Conflicts assumed resolved and staged; core.editor=true keeps the commit message. + git -c core.editor=true cherry-pick --continue +else + # A fresh run switches branches and cherry-picks, so tracked changes must be + # clean. Untracked files are fine, and --continue is exempt (its resolved + # conflicts are meant to be staged). + if [ -n "$(git status --porcelain --untracked-files=no)" ]; then + echo "error: working tree has uncommitted changes to tracked files; commit or stash them first." >&2 + exit 1 + fi + + echo "Backporting #${PR_NUMBER} (${MERGE_SHA}) onto ${TARGET_BRANCH} as ${BRANCH}" + + git fetch origin + git checkout -b "$BRANCH" "origin/${TARGET_BRANCH}" + + # -m 1 handles a real merge commit; plain cherry-pick handles a squash merge. + if ! { git cherry-pick -x -m 1 "$MERGE_SHA" 2>/dev/null || git cherry-pick -x "$MERGE_SHA"; }; then + echo "error: ${MERGE_SHA} did not cherry-pick cleanly onto ${TARGET_BRANCH}." >&2 + echo "Resolve the conflicts, 'git add' them, then re-run:" >&2 + echo " $0 ${PR_NUMBER} ${TARGET_VERSION} --continue" >&2 + exit 1 + fi +fi + +title="backport v${TARGET_VERSION}: ${PR_TITLE}" +body=$(printf 'Backport from #%s to release v%s\n\n---\n\n%s' \ + "$PR_NUMBER" "$TARGET_VERSION" "$PR_BODY") + +# Highest open milestone matching v.* (e.g. v1.11.1), if any. +MILESTONE=$(gh api "repos/{owner}/{repo}/milestones?state=open" \ + --jq ".[].title | select(startswith(\"v${TARGET_VERSION}.\"))" \ + | sort -V | tail -n1) +milestone_args=() +if [ -n "$MILESTONE" ]; then + milestone_args=(--milestone "$MILESTONE") +else + echo "warning: no open milestone matching v${TARGET_VERSION}.* found; PR will have no milestone" >&2 +fi + +echo +echo "About to run:" +printf ' git push -u --force-with-lease origin %q\n' "$BRANCH" +printf ' gh pr create --base %q --head %q --title %q --body %q' \ + "$TARGET_BRANCH" "$BRANCH" "$title" "$body" +[ -n "$MILESTONE" ] && printf ' --milestone %q' "$MILESTONE" +printf '\n' +echo +read -r -p "Open this pull request? [y/N] " reply +case "$reply" in + [yY] | [yY][eE][sS]) ;; + *) echo "Aborted. The cherry-pick is on local branch ${BRANCH}; push it manually if desired."; exit 0 ;; +esac + +git push -u --force-with-lease origin "$BRANCH" + +gh pr create \ + --base "$TARGET_BRANCH" \ + --head "$BRANCH" \ + --title "$title" \ + --body "$body" \ + ${milestone_args[@]+"${milestone_args[@]}"} + +# The backport now exists, so drop the label that flagged this PR for one. +gh pr edit "$PR_NUMBER" --remove-label needs-backport + +# Switch back to the original branch we were on before doing the backport +git checkout - From cf3f315cd3aa07344a1e0c66a41b1b091b7afd19 Mon Sep 17 00:00:00 2001 From: Nate Brown Date: Mon, 21 Sep 2026 12:09:51 -0500 Subject: [PATCH 04/31] IPv6FindUpperProtocol: fail closed when the extension header walk runs out of budget (#1883) --- iputil/packet.go | 8 +++++--- iputil/packet_test.go | 12 ++++++++++++ 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/iputil/packet.go b/iputil/packet.go index ce9eeb93..88f72dcc 100644 --- a/iputil/packet.go +++ b/iputil/packet.go @@ -351,8 +351,8 @@ func ipv6CreateRejectTCPPacket(packet []byte, out []byte, offset int) []byte { // 253/254, and real upper layer protocols like SCTP or GRE, is terminal. Walking those as extension headers // is a firewall bypass, so they fail closed. For a non-first fragment the returned protocol is the fragmented // protocol and offset points at the fragment header, there is no transport header to locate. Returns -// ErrIPv6CouldNotFindPayload if packet is smaller than an ipv6 header or the chain is truncated before a -// terminal protocol is reached. +// ErrIPv6CouldNotFindPayload if packet is smaller than an ipv6 header, or the chain is truncated or longer +// than the walk's budget before a terminal protocol is reached. func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragment bool, anyFragment bool, err error) { const maxIPv6ExtHeaders = 8 if len(packet) < ipv6.HeaderLen { @@ -398,7 +398,9 @@ func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragm return nextHeader, offset, isFragment, anyFragment, nil } } - return nextHeader, offset, isFragment, anyFragment, nil + // Budget spent without reaching a terminal protocol. We don't know where the transport starts, so fail + // closed instead of handing back a mid-chain offset a caller would read ports from. + return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload } func CreateICMPEchoResponse(packet, out []byte) []byte { diff --git a/iputil/packet_test.go b/iputil/packet_test.go index 3baaef21..38c5da10 100644 --- a/iputil/packet_test.go +++ b/iputil/packet_test.go @@ -529,6 +529,17 @@ func Test_IPv6FindUpperProtocol(t *testing.T) { firstFragToUDP := []byte{17, 0, 0, 1, 0, 0, 0, 1} // frag offset 0, M=1, next = UDP nonFirstFrag := []byte{17, 0, 0, 9, 0, 0, 0, 1} // frag offset non-zero, next = UDP transport := []byte{0, 80, 1, 187, 0, 0, 0, 0} // stand-in bytes, IPv6FindUpperProtocol never reads ports + extToHbH := []byte{0, 0, 0, 0, 0, 0, 0, 0} // len 0 -> 8 bytes, next = Hop-by-Hop + + // 8 walked headers spend the budget before the 9th slot's terminal is ever classified. This is the + // report 4040546 shape: without failing closed the walk hands back offset 104, and a caller reads ports + // from the attacker's bytes there. + var budgetChain []byte + for i := 0; i < 7; i++ { + budgetChain = append(budgetChain, extToHbH...) + } + budgetChain = append(budgetChain, extToTCP...) + budgetChain = append(budgetChain, transport...) tests := []struct { name string @@ -552,6 +563,7 @@ func Test_IPv6FindUpperProtocol(t *testing.T) { {"truncated extension header", 0, nil, 0, ipv6.HeaderLen, false, false, ErrIPv6CouldNotFindPayload}, // Destination Options with a declared length (255+1)*8 = 2048 that runs past the 48 byte buffer, next = SCTP {"extension length past buffer", 60, []byte{132, 255, 0, 0, 0, 0, 0, 0}, 132, ipv6.HeaderLen + 2048, false, false, ErrIPv6CouldNotFindPayload}, + {"budget exhausted fails closed", 0, budgetChain, 0, 0, false, false, ErrIPv6CouldNotFindPayload}, } for _, tt := range tests { From 019cc46a7f9bb8ff7cb8876876b4661878493174 Mon Sep 17 00:00:00 2001 From: Jack Doan Date: Mon, 21 Sep 2026 12:59:31 -0500 Subject: [PATCH 05/31] smoke: Prove windows honors NlMtu with a 9000 byte ping (#1873) --- .github/workflows/smoke/smoke-windows.ps1 | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/smoke/smoke-windows.ps1 b/.github/workflows/smoke/smoke-windows.ps1 index e94768a9..c8f55b51 100644 --- a/.github/workflows/smoke/smoke-windows.ps1 +++ b/.github/workflows/smoke/smoke-windows.ps1 @@ -263,6 +263,20 @@ try { } Write-Host "OK: WSL peer -> windows lighthouse over v6" + # The NlMtu check above proves the value was written, not that windows honors it. A payload this size only + # crosses the tunnel as fragments cut at NlMtu on the way out of the adapter. Left at the adapter default it + # reaches nebula whole, overflows the udp.MTU (9001) write buffer and is dropped, so the ping never answers. + # Anything smaller would ride out as one oversized datagram and survive on IP fragmentation of the underlay. + $BigPayload = 9000 + foreach ($target in @(@{ Family = 'v4'; Ip = $Ip2 }, @{ Family = 'v6'; Ip = $Ip6_2 })) { + Wait-Until -TimeoutSec 30 -What "$($target.Family) ping with a $BigPayload byte payload from windows lighthouse to WSL peer ($($target.Ip))" -Predicate { + if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the large $($target.Family) ping succeeded" } + $null = & ping.exe -n 1 -w 1000 -l $BigPayload $target.Ip + $LASTEXITCODE -eq 0 + } + Write-Host "OK: windows lighthouse -> WSL peer, $BigPayload byte $($target.Family) payload" + } + Write-Host '' Write-Host 'All smoke checks passed.' } From 3a45c01656f6798c39ed46b47ba9ffb3396d0109 Mon Sep 17 00:00:00 2001 From: Jack Doan Date: Mon, 21 Sep 2026 14:59:56 -0500 Subject: [PATCH 06/31] Advertise underlay addresses on Android (#1881) --- Makefile | 3 + go.mod | 2 +- go.sum | 6 +- hostmap.go | 58 ---------------- lighthouse.go | 29 +++++++- lighthouse_test.go | 31 +++++++++ nicfinder/localaddrs.go | 39 +++++++++++ nicfinder/localaddrs_android.go | 104 ++++++++++++++++++++++++++++ nicfinder/nicfinder.go | 59 ++++++++++++++++ nicfinder/nicfinder_test.go | 117 ++++++++++++++++++++++++++++++++ overlay/tun_linux.go | 2 +- 11 files changed, 385 insertions(+), 65 deletions(-) create mode 100644 nicfinder/localaddrs.go create mode 100644 nicfinder/localaddrs_android.go create mode 100644 nicfinder/nicfinder.go create mode 100644 nicfinder/nicfinder_test.go diff --git a/Makefile b/Makefile index 4161a09b..7dc15869 100644 --- a/Makefile +++ b/Makefile @@ -242,11 +242,14 @@ test-cov-html: go test -coverprofile=coverage.out go tool cover -html=coverage.out +# The package builds only compile. The final line links an android binary so a linker-only failure +# cannot pass CI. build-test-mobile: GOARCH=amd64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=arm64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=amd64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=arm64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/') + GOARCH=arm64 GOOS=android go build -o /dev/null ${NEBULA_CMD_PATH} bench: go test -bench=. diff --git a/go.mod b/go.mod index b2705dd8..a110b8a2 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.26.0 require ( dario.cat/mergo v1.0.2 filippo.io/bigmod v0.1.0 + github.com/DefinedNet/netlink v1.3.2-dn.1 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be github.com/armon/go-radix v1.0.0 github.com/flynn/noise v1.1.0 @@ -20,7 +21,6 @@ require ( github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 github.com/stretchr/testify v1.12.1 - github.com/vishvananda/netlink v1.3.1 go.uber.org/goleak v1.3.0 go.yaml.in/yaml/v3 v3.0.5 golang.org/x/crypto v0.57.0 diff --git a/go.sum b/go.sum index a20fd30a..78189af1 100644 --- a/go.sum +++ b/go.sum @@ -3,6 +3,8 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= filippo.io/bigmod v0.1.0 h1:UNzDk7y9ADKST+axd9skUpBQeW7fG2KrTZyOE4uGQy8= filippo.io/bigmod v0.1.0/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI= +github.com/DefinedNet/netlink v1.3.2-dn.1 h1:Md2PsUE1Xuy1Zxs/Bm6356n1kt5bq2HspXlPJ6pEspg= +github.com/DefinedNet/netlink v1.3.2-dn.1/go.mod h1:o4YSn8Jw7zoDcYXDmV77TRd4eS5sSLuE+vu2zAN67C8= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -138,8 +140,6 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= -github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0= -github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4= github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -195,8 +195,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/hostmap.go b/hostmap.go index f54df876..71814a2f 100644 --- a/hostmap.go +++ b/hostmap.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "log/slog" - "net" "net/netip" "slices" "sync" @@ -18,7 +17,6 @@ import ( "github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/header" - "github.com/slackhq/nebula/logging" ) const defaultPromoteEvery = 1000 // Count of packets sent before we try moving a tunnel to a preferred underlay ip address @@ -930,59 +928,3 @@ func (i *HostInfo) logger(l *slog.Logger) *slog.Logger { return li } - -// Utility functions - -func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr { - //FIXME: This function is pretty garbage - var finalAddrs []netip.Addr - ifaces, _ := net.Interfaces() - for _, i := range ifaces { - allow := allowList.AllowName(i.Name) - if l.Enabled(context.Background(), logging.LevelTrace) { - l.Log(context.Background(), logging.LevelTrace, "localAllowList.AllowName", - "interfaceName", i.Name, - "allow", allow, - ) - } - - if !allow { - continue - } - addrs, _ := i.Addrs() - for _, rawAddr := range addrs { - var addr netip.Addr - switch v := rawAddr.(type) { - case *net.IPNet: - //continue - addr, _ = netip.AddrFromSlice(v.IP) - case *net.IPAddr: - addr, _ = netip.AddrFromSlice(v.IP) - } - - if !addr.IsValid() { - if l.Enabled(context.Background(), slog.LevelDebug) { - l.Debug("addr was invalid", "localAddr", rawAddr) - } - continue - } - addr = addr.Unmap() - - if addr.IsLoopback() == false && addr.IsLinkLocalUnicast() == false { - isAllowed := allowList.Allow(addr) - if l.Enabled(context.Background(), logging.LevelTrace) { - l.Log(context.Background(), logging.LevelTrace, "localAllowList.Allow", - "localAddr", addr, - "allowed", isAllowed, - ) - } - if !isAllowed { - continue - } - - finalAddrs = append(finalAddrs, addr) - } - } - } - return finalAddrs -} diff --git a/lighthouse.go b/lighthouse.go index df279069..58037935 100644 --- a/lighthouse.go +++ b/lighthouse.go @@ -19,6 +19,7 @@ import ( "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/header" "github.com/slackhq/nebula/logging" + "github.com/slackhq/nebula/nicfinder" "github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/util" ) @@ -42,6 +43,10 @@ type LightHouse struct { // addresses rather than whatever this machine's NICs happen to be. Set it before Start. localAddrsFn func(*LocalAllowList) []netip.Addr + // lastLocalAddrsErr is the previous localAddrsFn failure. Enumeration runs on every update, so an + // unchanged failure is demoted to Debug rather than warning every lighthouse.interval forever. + lastLocalAddrsErr atomic.Pointer[string] + // Local cache of answers from light houses // map of vpn addr to answers addrMap map[netip.Addr]*RemoteList @@ -115,7 +120,9 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c l: l, } h.localAddrsFn = func(al *LocalAllowList) []netip.Addr { - return localAddrs(h.l, al) + addrs, err := nicfinder.Find(ctx, h.l, al) + h.logLocalAddrsErr(err) + return addrs } lighthouses := make([]netip.Addr, 0) @@ -916,6 +923,26 @@ func (lh *LightHouse) TriggerUpdate() { } } +// logLocalAddrsErr reports a localAddrs failure at Warn the first time it is seen and at Debug while +// it persists unchanged, so a permanent failure does not warn on every update forever. +func (lh *LightHouse) logLocalAddrsErr(err error) { + if err == nil { + lh.lastLocalAddrsErr.Store(nil) + return + } + + msg := err.Error() + prev := lh.lastLocalAddrsErr.Swap(&msg) + if prev != nil && *prev == msg { + if lh.l.Enabled(context.Background(), slog.LevelDebug) { + lh.l.Debug("Failed to collect local addresses to advertise", "error", err) + } + return + } + + lh.l.Warn("Failed to collect local addresses to advertise", "error", err) +} + func (lh *LightHouse) SendUpdate() { var v4 []*V4AddrPort var v6 []*V6AddrPort diff --git a/lighthouse_test.go b/lighthouse_test.go index 9f3ffddf..8fc30223 100644 --- a/lighthouse_test.go +++ b/lighthouse_test.go @@ -1,7 +1,9 @@ package nebula import ( + "bytes" "encoding/binary" + "errors" "fmt" "net/netip" "testing" @@ -764,3 +766,32 @@ func TestLighthouse_DeletesWork(t *testing.T) { out = lh.Query(testHost) assert.Nil(t, out) } + +func TestLightHouse_logLocalAddrsErr(t *testing.T) { + out := &bytes.Buffer{} + lh := &LightHouse{l: test.NewLoggerWithOutput(out)} + + // The first sighting of a failure warns. + lh.logLocalAddrsErr(errors.New("permission denied")) + assert.Contains(t, out.String(), "level=WARN") + assert.Contains(t, out.String(), "permission denied") + + // Repeating unchanged does not warn again, which is what keeps a permanent failure from warning + // on every lighthouse.interval for the life of the process. + out.Reset() + lh.logLocalAddrsErr(errors.New("permission denied")) + assert.NotContains(t, out.String(), "level=WARN") + + // A different failure is a new event and warns. + out.Reset() + lh.logLocalAddrsErr(errors.New("something else")) + assert.Contains(t, out.String(), "level=WARN") + assert.Contains(t, out.String(), "something else") + + // Recovering resets, so the same failure returning later warns again. + out.Reset() + lh.logLocalAddrsErr(nil) + assert.Empty(t, out.String()) + lh.logLocalAddrsErr(errors.New("something else")) + assert.Contains(t, out.String(), "level=WARN") +} diff --git a/nicfinder/localaddrs.go b/nicfinder/localaddrs.go new file mode 100644 index 00000000..2f7bd81e --- /dev/null +++ b/nicfinder/localaddrs.go @@ -0,0 +1,39 @@ +//go:build !android + +package nicfinder + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net" + "net/netip" +) + +func localAddrs(ctx context.Context, l *slog.Logger, filter Filter) ([]netip.Addr, error) { + ifaces, err := net.Interfaces() + if err != nil { + return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err) + } + + var out []netip.Addr + var errs []error + + for _, i := range ifaces { + if !allowName(ctx, l, filter, i.Name) { + continue + } + addrs, err := i.Addrs() + if err != nil { + errs = append(errs, fmt.Errorf("failed to get addresses for %s: %w", i.Name, err)) + continue + } + for _, rawAddr := range addrs { + if addr, ok := allowedAddr(ctx, l, filter, rawAddr); ok { + out = append(out, addr) + } + } + } + return out, errors.Join(errs...) +} diff --git a/nicfinder/localaddrs_android.go b/nicfinder/localaddrs_android.go new file mode 100644 index 00000000..b66ecbbf --- /dev/null +++ b/nicfinder/localaddrs_android.go @@ -0,0 +1,104 @@ +//go:build android + +package nicfinder + +import ( + "context" + "errors" + "fmt" + "log/slog" + "net/netip" + "os" + + "github.com/DefinedNet/netlink" + "github.com/DefinedNet/netlink/nl" + "golang.org/x/sys/unix" +) + +// Android denies bind() on netlink route sockets and RTM_GETLINK, so net.Interfaces fails with EACCES. +// Enumerate with an RTM_GETADDR dump on an unbound socket instead. + +// localInterfaces returns every interface that has an address, with its addresses. It lists +// addresses over a netlink socket that is never bound and names each index with SIOCGIFNAME, the +// same operations Bionic's getifaddrs uses for app UIDs, which the sandbox permits. +func localAddrs(ctx context.Context, l *slog.Logger, filter Filter) ([]netip.Addr, error) { + var errs []error + addrs, err := netlinkAddrList() + if err != nil { + if !errors.Is(err, netlink.ErrDumpInterrupted) { + return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err) + } + // The address table kept changing under the dump and netlink gave up retrying. What it + // returned is still worth advertising. + errs = append(errs, fmt.Errorf("local addresses may be incomplete: %w", err)) + } + + // Any socket will do for SIOCGIFNAME. + sfd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0) + if err != nil { + return nil, os.NewSyscallError("socket", err) + } + defer unix.Close(sfd) + + var out []netip.Addr + nameAllowed := make(map[int]bool) // by interface index, once named + for _, a := range addrs { + allow, named := nameAllowed[a.LinkIndex] + if (named && !allow) || a.IPNet == nil { + continue + } + addr, ok := allowedAddr(ctx, l, filter, a.IPNet) + if !ok { + continue + } + if !named { + name, err := interfaceNameByIndex(sfd, a.LinkIndex) + // ENODEV means the interface went away after the dump. Any other failure is reported + // and the interface skipped, so the rest are still returned. + if err != nil && !errors.Is(err, unix.ENODEV) { + errs = append(errs, fmt.Errorf("failed to resolve name of interface %d: %w", a.LinkIndex, err)) + } + allow = err == nil && allowName(ctx, l, filter, name) + nameAllowed[a.LinkIndex] = allow + } + if allow { + out = append(out, addr) + } + } + return out, errors.Join(errs...) +} + +// netlinkAddrList is netlink.AddrList over a socket that is never bound. The kernel autobinds it +// on the first send, which the sandbox permits. +func netlinkAddrList() ([]netlink.Addr, error) { + fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE) + if err != nil { + return nil, os.NewSyscallError("socket", err) + } + s, err := nl.NewNetlinkSocketFromFd(fd) + if err != nil { + unix.Close(fd) + return nil, err + } + + h := netlink.NewHandleFromSockets( + map[int]*nl.SocketHandle{unix.NETLINK_ROUTE: {Socket: s}}, + netlink.HandleOptions{RetryInterrupted: true}, + ) + defer h.Close() + if err := h.SetSocketTimeout(netlink.GetSocketTimeout()); err != nil { + return nil, err + } + + return h.AddrList(nil, netlink.FAMILY_ALL) +} + +// interfaceNameByIndex resolves an interface index with SIOCGIFNAME. +func interfaceNameByIndex(fd int, index int) (string, error) { + var ifr unix.Ifreq + ifr.SetUint32(uint32(index)) // ifr_ifindex is in the ifr_ifru union + if err := unix.IoctlIfreq(fd, unix.SIOCGIFNAME, &ifr); err != nil { + return "", err + } + return ifr.Name(), nil +} diff --git a/nicfinder/nicfinder.go b/nicfinder/nicfinder.go new file mode 100644 index 00000000..064e59e6 --- /dev/null +++ b/nicfinder/nicfinder.go @@ -0,0 +1,59 @@ +package nicfinder + +import ( + "context" + "log/slog" + "net" + "net/netip" + + "github.com/slackhq/nebula/logging" +) + +// Filter decides which interfaces, by name, and which of their addresses Find returns. +type Filter interface { + AllowName(string) bool + Allow(netip.Addr) bool +} + +// Find returns the addresses of this host's network interfaces that allowList accepts, with +// loopback and link-local addresses left out. When only some interfaces could be read it returns +// their addresses alongside an error for the rest. allowList must not be nil; it is called as +// given, so a nil pointer of a type whose methods accept a nil receiver is fine. +func Find(ctx context.Context, l *slog.Logger, allowList Filter) ([]netip.Addr, error) { + return localAddrs(ctx, l, allowList) +} + +// allowName asks filter about an interface name and traces the answer. +func allowName(ctx context.Context, l *slog.Logger, filter Filter, name string) bool { + allow := filter.AllowName(name) + if l.Enabled(ctx, logging.LevelTrace) { + l.Log(ctx, logging.LevelTrace, "localAllowList.AllowName", "interfaceName", name, "allow", allow) + } + return allow +} + +// allowedAddr converts one address a platform reports and says whether to advertise it: it must +// be a *net.IPNet or *net.IPAddr carrying an IP, not be loopback or link-local, and be accepted by +// filter, whose answer is traced. IPv4 comes back unmapped. +func allowedAddr(ctx context.Context, l *slog.Logger, filter Filter, rawAddr net.Addr) (netip.Addr, bool) { + var ip net.IP + switch v := rawAddr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + addr, ok := netip.AddrFromSlice(ip) + if !ok { + return netip.Addr{}, false + } + addr = addr.Unmap() + if addr.IsLoopback() || addr.IsLinkLocalUnicast() { + return netip.Addr{}, false + } + isAllowed := filter.Allow(addr) + if l.Enabled(ctx, logging.LevelTrace) { + l.Log(ctx, logging.LevelTrace, "localAllowList.Allow", "localAddr", addr, "allowed", isAllowed) + } + return addr, isAllowed +} diff --git a/nicfinder/nicfinder_test.go b/nicfinder/nicfinder_test.go new file mode 100644 index 00000000..5b56a518 --- /dev/null +++ b/nicfinder/nicfinder_test.go @@ -0,0 +1,117 @@ +package nicfinder + +import ( + "context" + "net" + "net/netip" + "regexp" + "testing" + + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/assert" +) + +type allowAll struct{} + +func (allowAll) Allow(netip.Addr) bool { return true } +func (allowAll) AllowName(string) bool { return true } + +type testAllowList struct { + bannedName *regexp.Regexp + bannedAddr netip.Addr +} + +func (l *testAllowList) Allow(addr netip.Addr) bool { + return addr != l.bannedAddr +} + +func (l *testAllowList) AllowName(n string) bool { + return !l.bannedName.MatchString(n) +} + +func newTestAllowList(bannedName string) *testAllowList { + return &testAllowList{ + bannedName: regexp.MustCompile(bannedName), + } +} + +func TestAllowedAddr(t *testing.T) { + ctx, l := context.Background(), test.NewLogger() + allowed := func(filter Filter, addrs []net.Addr) []netip.Addr { + var out []netip.Addr + for _, rawAddr := range addrs { + if addr, ok := allowedAddr(ctx, l, filter, rawAddr); ok { + out = append(out, addr) + } + } + return out + } + addrs := []net.Addr{ + &net.IPNet{IP: net.ParseIP("127.0.0.1"), Mask: net.CIDRMask(8, 32)}, + &net.IPNet{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)}, + &net.IPNet{IP: net.ParseIP("10.0.0.5"), Mask: net.CIDRMask(24, 32)}, + &net.IPNet{IP: net.ParseIP("169.254.10.10"), Mask: net.CIDRMask(16, 32)}, + &net.IPNet{IP: net.ParseIP("fe80::1"), Mask: net.CIDRMask(64, 128)}, + &net.IPAddr{IP: net.ParseIP("fd00::5")}, + &net.IPNet{}, + otherAddr{}, + nil, + } + + // Loopback, link local and anything without an IP are dropped, and IPv4 comes back unmapped + // even though net.ParseIP hands over a 16 byte slice. + assert.Equal(t, []netip.Addr{ + netip.MustParseAddr("10.0.0.5"), + netip.MustParseAddr("fd00::5"), + }, allowed(allowAll{}, addrs)) + + // The filter sees the unmapped form and can drop a single address. + al := newTestAllowList("none") + al.bannedAddr = netip.MustParseAddr("10.0.0.5") + assert.Equal(t, []netip.Addr{netip.MustParseAddr("fd00::5")}, allowed(al, addrs)) +} + +// otherAddr is a net.Addr that is neither *net.IPNet nor *net.IPAddr. netlink.Addr is one such +// type: it satisfies net.Addr through an embedded *net.IPNet, and passing it directly once +// silently produced no addresses. +type otherAddr struct{} + +func (otherAddr) Network() string { return "other" } +func (otherAddr) String() string { return "other" } + +// hostFilter counts AllowName calls and can reject every name or every address. +type hostFilter struct { + rejectNames bool + rejectAddrs bool + asked map[string]int +} + +func (f *hostFilter) Allow(netip.Addr) bool { return !f.rejectAddrs } +func (f *hostFilter) AllowName(name string) bool { + f.asked[name]++ + return !f.rejectNames +} + +// Runs this platform's localAddrs against the host and checks it honors the whole filter. +func TestLocalAddrsAppliesFilter(t *testing.T) { + ctx, l := context.Background(), test.NewLogger() + + f := &hostFilter{asked: map[string]int{}} + all, err := localAddrs(ctx, l, f) + if len(f.asked) == 0 { + t.Skipf("no interfaces to enumerate here: %v", err) + } + for name, n := range f.asked { + assert.Equal(t, 1, n, "AllowName asked %d times for %s", n, name) + } + // Loopback and link-local never come back, whatever the filter says. + for _, addr := range all { + assert.False(t, addr.IsLoopback() || addr.IsLinkLocalUnicast(), "%s", addr) + } + + none, _ := localAddrs(ctx, l, &hostFilter{rejectNames: true, asked: map[string]int{}}) + assert.Empty(t, none, "addresses got past a filter that rejects every interface name") + + none, _ = localAddrs(ctx, l, &hostFilter{rejectAddrs: true, asked: map[string]int{}}) + assert.Empty(t, none, "addresses got past a filter that rejects every address") +} diff --git a/overlay/tun_linux.go b/overlay/tun_linux.go index 7e7e0981..358b1421 100644 --- a/overlay/tun_linux.go +++ b/overlay/tun_linux.go @@ -15,12 +15,12 @@ import ( "time" "unsafe" + "github.com/DefinedNet/netlink" "github.com/gaissmai/bart" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/overlay/tio" "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/util" - "github.com/vishvananda/netlink" "golang.org/x/sys/unix" ) From 9651ffdb0df4254c1c16d11948482a417f33d802 Mon Sep 17 00:00:00 2001 From: Caleb Jasik Date: Tue, 22 Sep 2026 11:52:28 -0500 Subject: [PATCH 07/31] Cancel the context when Main hands back no Control (#1889) The goroutines Main starts stop only when its context does, and only a caller holding the Control can arrange that. Cancelling solely on error left a config test, which returns no Control and no error, with the lighthouse query worker and a hostname resolver per dns named static host running for the life of the process. Every embedder that validates a config in process rather than by exec'ing pays for that on each config load, dnclient and the apple clients included. Co-authored-by: Nate Brown --- main.go | 7 +++-- main_test.go | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/main.go b/main.go index ee52e7b7..52f7bea8 100644 --- a/main.go +++ b/main.go @@ -32,9 +32,12 @@ const _ = noiseutil.RejectHeadroom - 4*maxRoutines func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) { ctx, cancel := context.WithCancel(context.Background()) - // Automatically cancel the context if Main returns an error, to signal all created goroutines to quit. + // The goroutines started below stop only when this context does, and only a caller holding the + // Control can arrange that. Cancel whenever we are not handing one back, which covers an error + // and a config test alike: a config test used to leave the lighthouse query worker, and a + // hostname resolver per dns named static host, running for the life of the process. defer func() { - if reterr != nil { + if retcon == nil { cancel() } }() diff --git a/main_test.go b/main_test.go index bbaef347..e217ffbf 100644 --- a/main_test.go +++ b/main_test.go @@ -1,12 +1,21 @@ package nebula import ( + "fmt" + "net/netip" + "os" + "path/filepath" "testing" + "time" + "github.com/slackhq/nebula/cert" + cert_test "github.com/slackhq/nebula/cert_test" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/test" "github.com/slackhq/nebula/util" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/goleak" ) func TestParseCpuAffinity(t *testing.T) { @@ -49,3 +58,68 @@ func TestParseCpuAffinity(t *testing.T) { assert.Nil(t, parseCpuAffinity(newConfig([]any{1 << 20}), l, 1)) } } + +// TestMain_ConfigTestReleasesItsGoroutines pins the rule that Main only leaves goroutines running +// when it hands back a Control to stop them with. +// +// A config test gets no Control, so anything it started had nothing to stop it: the lighthouse +// query worker, and a hostname resolver per dns named static host, ran for the life of the +// process. That matters to every embedder that validates a config in process rather than by +// exec'ing, dnclient and the apple clients included, because they do it on each config load and +// the leak accumulates. +func TestMain_ConfigTestReleasesItsGoroutines(t *testing.T) { + defer goleak.VerifyNone(t, goleak.IgnoreCurrent()) + + l := test.NewLogger() + dir := t.TempDir() + + before := time.Now().Add(-time.Hour) + after := time.Now().Add(time.Hour) + ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, before, after, nil, nil, nil) + networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")} + _, _, keyPEM, certPEM := cert_test.NewTestCert( + cert.Version2, cert.Curve_CURVE25519, ca, caKey, "config-test", before, after, networks, nil, nil) + + caPath := filepath.Join(dir, "ca.pem") + certPath := filepath.Join(dir, "cert.pem") + keyPath := filepath.Join(dir, "key.pem") + require.NoError(t, os.WriteFile(caPath, caPEM, 0o600)) + require.NoError(t, os.WriteFile(certPath, certPEM, 0o600)) + require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600)) + + // A static host by address, not by name: the query worker is the goroutine under test and a + // hostname would drag a real dns lookup into a unit test. + configBody := fmt.Sprintf(` +pki: + ca: %s + cert: %s + key: %s +static_host_map: + "10.0.0.2": ["192.0.2.1:4242"] +lighthouse: + hosts: + - "10.0.0.2" +listen: + host: 127.0.0.1 + port: 0 +tun: + disabled: true +firewall: + outbound: + - port: any + proto: any + host: any + inbound: + - port: any + proto: any + host: any +`, caPath, certPath, keyPath) + require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(configBody), 0o600)) + + c := config.NewC(l) + require.NoError(t, c.Load(dir)) + + ctrl, err := Main(c, true, "config-test", l, nil) + require.NoError(t, err) + require.Nil(t, ctrl, "a config test hands back nothing to stop, so it must stop itself") +} From e20569ce5110a75891a8aa1994a1fef35ed386c3 Mon Sep 17 00:00:00 2001 From: Caleb Jasik Date: Tue, 22 Sep 2026 11:54:30 -0500 Subject: [PATCH 08/31] Let the darwin tun adopt an fd the host already configured (#1890) newTunFromFd was a stub on darwin, so an embedder handed a utun by the OS had no way to use it. A darwin network extension is given exactly that: NEPacketTunnelProvider creates the device and applies addressing, mtu and routes from its network settings, and a sandboxed extension cannot change them afterwards. Adopting that fd means everything about moving packets is shared with newTun and only the setup differs, so the tun tracks whether the host owns it and leaves the system routing table and interface config alone when it does. Co-authored-by: Nate Brown --- overlay/tun_darwin.go | 64 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/overlay/tun_darwin.go b/overlay/tun_darwin.go index bb5528e3..c086eda2 100644 --- a/overlay/tun_darwin.go +++ b/overlay/tun_darwin.go @@ -29,7 +29,10 @@ type tun struct { Routes atomic.Pointer[[]Route] routeTree atomic.Pointer[bart.Table[routing.Gateways]] linkAddr *netroute.LinkAddr - l *slog.Logger + // hostOwned means the fd arrived from the OS, which has already configured addressing, mtu + // and routes for it. NEPacketTunnelProvider on darwin does this. + hostOwned bool + l *slog.Logger } type ifReq struct { @@ -149,8 +152,48 @@ func (t *tun) deviceBytes() (o [16]byte) { return } -func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) { - return nil, fmt.Errorf("newTunFromFd not supported in Darwin") +// newTunFromFd adopts a utun the host already created and configured, which is how a darwin +// network extension is handed its device. Everything about moving packets is shared with newTun, +// only the setup differs: the host owns addressing and routing here. +func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) { + if err := unix.SetNonblock(deviceFd, true); err != nil { + // We own the fd from the moment it is handed to us + _ = unix.Close(deviceFd) + return nil, fmt.Errorf("failed to set the tun fd to non-blocking mode: %w", err) + } + + file := os.NewFile(uintptr(deviceFd), "/dev/tun") + t := &tun{ + f: file, + Device: utunNameFromFd(deviceFd), + vpnNetworks: vpnNetworks, + DefaultMTU: c.GetInt("tun.mtu", DefaultMTU), + hostOwned: true, + l: l, + } + + if err := t.reload(c, true); err != nil { + _ = file.Close() + return nil, err + } + + c.RegisterReloadCallback(func(c *config.C) { + if err := t.reload(c, false); err != nil { + util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + } + }) + + return t, nil +} + +// utunNameFromFd asks the socket what interface it is, for logs. A blank name is not worth +// failing a tunnel over, so an error just leaves it empty. +func utunNameFromFd(fd int) string { + name, err := unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME) + if err != nil { + return "" + } + return name } func (t *tun) Close() error { @@ -161,6 +204,12 @@ func (t *tun) Close() error { } func (t *tun) Activate() error { + // The host handed us a configured device. Its addresses, mtu and routes come from the network + // settings it applied, and a sandboxed extension cannot change them anyway. + if t.hostOwned { + return nil + } + devName := t.deviceBytes() s, err := unix.Socket( @@ -374,6 +423,11 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) { } func (t *tun) addRoutes(logErrors bool) error { + // The route tree is still ours, the system routing table is not + if t.hostOwned { + return nil + } + routes := *t.Routes.Load() for _, r := range routes { @@ -403,6 +457,10 @@ func (t *tun) addRoutes(logErrors bool) error { } func (t *tun) removeRoutes(routes []Route) error { + if t.hostOwned { + return nil + } + for _, r := range routes { if !r.Install { continue From 7cfa47d569e8cf89623f6f7078ebbd95067d76a5 Mon Sep 17 00:00:00 2001 From: Jack Doan Date: Tue, 22 Sep 2026 15:45:01 -0500 Subject: [PATCH 09/31] tweaks to error handling in iputil.IPv6FindUpperProtocol (#1893) --- iputil/checksum.go | 16 +++------------- iputil/packet.go | 31 +++++++++++++------------------ iputil/packet_test.go | 12 ++++++------ outside.go | 2 +- outside_test.go | 14 +++++++------- 5 files changed, 30 insertions(+), 45 deletions(-) diff --git a/iputil/checksum.go b/iputil/checksum.go index 08efb7fd..903a1514 100644 --- a/iputil/checksum.go +++ b/iputil/checksum.go @@ -45,8 +45,7 @@ func setTransportChecksum4(packet []byte) { if ihl < ipv4.HeaderLen || end < ihl || end > len(packet) { return } - // The checksum covers the whole datagram, which a fragment (MF set or a - // non-zero offset) does not carry. + // do not attempt to set checksums for fragments if binary.BigEndian.Uint16(packet[6:8])&0x3fff != 0 { return } @@ -68,10 +67,7 @@ func setTransportChecksum6(packet []byte) { return } - // The checksum covers the whole datagram, which a fragment does not carry. - // An unknown extension header hides where the transport header starts. A - // chain longer than the walk's budget ends it early, at an offset that was - // never checked against the packet. + // we must find the transport header in order to write a checksum proto, offset, _, anyFragment, err := IPv6FindUpperProtocol(packet[:end]) if err != nil || anyFragment || offset >= end { return @@ -109,8 +105,7 @@ func transportExtent(transport []byte, proto uint8) ([]byte, bool) { // writeTransportChecksum stores the checksum of transport, taken over the // pseudo-header sum csum, in the header's checksum field. A UDP checksum that // computes to zero goes on the wire as 0xffff: zero means no checksum was -// computed (RFC 768), and over IPv6 the checksum is mandatory (RFC 8200 -// section 8.1). +// computed (RFC 768), and over IPv6 the checksum is mandatory (RFC 8200 section 8.1). func writeTransportChecksum(transport []byte, proto uint8, csum uint32) { var at, minLen int switch proto { @@ -133,11 +128,6 @@ func writeTransportChecksum(transport []byte, proto uint8, csum uint32) { binary.BigEndian.PutUint16(transport[at:], sum) } -// fold reduces a pseudo-header sum to the 16 bit seed Checksum takes. Carrying -// the high half back into the low half is what keeps the reduction lossless, so -// the seed sums exactly as the wider value would; 0xffff is its fixed point. -// Every term of that sum comes from a 16 bit field, so it stays far below the -// width at which the accumulator would wrap. func fold(csum uint32) uint16 { for csum > 0xffff { csum = (csum >> 16) + (csum & 0xffff) diff --git a/iputil/packet.go b/iputil/packet.go index 88f72dcc..166ede60 100644 --- a/iputil/packet.go +++ b/iputil/packet.go @@ -345,14 +345,11 @@ func ipv6CreateRejectTCPPacket(packet []byte, out []byte, offset int) []byte { return out } -// IPv6FindUpperProtocol walks the ipv6 extension header chain and returns the upper layer protocol, the -// offset it begins at, and whether the packet is a non-first fragment. Only the RFC 8200 and IANA extension -// headers below are walked. Everything else, including Mobility (135), HIP (139), Shim6 (140), experimental -// 253/254, and real upper layer protocols like SCTP or GRE, is terminal. Walking those as extension headers -// is a firewall bypass, so they fail closed. For a non-first fragment the returned protocol is the fragmented -// protocol and offset points at the fragment header, there is no transport header to locate. Returns -// ErrIPv6CouldNotFindPayload if packet is smaller than an ipv6 header, or the chain is truncated or longer -// than the walk's budget before a terminal protocol is reached. +// IPv6FindUpperProtocol walks the ipv6 extension header chain and returns the upper layer protocol, the offset it begins at, and whether the packet is a non-first fragment. +// Only the RFC 8200 and IANA extension headers below are walked. +// Everything else, including Mobility (135), HIP (139), Shim6 (140), experimental 253/254, and real upper layer protocols like SCTP or GRE, is terminal. +// For a non-first fragment the returned protocol is the fragmented protocol and offset points at the fragment header, there is no transport header to locate. +// Returns ErrIPv6CouldNotFindPayload if packet is smaller than an ipv6 header, or the chain is truncated, or if more than 8 extension headers are present. func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragment bool, anyFragment bool, err error) { const maxIPv6ExtHeaders = 8 if len(packet) < ipv6.HeaderLen { @@ -365,14 +362,14 @@ func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragm switch nextHeader { case 0, 43, 60: // Hop-by-Hop, Routing, Destination if len(packet) < offset+2 { - return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload + return 0, 0, false, false, ErrIPv6CouldNotFindPayload } nextHeader = packet[offset] offset += (int(packet[offset+1]) + 1) << 3 case 44: // Fragment if len(packet) < offset+8 { - return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload + return 0, 0, false, false, ErrIPv6CouldNotFindPayload } anyFragment = true // Non-first fragments carry no transport header, report the fragmented protocol and stop @@ -384,23 +381,21 @@ func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragm case 51: // AH if len(packet) < offset+2 { - return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload + return 0, 0, false, false, ErrIPv6CouldNotFindPayload } nextHeader = packet[offset] offset += (int(packet[offset+1]) + 2) << 2 default: - // A prior extension header can declare a length that advances offset past the packet. The terminal - // protocol's header isn't actually here, so treat the chain as truncated rather than classifying it. if offset > len(packet) { - return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload + // A prior extension header can declare a length that advances offset past the packet, reject this. + return 0, 0, false, false, ErrIPv6CouldNotFindPayload } - return nextHeader, offset, isFragment, anyFragment, nil + return nextHeader, offset, isFragment, anyFragment, nil //success! } } - // Budget spent without reaching a terminal protocol. We don't know where the transport starts, so fail - // closed instead of handing back a mid-chain offset a caller would read ports from. - return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload + // Fail closed + return 0, 0, false, false, ErrIPv6CouldNotFindPayload } func CreateICMPEchoResponse(packet, out []byte) []byte { diff --git a/iputil/packet_test.go b/iputil/packet_test.go index 38c5da10..b88a7926 100644 --- a/iputil/packet_test.go +++ b/iputil/packet_test.go @@ -535,7 +535,7 @@ func Test_IPv6FindUpperProtocol(t *testing.T) { // report 4040546 shape: without failing closed the walk hands back offset 104, and a caller reads ports // from the attacker's bytes there. var budgetChain []byte - for i := 0; i < 7; i++ { + for range 7 { budgetChain = append(budgetChain, extToHbH...) } budgetChain = append(budgetChain, extToTCP...) @@ -560,9 +560,9 @@ func Test_IPv6FindUpperProtocol(t *testing.T) { {"first fragment walks to transport", 44, append(firstFragToUDP, transport...), 17, ipv6.HeaderLen + 8, false, true, nil}, {"non-first fragment stops", 44, append(nonFirstFrag, transport...), 17, ipv6.HeaderLen, true, true, nil}, {"unknown protocol is terminal", 132, transport, 132, ipv6.HeaderLen, false, false, nil}, // SCTP - {"truncated extension header", 0, nil, 0, ipv6.HeaderLen, false, false, ErrIPv6CouldNotFindPayload}, + {"truncated extension header", 0, nil, 0, 0, false, false, ErrIPv6CouldNotFindPayload}, // Destination Options with a declared length (255+1)*8 = 2048 that runs past the 48 byte buffer, next = SCTP - {"extension length past buffer", 60, []byte{132, 255, 0, 0, 0, 0, 0, 0}, 132, ipv6.HeaderLen + 2048, false, false, ErrIPv6CouldNotFindPayload}, + {"extension length past buffer", 0, []byte{132, 255, 0, 0, 0, 0, 0, 0}, 0, 0, false, false, ErrIPv6CouldNotFindPayload}, {"budget exhausted fails closed", 0, budgetChain, 0, 0, false, false, ErrIPv6CouldNotFindPayload}, } @@ -571,10 +571,10 @@ func Test_IPv6FindUpperProtocol(t *testing.T) { packet := makeIPv6Packet(src, dst, tt.nextHeader, tt.payload) proto, offset, isFragment, anyFragment, err := IPv6FindUpperProtocol(packet) if tt.wantErr != nil { - assert.ErrorIs(t, err, tt.wantErr) - return + require.ErrorIs(t, err, tt.wantErr) + } else { + require.NoError(t, err) } - require.NoError(t, err) assert.Equal(t, tt.wantProto, proto) assert.Equal(t, tt.wantOffset, offset) assert.Equal(t, tt.wantFragment, isFragment) diff --git a/outside.go b/outside.go index 8c3ed0a3..c28fe1ea 100644 --- a/outside.go +++ b/outside.go @@ -354,7 +354,7 @@ func parseV6(data []byte, incoming bool, fp *firewall.ParsedPacket) error { // and cannot drift into misreading an unknown protocol (SCTP, GRE, etc.) as a forged transport. proto, offset, isFragment, anyFragment, err := iputil.IPv6FindUpperProtocol(data) if err != nil { - return ErrIPv6PacketTooShort + return err } fp.Protocol = proto diff --git a/outside_test.go b/outside_test.go index 6847b887..cf76fc47 100644 --- a/outside_test.go +++ b/outside_test.go @@ -117,7 +117,7 @@ func Test_newPacket_v6(t *testing.T) { require.NoError(t, err) err = newPacket(buffer.Bytes(), true, p) - require.ErrorIs(t, err, ErrIPv6PacketTooShort) + require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload) // A v6 packet with a hop-by-hop extension // ICMPv6 Payload (Echo Request) @@ -151,12 +151,12 @@ func Test_newPacket_v6(t *testing.T) { // A full IPv6 header and 1 byte in the first extension, but missing // the length byte. err = newPacket(buffer.Bytes()[:41], true, p) - require.ErrorIs(t, err, ErrIPv6PacketTooShort) + require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload) // A full IPv6 header plus 1 full extension, but only 1 byte of the // next layer, missing length byte err = newPacket(buffer.Bytes()[:49], true, p) - require.ErrorIs(t, err, ErrIPv6PacketTooShort) + require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload) err = nil // A good ICMP packet @@ -354,12 +354,12 @@ func Test_newPacket_v6(t *testing.T) { // Ensure buffer bounds checking during processing, a truncated AH header can't reach the payload err = newPacket(b[:41], true, p) - require.ErrorIs(t, err, ErrIPv6PacketTooShort) + require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload) // Invalid AH header b = buffer.Bytes() err = newPacket(b, true, p) - require.ErrorIs(t, err, ErrIPv6PacketTooShort) + require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload) } func Test_newPacket_ipv6Fragment(t *testing.T) { @@ -466,7 +466,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) { // Too short of a fragment packet err = newPacket(secondFrag[:len(secondFrag)-10], false, p) - require.ErrorIs(t, err, ErrIPv6PacketTooShort) + require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload) } func BenchmarkParseV6(b *testing.B) { @@ -707,7 +707,7 @@ func Test_newPacket_v6ExtHeaderPastBuffer(t *testing.T) { pkt[40] = byte(layers.IPProtocolSCTP) // Dest Options next header = SCTP pkt[41] = 255 // declared length (255+1)*8 = 2048, past the 48 byte buffer - require.ErrorIs(t, newPacket(pkt, true, p), ErrIPv6PacketTooShort) + require.ErrorIs(t, newPacket(pkt, true, p), iputil.ErrIPv6CouldNotFindPayload) } // Test_newPacket_v6ExtHeaderConfusion is a regression test for parseV6 walking any unrecognized From d5062dacf0f8085ded1b7be9f8c0a57349baf71f Mon Sep 17 00:00:00 2001 From: Nate Brown Date: Wed, 30 Sep 2026 15:26:34 -0500 Subject: [PATCH 10/31] lighthouse: Never block in QueryServer, drop queries when the channel is full (#1910) --- connection_manager_test.go | 3 +++ examples/config.yml | 4 +-- lighthouse.go | 15 ++++++++--- lighthouse_test.go | 54 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 5 deletions(-) diff --git a/connection_manager_test.go b/connection_manager_test.go index 6be63f7a..68b4dc01 100644 --- a/connection_manager_test.go +++ b/connection_manager_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/rcrowley/go-metrics" "github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/overlay/overlaytest" @@ -21,6 +22,8 @@ func newTestLighthouse() *LightHouse { l: test.NewLogger(), addrMap: map[netip.Addr]*RemoteList{}, queryChan: make(chan netip.Addr, 10), + // Nothing drains queryChan here, so an 11th query is dropped. + queryDropped: metrics.NilCounter{}, } lighthouses := []netip.Addr{} staticList := map[netip.Addr]struct{}{} diff --git a/examples/config.yml b/examples/config.yml index 20a85883..0edb4dc6 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -410,8 +410,8 @@ logging: #try_interval: 100ms #retries: 20 - # query_buffer is the size of the buffer channel for querying lighthouses - #query_buffer: 64 + # query_buffer is the size of the buffer channel for querying lighthouses, drops are recorded in the lighthouse.query.dropped metric. + #query_buffer: 1024 # trigger_buffer is the size of the buffer channel for quickly sending handshakes # after receiving the response for lighthouse queries diff --git a/lighthouse.go b/lighthouse.go index 58037935..21937bcd 100644 --- a/lighthouse.go +++ b/lighthouse.go @@ -15,6 +15,7 @@ import ( "time" "github.com/gaissmai/bart" + "github.com/rcrowley/go-metrics" "github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/header" @@ -81,6 +82,8 @@ type LightHouse struct { updateTrigger chan struct{} queryChan chan netip.Addr + // queryDropped counts queries QueryServer dropped because queryChan was full. + queryDropped metrics.Counter calculatedRemotes atomic.Pointer[bart.Table[[]*calculatedRemote]] // Maps VpnAddr to []*calculatedRemote @@ -116,7 +119,8 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c nebulaPort: nebulaPort, punchy: p, updateTrigger: make(chan struct{}, 1), - queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)), + queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 1024)), + queryDropped: metrics.GetOrRegisterCounter("lighthouse.query.dropped", nil), l: l, } h.localAddrsFn = func(al *LocalAllowList) []netip.Addr { @@ -512,14 +516,19 @@ func (lh *LightHouse) Query(vpnAddr netip.Addr) *RemoteList { return nil } -// QueryServer is asynchronous so no reply should be expected +// QueryServer is asynchronous and non-blocking, no reply should be expected. +// If the query channel is full, the query is dropped. func (lh *LightHouse) QueryServer(vpnAddr netip.Addr) { // Don't put lighthouse addrs in the query channel because we can't query lighthouses about lighthouses if lh.amLighthouse || lh.IsLighthouseAddr(vpnAddr) { return } - lh.queryChan <- vpnAddr + select { + case lh.queryChan <- vpnAddr: + default: + lh.queryDropped.Inc(1) + } } func (lh *LightHouse) QueryCache(vpnAddrs []netip.Addr) *RemoteList { diff --git a/lighthouse_test.go b/lighthouse_test.go index 8fc30223..c39ffa16 100644 --- a/lighthouse_test.go +++ b/lighthouse_test.go @@ -6,7 +6,9 @@ import ( "errors" "fmt" "net/netip" + "sync" "testing" + "time" "github.com/gaissmai/bart" "github.com/slackhq/nebula/cert" @@ -795,3 +797,55 @@ func TestLightHouse_logLocalAddrsErr(t *testing.T) { lh.logLocalAddrsErr(errors.New("something else")) assert.Contains(t, out.String(), "level=WARN") } + +// heldEncWriter holds the query worker in its first GetHostInfo until release closes. +type heldEncWriter struct { + mockEncWriter + once sync.Once + held chan struct{} + release chan struct{} +} + +func (w *heldEncWriter) GetHostInfo(netip.Addr) *HostInfo { + w.once.Do(func() { close(w.held) }) + <-w.release + return nil +} + +// QueryServer never waits on the query worker, since the packet readers call it and nothing drains the channel at +// shutdown; a query that finds the channel full is dropped and counted. The worker is held on its first query so the +// channel fills. +func TestLighthouse_QueryServerDoesNotBlock(t *testing.T) { + l := test.NewLogger() + cs := testCertState(netip.MustParsePrefix("10.128.0.1/16")) + lh1 := "10.128.0.2" + c := config.NewC(l) + c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1}} + c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}} + c.Settings["handshakes"] = map[string]any{"query_buffer": 1} + lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil) + require.NoError(t, err) + w := &heldEncWriter{held: make(chan struct{}), release: make(chan struct{})} + lh.ifce = w + t.Cleanup(func() { close(w.release) }) + + lh.QueryServer(netip.MustParseAddr("10.128.0.10")) + select { + case <-w.held: + case <-time.After(5 * time.Second): + t.Fatal("the query worker never took the first query") + } + dropped := lh.queryDropped.Count() + done := make(chan struct{}) + go func() { + lh.QueryServer(netip.MustParseAddr("10.128.0.11")) + lh.QueryServer(netip.MustParseAddr("10.128.0.12")) + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("QueryServer blocked on a full query channel") + } + assert.EqualValues(t, 1, lh.queryDropped.Count()-dropped) +} From f9b08534764f3ac55f83e1055f118655d534c958 Mon Sep 17 00:00:00 2001 From: Matt Richardson Date: Wed, 7 Oct 2026 11:22:50 -0400 Subject: [PATCH 11/31] Attach a pending handshake's RemoteList when the handshake starts (#1909) StartHandshake returned a hostinfo with no RemoteList; handleOutbound attached one lazily on its first tick. Anything that touched the pending host before then, notably create-tunnel -address, dereferenced nil and took the daemon down. StartHandshake now attaches the lighthouse's list as it builds the hostinfo. To keep the surrounding locking sound, a lighthouse now resolves a punch target before taking the querier's RemoteList lock, and closeTunnel deletes the host and evicts its RemoteList under a single hostmap write lock so a completing handshake cannot land in between. The lock order, now documented on the handshake manager mutex, is hostmap, then handshake manager, then lighthouse, then RemoteList; no RemoteList lock is ever held across a hostmap lookup. --- CHANGELOG.md | 7 +++ connection_manager.go | 6 +-- handshake_manager.go | 21 ++++---- handshake_manager_test.go | 70 ++++++++++++++++++++++++- lighthouse.go | 5 +- lighthouse_test.go | 106 ++++++++++++++++++++++++++++++++++++++ outside.go | 13 +++-- ssh.go | 12 +++-- 8 files changed, 213 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed089185..4f27bc82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- `create-tunnel -address ` panicked on a nil RemoteList and, over the ssh debug server, took nebula + down with it. A hostinfo returned by StartHandshake had no RemoteList until the handshake loop's first + pass attached one, and `create-tunnel -address` set the remote before that. The list is now attached when + the handshake starts, so a pending hostinfo never exists without one. + ## [1.11.1] - 2026-08-21 See the [v1.11.1](https://github.com/slackhq/nebula/milestone/30?closed=1) milestone for a complete list of changes. diff --git a/connection_manager.go b/connection_manager.go index 83d2b0ca..4ec9f614 100644 --- a/connection_manager.go +++ b/connection_manager.go @@ -174,10 +174,8 @@ func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte switch decision { case deleteTunnel: - if cm.hostMap.DeleteHostInfo(hostinfo) { - // Only clearing the lighthouse cache if this is the last hostinfo for this vpn ip in the hostmap - cm.intf.lightHouse.DeleteVpnAddrs(hostinfo.vpnAddrs) - } + // Local delete only, clearing the lighthouse cache if this was the last hostinfo for the vpn addr + cm.intf.closeTunnel(hostinfo) case closeTunnel: cm.intf.sendCloseTunnel(hostinfo) diff --git a/handshake_manager.go b/handshake_manager.go index fa9ae154..ff80693c 100644 --- a/handshake_manager.go +++ b/handshake_manager.go @@ -54,7 +54,8 @@ type HandshakeConfig struct { } type HandshakeManager struct { - // Mutex for interacting with the vpnIps and indexes maps + // Mutex for interacting with the vpnIps and indexes maps. + // Lock order is hostmap, then this lock, then the lighthouse, then a RemoteList. sync.RWMutex vpnIps map[netip.Addr]*HandshakeHostInfo @@ -259,14 +260,6 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered "style": header.SubTypeName(header.Handshake, hh.machine.Subtype()), } - // Get a remotes object if we don't already have one. - // This is mainly to protect us as this should never be the case - // NB ^ This comment doesn't jive. It's how the thing gets initialized. - // It's the common path. Should it update every time, in case a future LH query/queries give us more info? - if hostinfo.remotes == nil { - hostinfo.remotes = hm.lightHouse.QueryCache([]netip.Addr{vpnIp}) - } - remotes := hostinfo.remotes.CopyAddrs(hm.mainHostMap.GetPreferredRanges()) remotesHaveChanged := !slices.Equal(remotes, hh.lastRemotes) @@ -371,6 +364,9 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han hostinfo := &HostInfo{ vpnAddrs: []netip.Addr{vpnAddr}, HandshakePacket: make(map[uint8][]byte, 0), + // Attached here so a pending hostinfo never exists without one: SetRemote dereferences + // it, and create-tunnel -address can reach SetRemote before the handshake loop runs. + remotes: hm.lightHouse.QueryCache([]netip.Addr{vpnAddr}), relayState: RelayState{ relays: nil, relayForByAddr: map[netip.Addr]*Relay{}, @@ -946,13 +942,14 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn hm.DeleteHostInfo(hostinfo) hm.StartHandshake(hostinfo.vpnAddrs[0], func(newHH *HandshakeHostInfo) { - newHH.hostinfo.remotes = hostinfo.remotes newHH.hostinfo.remotes.BlockRemote(via) newHH.packetStore = hh.packetStore hh.packetStore = []*cachedPacket{} - hostinfo.vpnAddrs = vpnAddrs - f.sendCloseTunnel(hostinfo) }) + // sendCloseTunnel may take the hostmap lock, so it must not run under the handshake + // manager lock that StartHandshake holds around the callback. + hostinfo.vpnAddrs = vpnAddrs + f.sendCloseTunnel(hostinfo) return } diff --git a/handshake_manager_test.go b/handshake_manager_test.go index 03483d87..acf13539 100644 --- a/handshake_manager_test.go +++ b/handshake_manager_test.go @@ -11,6 +11,7 @@ import ( "github.com/slackhq/nebula/test" "github.com/slackhq/nebula/udp" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_NewHandshakeManagerVpnIp(t *testing.T) { @@ -42,8 +43,6 @@ func Test_NewHandshakeManagerVpnIp(t *testing.T) { i2 := blah.StartHandshake(ip, nil) assert.Same(t, i, i2) - i.remotes = NewRemoteList([]netip.Addr{}, nil) - // Adding something to pending should not affect the main hostmap assert.Empty(t, mainHM.Hosts) @@ -235,3 +234,70 @@ func TestHandleIncomingDispatch(t *testing.T) { assert.Empty(t, hm.indexes, "orphan stage-2 must not create state") }) } + +// Test_PendingHandshakeHasRemotes covers create-tunnel -address: a pending hostinfo must already +// carry its RemoteList when StartHandshake runs its callback, so SetRemote there, before the +// handshake loop has run, is safe and the address becomes one the handshake is sent to. +func Test_PendingHandshakeHasRemotes(t *testing.T) { + l := test.NewLogger() + mainHM := newHostMap(l) + preferredRanges := []netip.Prefix{} + mainHM.preferredRanges.Store(&preferredRanges) + + hm := NewHandshakeManager(l, mainHM, newTestLighthouse(), &udp.NoopConn{}, defaultHandshakeConfig) + + vpnIp := netip.MustParseAddr("172.1.1.2") + remote := netip.MustParseAddrPort("192.168.1.9:4242") + + hi := hm.StartHandshake(vpnIp, func(hh *HandshakeHostInfo) { + require.NotNil(t, hh.hostinfo.remotes, "a pending handshake must never be without its RemoteList") + hh.hostinfo.SetRemote(remote) + }) + assert.Contains(t, hi.remotes.CopyAddrs(preferredRanges), remote, "the handshake must be sent to the supplied remote") + assert.Equal(t, remote, hi.GetRemote()) + + // A RemoteList keeps one learned address per host, so a second remote replaces the + // first — the same behaviour change-remote has on an established tunnel. + other := netip.MustParseAddrPort("192.168.1.10:4242") + hi.SetRemote(other) + assert.Equal(t, []netip.AddrPort{other}, hi.remotes.CopyAddrs(preferredRanges)) + assert.Equal(t, other, hi.GetRemote()) +} + +// newRemotesRaceFixture builds a handshake manager with an established tunnel to vpnAddr whose +// RemoteList is the lighthouse's cached list, the state every teardown-during-handshake race starts from. +func newRemotesRaceFixture(t *testing.T, vpnAddr netip.Addr) (*HandshakeManager, *Interface, *HostInfo) { + t.Helper() + l := test.NewLogger() + mainHM := newHostMap(l) + preferredRanges := []netip.Prefix{} + mainHM.preferredRanges.Store(&preferredRanges) + + lh := newTestLighthouse() + hm := NewHandshakeManager(l, mainHM, lh, &udp.NoopConn{}, defaultHandshakeConfig) + f := &Interface{hostMap: mainHM, lightHouse: lh, handshakeManager: hm, l: l} + hm.f = f + + established := &HostInfo{ + vpnAddrs: []netip.Addr{vpnAddr}, + localIndexId: 1, + remoteIndexId: 101, + remotes: lh.QueryCache([]netip.Addr{vpnAddr}), + HandshakePacket: map[uint8][]byte{}, + } + mainHM.Lock() + mainHM.unlockedAddHostInfo(established, f) + mainHM.Unlock() + + return hm, f, established +} + +// With no handshake pending, tearing down the last tunnel still evicts the list. +func Test_CloseTunnelEvictsWithoutPendingHandshake(t *testing.T) { + vpnAddr := netip.MustParseAddr("172.1.1.2") + _, f, established := newRemotesRaceFixture(t, vpnAddr) + + f.closeTunnel(established) + assert.NotContains(t, f.hostMap.Hosts, vpnAddr) + assert.NotContains(t, f.lightHouse.addrMap, vpnAddr) +} diff --git a/lighthouse.go b/lighthouse.go index 21937bcd..837a8df4 100644 --- a/lighthouse.go +++ b/lighthouse.go @@ -1244,10 +1244,13 @@ func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, fromVpnAddrs []neti // sendHostPunchNotification signals the other side to punch some zero byte udp packets func (lhh *LightHouseHandler) sendHostPunchNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, punchNotifDest netip.Addr, w EncWriter) { whereToPunch := fromVpnAddrs[0] + // Resolve the target before taking the querier's RemoteList lock below. GetHostInfo takes the + // hostmap lock, and the hostmap is taken before the lighthouse, which is taken before a + // RemoteList, so looking it up inside the callback would deadlock. + targetHI := lhh.lh.ifce.GetHostInfo(punchNotifDest) found, ln, err := lhh.lh.queryAndPrepMessage(whereToPunch, func(c *cache) (int, error) { n = lhh.resetMeta() n.Type = NebulaMeta_HostPunchNotification - targetHI := lhh.lh.ifce.GetHostInfo(punchNotifDest) var useVersion cert.Version if targetHI == nil { useVersion = lhh.lh.ifce.GetCertState().initiatingVersion diff --git a/lighthouse_test.go b/lighthouse_test.go index c39ffa16..bb588429 100644 --- a/lighthouse_test.go +++ b/lighthouse_test.go @@ -15,6 +15,7 @@ import ( "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/header" "github.com/slackhq/nebula/test" + "github.com/slackhq/nebula/udp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.yaml.in/yaml/v3" @@ -849,3 +850,108 @@ func TestLighthouse_QueryServerDoesNotBlock(t *testing.T) { } assert.EqualValues(t, 1, lh.queryDropped.Count()-dropped) } + +// hostmapEncWriter answers GetHostInfo from a real hostmap, the way Interface does. +type hostmapEncWriter struct { + mockEncWriter + hm *HostMap +} + +func (w *hostmapEncWriter) GetHostInfo(a netip.Addr) *HostInfo { return w.hm.QueryVpnAddr(a) } +func (w *hostmapEncWriter) GetCertState() *CertState { + return &CertState{initiatingVersion: cert.Version1} +} + +// A lighthouse runs four paths that each hold one lock while waiting for the next: CheckAndComplete +// holds the hostmap and takes the handshake manager, StartHandshake holds the handshake manager and +// takes the lighthouse, a host update holds the lighthouse and takes the sender's RemoteList, and a +// punch notification holds the querier's RemoteList while it resolves the target. If that last step +// reaches the hostmap under the RemoteList lock, the four deadlock. Drive them all at once. +func TestLighthouse_PunchNotificationDoesNotCloseLockCycle(t *testing.T) { + l := test.NewLogger() + myVpnNet := netip.MustParsePrefix("10.128.0.1/24") + + c := config.NewC(l) + c.Settings["lighthouse"] = map[string]any{"am_lighthouse": true} + c.Settings["listen"] = map[string]any{"port": 4242} + lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(myVpnNet), nil, nil) + require.NoError(t, err) + + mainHM := newHostMap(l) + pr := []netip.Prefix{} + mainHM.preferredRanges.Store(&pr) + lh.ifce = &hostmapEncWriter{hm: mainHM} + + hm := NewHandshakeManager(l, mainHM, lh, &udp.NoopConn{}, defaultHandshakeConfig) + f := &Interface{hostMap: mainHM, lightHouse: lh, handshakeManager: hm, l: l} + hm.f = f + + peer := netip.MustParseAddr("10.128.0.2") + peerUdp := netip.MustParseAddrPort("10.0.0.2:4242") + target := netip.MustParseAddr("10.128.0.3") + targetUdp := netip.MustParseAddrPort("10.0.0.3:4242") + outbound := netip.MustParseAddr("10.128.0.4") + inbound := netip.MustParseAddr("10.128.0.5") + + seed := lh.NewRequestHandler() + newLHHostUpdate(peerUdp, peer, []netip.AddrPort{peerUdp}, seed) + newLHHostUpdate(targetUdp, target, []netip.AddrPort{targetUdp}, seed) + + stop := make(chan struct{}) + var wg sync.WaitGroup + loop := func(body func()) { + defer wg.Done() + for { + select { + case <-stop: + return + default: + body() + } + } + } + wg.Add(4) + + // Outbound handshakes keep starting and being dropped. + go loop(func() { + hm.DeleteHostInfo(hm.StartHandshake(outbound, nil)) + }) + + // Inbound handshakes keep completing and being torn down. + var idx uint32 + go loop(func() { + idx++ + hi := &HostInfo{ + vpnAddrs: []netip.Addr{inbound}, + localIndexId: idx, + remoteIndexId: idx, + remotes: lh.QueryCache([]netip.Addr{inbound}), + HandshakePacket: map[uint8][]byte{handshakePacketStage0: {byte(idx)}}, + } + if _, err := hm.CheckAndComplete(hi, handshakePacketStage0, f); err == nil { + mainHM.DeleteHostInfo(hi) + } + }) + + // The peer keeps sending host updates. + updates := lh.NewRequestHandler() + go loop(func() { + newLHHostUpdate(peerUdp, peer, []netip.AddrPort{peerUdp}, updates) + }) + + // The same peer keeps querying for the target, which ends in a punch notification. + queries := lh.NewRequestHandler() + go loop(func() { + sendLHHostRequest(peerUdp, peer, target, queries, nil) + }) + + time.Sleep(time.Second) + close(stop) + done := make(chan struct{}) + go func() { wg.Wait(); close(done) }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("deadlock: the lighthouse paths never finished after stop was signalled") + } +} diff --git a/outside.go b/outside.go index c28fe1ea..12f2160c 100644 --- a/outside.go +++ b/outside.go @@ -263,11 +263,16 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender, // closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote func (f *Interface) closeTunnel(hostInfo *HostInfo) { - final := f.hostMap.DeleteHostInfo(hostInfo) - if final { - // We no longer have any tunnels with this vpn addr, clear learned lighthouse state to lower memory usage - f.lightHouse.DeleteVpnAddrs(hostInfo.vpnAddrs) + // Hold the hostmap lock across both the delete and the lighthouse eviction, + // to avoid a race with handshake completion. + f.hostMap.Lock() + defer f.hostMap.Unlock() + if !f.hostMap.unlockedDeleteHostInfo(hostInfo) { + return } + + // We no longer have any tunnels with this vpn addr, clear learned lighthouse state to lower memory usage + f.lightHouse.DeleteVpnAddrs(hostInfo.vpnAddrs) } // sendCloseTunnel is a helper function to send a proper close tunnel packet to a remote diff --git a/ssh.go b/ssh.go index 3863b5ec..2a440d98 100644 --- a/ssh.go +++ b/ssh.go @@ -684,10 +684,14 @@ func sshCreateTunnel(ifce *Interface, fs any, a []string, w sshd.StringWriter) e } } - hostInfo = ifce.handshakeManager.StartHandshake(vpnAddr, nil) - if addr.IsValid() { - hostInfo.SetRemote(addr) - } + // Set the remote from the callback, which StartHandshake runs under the handshake manager lock + // before the hostinfo reaches the handshake loop or the static-host trigger, so the address is + // in the RemoteList before the first handshake is sent. + ifce.handshakeManager.StartHandshake(vpnAddr, func(hh *HandshakeHostInfo) { + if addr.IsValid() { + hh.hostinfo.SetRemote(addr) + } + }) return w.WriteLine("Created") } From 8d8ae0f7443348bdf25bbf86d0738d5cdc679330 Mon Sep 17 00:00:00 2001 From: John Maguire Date: Wed, 7 Oct 2026 11:42:08 -0400 Subject: [PATCH 12/31] Disable DNS registration on the Windows tunnel adapter (#1914) On a domain-joined Windows host the DNS Client registers every adapter's addresses under the host name in AD DNS when "Register this connection's addresses in DNS" is on, which is the default for a new adapter. When the Nebula adapter registers, LAN clients that are not on the overlay resolve the host name to its overlay IP and RDP, SMB and PowerShell Remoting to it fail intermittently, with nothing pointing at the tunnel as the cause. Windows registers on address assignment, periodically, and on every ipconfig /registerdns. Right after the wintun adapter is created and before Activate assigns any address, write RegistrationEnabled=0, DisableDynamicUpdate=1 and MaxNumberOfAddressesToRegister=0 under both the Tcpip and Tcpip6 Parameters\Interfaces\{GUID} keys. The adapter checkbox alone has proven insufficient in practice, as has disabling dynamic updates alone, so all three are set, matching what Tailscale and NetBird do. A failure to write the keys is logged as a warning and does not fail adapter creation. --- overlay/dns_registration_windows.go | 57 +++++++++++++++++++++++++++++ overlay/tun_windows.go | 3 ++ 2 files changed, 60 insertions(+) create mode 100644 overlay/dns_registration_windows.go diff --git a/overlay/dns_registration_windows.go b/overlay/dns_registration_windows.go new file mode 100644 index 00000000..312d3057 --- /dev/null +++ b/overlay/dns_registration_windows.go @@ -0,0 +1,57 @@ +//go:build !e2e_testing + +package overlay + +import ( + "fmt" + "log/slog" + + "golang.org/x/sys/windows" + "golang.org/x/sys/windows/registry" +) + +var tcpipInterfaceKeys = []string{ + `SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces`, + `SYSTEM\CurrentControlSet\Services\Tcpip6\Parameters\Interfaces`, +} + +// disableDNSRegistration prevents publishing overlay addresses in the host's +// DNS records, where clients outside the overlay cannot reach them. +// Failures are non-fatal so registry access cannot prevent tunnel creation. +func disableDNSRegistration(l *slog.Logger, guid windows.GUID) { + id := guid.String() + for _, base := range tcpipInterfaceKeys { + path := base + `\` + id + if err := setDNSRegistrationValues(path); err != nil { + l.Warn("Failed to disable DNS registration on the tun adapter", "error", err, "key", path) + continue + } + l.Debug("Disabled DNS registration on the tun adapter", "key", path) + } +} + +func setDNSRegistrationValues(path string) error { + // The stack may not have created the adapter's interface key yet. + k, _, err := registry.CreateKey(registry.LOCAL_MACHINE, path, registry.SET_VALUE) + if err != nil { + return fmt.Errorf("open HKLM\\%s: %w", path, err) + } + defer k.Close() + + // Neither RegistrationEnabled nor DisableDynamicUpdate alone reliably + // prevents registration, so all three settings are required. + values := []struct { + name string + data uint32 + }{ + {"RegistrationEnabled", 0}, + {"DisableDynamicUpdate", 1}, + {"MaxNumberOfAddressesToRegister", 0}, + } + for _, v := range values { + if err := k.SetDWordValue(v.name, v.data); err != nil { + return fmt.Errorf("set %s: %w", v.name, err) + } + } + return nil +} diff --git a/overlay/tun_windows.go b/overlay/tun_windows.go index 4383481b..b8d49521 100644 --- a/overlay/tun_windows.go +++ b/overlay/tun_windows.go @@ -104,6 +104,9 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w } t.tun = tunDevice.(*wintun.NativeTun) + // Disable registration before Activate assigns addresses and triggers it. + disableDNSRegistration(l, *guid) + c.RegisterReloadCallback(func(c *config.C) { err := t.reload(c, false) if err != nil { From f57539a163ddb9b675033d84b3b59c5382ff7926 Mon Sep 17 00:00:00 2001 From: Nate Brown Date: Wed, 7 Oct 2026 10:43:11 -0500 Subject: [PATCH 13/31] overlay/linux: Keep unsafe routes when a reload only changes tun.mtu (#1918) --- overlay/tun_linux.go | 11 ++++++- overlay/tun_reload_linux_test.go | 56 ++++++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 overlay/tun_reload_linux_test.go diff --git a/overlay/tun_linux.go b/overlay/tun_linux.go index 358b1421..10904450 100644 --- a/overlay/tun_linux.go +++ b/overlay/tun_linux.go @@ -253,6 +253,15 @@ func (t *tun) reload(c *config.C, initial bool) error { return nil } + // Only the mtu changed. The routes are the same, parse them again so the ones without their own mtu pick up the + // new default. Without this an mtu change would remove them all + if !routeChange { + _, routes, err = getAllRoutesFromConfig(c, t.vpnNetworks, true) + if err != nil { + return err + } + } + routeTree, err := makeRouteTree(t.l, routes, true) if err != nil { return err @@ -274,7 +283,7 @@ func (t *tun) reload(c *config.C, initial bool) error { routes[i].MTU = newDefaultMTU } - if r.MTU > t.MaxMTU { + if r.MTU > newMaxMTU { newMaxMTU = r.MTU } } diff --git a/overlay/tun_reload_linux_test.go b/overlay/tun_reload_linux_test.go new file mode 100644 index 00000000..0fcf48f4 --- /dev/null +++ b/overlay/tun_reload_linux_test.go @@ -0,0 +1,56 @@ +//go:build !e2e_testing + +package overlay + +import ( + "net/netip" + "testing" + + "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/routing" + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Only tun.mtu changes. The system calls fail without a real device, what matters is the routes nebula keeps +func TestReloadMTUOnlyKeepsRoutes(t *testing.T) { + routes := " unsafe_routes:\n - route: 192.168.100.0/24\n via: 10.1.0.1\n" + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString("tun:\n mtu: 1300\n"+routes)) + + tn := &tun{ + l: test.NewLogger(), + vpnNetworks: []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, + ioctlFd: ^uintptr(0), + routesFromSystem: map[netip.Prefix]routing.Gateways{}, + } + require.NoError(t, tn.reload(c, true)) + + require.NoError(t, c.ReloadConfigString("tun:\n mtu: 1400\n"+routes)) + _ = tn.reload(c, false) + + got := *tn.Routes.Load() + require.Len(t, got, 1) + assert.Equal(t, netip.MustParsePrefix("192.168.100.0/24"), got[0].Cidr) + assert.Equal(t, 1400, got[0].MTU) + assert.Equal(t, 1400, tn.DefaultMTU) +} + +// The device mtu has to fit the largest route mtu, wherever it sits in the list +func TestReloadMaxMTUIsTheLargestRoute(t *testing.T) { + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString("tun:\n mtu: 1300\n unsafe_routes:\n"+ + " - route: 192.168.100.0/24\n via: 10.1.0.1\n mtu: 1500\n"+ + " - route: 192.168.101.0/24\n via: 10.1.0.1\n mtu: 1400\n")) + + tn := &tun{ + l: test.NewLogger(), + vpnNetworks: []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, + ioctlFd: ^uintptr(0), + routesFromSystem: map[netip.Prefix]routing.Gateways{}, + } + require.NoError(t, tn.reload(c, true)) + assert.Equal(t, 1500, tn.MaxMTU) + assert.Equal(t, 1300, tn.DefaultMTU) +} From d47c8c71f1fb58210f43c9f920f2e63f1ed49b4b Mon Sep 17 00:00:00 2001 From: Nate Brown Date: Wed, 7 Oct 2026 10:52:42 -0500 Subject: [PATCH 14/31] lighthouse: Send an update when the relays we advertise change on reload (#1916) --- lighthouse.go | 10 ++++++++++ lighthouse_test.go | 21 +++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/lighthouse.go b/lighthouse.go index 837a8df4..f3a6f737 100644 --- a/lighthouse.go +++ b/lighthouse.go @@ -360,6 +360,11 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error { } if initial || c.HasChanged("relay.relays") { + var before []netip.Addr + if !initial { + before = lh.GetRelaysForMe() + } + switch c.GetBool("relay.am_relay", false) { case true: // Relays aren't allowed to specify other relays @@ -384,6 +389,11 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error { } lh.relaysForMe.Store(&relaysForMe) } + + // Tell the lighthouses now rather than at the next interval, but only when what we advertise changed + if !initial && !slices.Equal(before, lh.GetRelaysForMe()) { + lh.TriggerUpdate() + } } return nil diff --git a/lighthouse_test.go b/lighthouse_test.go index bb588429..fab44cb9 100644 --- a/lighthouse_test.go +++ b/lighthouse_test.go @@ -955,3 +955,24 @@ func TestLighthouse_PunchNotificationDoesNotCloseLockCycle(t *testing.T) { t.Fatal("deadlock: the lighthouse paths never finished after stop was signalled") } } + +func TestLighthouse_reloadRelaysTriggersUpdate(t *testing.T) { + l := test.NewLogger() + c := config.NewC(l) + base := "lighthouse:\n am_lighthouse: true\nlisten:\n port: 4242\n" + require.NoError(t, c.LoadString(base+"relay:\n relays: [10.128.0.5]\n")) + + lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil) + require.NoError(t, err) + assert.Empty(t, lh.updateTrigger) + + require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6]\n")) + assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.6")}, lh.GetRelaysForMe()) + require.Len(t, lh.updateTrigger, 1) + <-lh.updateTrigger + + // The config changed but what we advertise did not, a bad entry is skipped + require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6, nope]\n")) + assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.6")}, lh.GetRelaysForMe()) + assert.Empty(t, lh.updateTrigger) +} From 5f26c1a7e5c9b8856133ae6be8a5ffc6f50d54c2 Mon Sep 17 00:00:00 2001 From: Nate Brown Date: Wed, 7 Oct 2026 10:53:37 -0500 Subject: [PATCH 15/31] Add a simple rate limiter with a helper to use it in logging (#1915) --- inside.go | 25 ++-- interface.go | 15 +++ logging/ratelimit.go | 31 +++++ logging/ratelimit_test.go | 106 +++++++++++++++ ratelimit/limiter.go | 91 +++++++++++++ ratelimit/limiter_test.go | 264 ++++++++++++++++++++++++++++++++++++++ 6 files changed, 523 insertions(+), 9 deletions(-) create mode 100644 logging/ratelimit.go create mode 100644 logging/ratelimit_test.go create mode 100644 ratelimit/limiter.go create mode 100644 ratelimit/limiter_test.go diff --git a/inside.go b/inside.go index 8eedbc49..4a0ac422 100644 --- a/inside.go +++ b/inside.go @@ -10,6 +10,7 @@ import ( "github.com/slackhq/nebula/firewall" "github.com/slackhq/nebula/header" "github.com/slackhq/nebula/iputil" + "github.com/slackhq/nebula/logging" "github.com/slackhq/nebula/noiseutil" "github.com/slackhq/nebula/overlay/batch" "github.com/slackhq/nebula/overlay/tio" @@ -517,7 +518,9 @@ func (f *Interface) SendVia(via *HostInfo, relay *Relay, ad, nb, out []byte, noc err = f.writers[q].WriteTo(toSend, via.GetRemote()) if err != nil { - via.logger(f.l).Info("Failed to WriteTo in sendVia", "error", err) + if l, ok := logging.RateLimited(f.l, f.udpRelayTxErrLimiter); ok { + via.logger(l).Info("Failed to WriteTo in sendVia", "error", err) + } } } @@ -582,18 +585,22 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType if remote.IsValid() { err = f.writers[q].WriteTo(out, remote) if err != nil { - hostinfo.logger(f.l).Error("Failed to write outgoing packet", - "error", err, - "udpAddr", remote, - ) + if l, ok := logging.RateLimited(f.l, f.udpTxErrLimiter); ok { + hostinfo.logger(l).Error("Failed to write outgoing packet", + "error", err, + "udpAddr", remote, + ) + } } } else if hr := hostinfo.GetRemote(); hr.IsValid() { err = f.writers[q].WriteTo(out, hr) if err != nil { - hostinfo.logger(f.l).Error("Failed to write outgoing packet", - "error", err, - "udpAddr", hr, - ) + if l, ok := logging.RateLimited(f.l, f.udpTxErrLimiter); ok { + hostinfo.logger(l).Error("Failed to write outgoing packet", + "error", err, + "udpAddr", hr, + ) + } } } else { // Try to send via a relay diff --git a/interface.go b/interface.go index 1e0e3db8..2b9ff5eb 100644 --- a/interface.go +++ b/interface.go @@ -24,6 +24,7 @@ import ( "github.com/slackhq/nebula/overlay" "github.com/slackhq/nebula/overlay/batch" "github.com/slackhq/nebula/overlay/tio" + "github.com/slackhq/nebula/ratelimit" "github.com/slackhq/nebula/udp" ) @@ -124,6 +125,9 @@ type Interface struct { batchers []*batch.MultiCoalescer wg sync.WaitGroup + udpTxErrLimiter *ratelimit.Limiter // udp direct transmit error log limiter + udpRelayTxErrLimiter *ratelimit.Limiter // udp relay transmit error log limiter + // fatalErr holds the first unexpected reader error that caused shutdown. // nil means "no fatal error" (yet) fatalErr atomic.Pointer[error] @@ -198,6 +202,15 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) { return nil, errors.New("no connection manager") } + udpTxErrLimiter, err := ratelimit.New(1, time.Second) + if err != nil { + return nil, err + } + udpRelayTxErrLimiter, err := ratelimit.New(1, time.Second) + if err != nil { + return nil, err + } + if c.routines <= 1 { c.PinThreads = false //pinning is not useful unless there's more than one tun reader } @@ -230,6 +243,8 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) { conntrackCacheTimeout: c.ConntrackCacheTimeout, cpuAffinity: c.CpuAffinity, pinThreads: c.PinThreads, + udpTxErrLimiter: udpTxErrLimiter, + udpRelayTxErrLimiter: udpRelayTxErrLimiter, metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)), metricTxDropped: metrics.GetOrRegisterCounter("udp.tx.dropped", nil), diff --git a/logging/ratelimit.go b/logging/ratelimit.go new file mode 100644 index 00000000..ad8e9c14 --- /dev/null +++ b/logging/ratelimit.go @@ -0,0 +1,31 @@ +package logging + +import "log/slog" + +// RateLimiter is what RateLimited needs from a rate limiter, such as a *ratelimit.Limiter. +// Allow reports whether this call may proceed, and how many calls were denied since the last one that did. +type RateLimiter interface { + Allow() (denied uint64, ok bool) +} + +// RateLimited checks lim before anything is built. +// When allowed it returns l, with a "suppressed" count attached if lim denied calls since its last allowed one. +// A nil lim allows everything. +// +// if l, ok := logging.RateLimited(f.l, f.udpTxErrLimiter); ok { +// hostinfo.logger(l).Error("Failed to write outgoing packet", "error", err) +// } +func RateLimited(l *slog.Logger, lim RateLimiter) (*slog.Logger, bool) { + if lim == nil { + return l, true + } + + denied, ok := lim.Allow() + if !ok { + return nil, false + } + if denied > 0 { + return l.With("suppressed", denied), true + } + return l, true +} diff --git a/logging/ratelimit_test.go b/logging/ratelimit_test.go new file mode 100644 index 00000000..b4fed23b --- /dev/null +++ b/logging/ratelimit_test.go @@ -0,0 +1,106 @@ +package logging + +import ( + "bytes" + "errors" + "io" + "log/slog" + "net/netip" + "strings" + "testing" + "time" + + "github.com/slackhq/nebula/ratelimit" +) + +type fakeLimiter struct { + denied uint64 + ok bool +} + +func (f fakeLimiter) Allow() (uint64, bool) { return f.denied, f.ok } + +func TestRateLimited(t *testing.T) { + var buf bytes.Buffer + base := NewLogger(&buf) + + if l, ok := RateLimited(base, fakeLimiter{ok: false}); ok || l != nil { + t.Fatalf("denied call: ok=%v logger=%v", ok, l) + } + + if l, ok := RateLimited(base, fakeLimiter{ok: true}); !ok || l != base { + t.Fatal("nothing denied should hand back the same logger") + } + + if l, ok := RateLimited(base, nil); !ok || l != base { + t.Fatal("a nil limiter should allow with the same logger") + } + + l, ok := RateLimited(base, fakeLimiter{denied: 7, ok: true}) + if !ok { + t.Fatal("allowed call denied") + } + l.Info("hello", "k", "v") + if !strings.Contains(buf.String(), "suppressed=7") { + t.Fatalf("want suppressed=7, got %q", buf.String()) + } +} + +// The benchmarks compare a log line through slog directly against the same line behind RateLimited: logged with +// nothing denied, denied, and logged after denials. + +var ( + benchErr = errors.New("sendto: no buffer space available") + benchAddr = netip.MustParseAddrPort("192.0.2.1:4242") +) + +func benchLogger() *slog.Logger { + return NewLogger(io.Discard).With("localIndex", 1234, "remoteIndex", 5678) +} + +func BenchmarkRateLimitedBaseline(b *testing.B) { + lg := benchLogger() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + lg.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr) + } +} + +func BenchmarkRateLimitedAllowed(b *testing.B) { + lg, lim := benchLogger(), mustLimiter(b, 1, 0) + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if l, ok := RateLimited(lg, lim); ok { + l.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr) + } + } +} + +func BenchmarkRateLimitedDenied(b *testing.B) { + lg, lim := benchLogger(), mustLimiter(b, 1, time.Hour) + lim.Allow() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if l, ok := RateLimited(lg, lim); ok { + l.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr) + } + } +} + +func BenchmarkRateLimitedAfterDenied(b *testing.B) { + lg, lim := benchLogger(), fakeLimiter{denied: 5, ok: true} + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if l, ok := RateLimited(lg, lim); ok { + l.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr) + } + } +} + +func mustLimiter(b *testing.B, burst int, every time.Duration) *ratelimit.Limiter { + l, err := ratelimit.New(burst, every) + if err != nil { + b.Fatal(err) + } + return l +} diff --git a/ratelimit/limiter.go b/ratelimit/limiter.go new file mode 100644 index 00000000..ae8480f0 --- /dev/null +++ b/ratelimit/limiter.go @@ -0,0 +1,91 @@ +package ratelimit + +import ( + "fmt" + "math" + "sync/atomic" + "time" +) + +// Limiter is a token bucket: `burst` calls are allowed at once, then one more per `every`. +// It counts the calls it denies and hands that count to the next allowed call. +// +// The budget belongs to the Limiter instance. +// Every Allow that returns true spends from it, whatever the caller then does. +// +// Time follows Go's monotonic clock. +// +// Create one Limiter for each thing you want to limit, +// sharing one between unrelated events lets the noisy one starve the quiet one. +// +// Check Allow before doing the work being limited, so a denied call stays cheap: +// +// if denied, ok := probeLimiter.Allow(); ok { +// sendProbe() +// probesDenied.Inc(int64(denied)) +// } +// +// For log lines use logging.RateLimited, which attaches the denied count to the logger as "suppressed". +type Limiter struct { + every int64 + slack int64 + tat atomic.Int64 + denied atomic.Uint64 +} + +// maxSpan bounds `burst` * `every` to half of int64, leaving the other half for process uptime so the bucket +// math can't overflow. +const maxSpan = math.MaxInt64 / 2 + +// New returns a Limiter allowing `burst` calls at once and one more per `every`. +// A `burst` under 1 is 1, and an `every` of 0 or less never limits. +// It returns an error if `burst` * `every` is over about 146 years. +func New(burst int, every time.Duration) (*Limiter, error) { + l := &Limiter{} + if every > 0 { + burst = max(burst, 1) + if int64(burst) > maxSpan/int64(every) { + return nil, fmt.Errorf("ratelimit: burst %d * every %v is too long", burst, every) + } + l.every = int64(every) + l.slack = int64(burst-1) * l.every + } + return l, nil +} + +var monoStart = time.Now() + +func monoNow() int64 { return int64(time.Since(monoStart)) } + +// Allow reports whether this call is allowed. +// If a token is available it spends it and returns the calls denied since the last allowed one, and true. +// Otherwise it returns 0 and false, and counts the denial. +// A nil Limiter allows everything. +func (l *Limiter) Allow() (uint64, bool) { + if l == nil { + return 0, true + } + return l.allow(monoNow()) +} + +// allow is Allow at a given monotonic time. +func (l *Limiter) allow(now int64) (uint64, bool) { + // Concurrent Allow calls read the clock before racing for the bucket, so one can see a time another already spent + // past and be denied. An unlimited Limiter must never deny, so it skips the bucket. + if l.every == 0 { + return 0, true + } + + for { + tat := l.tat.Load() + next := max(tat, now) + if next-now > l.slack { + l.denied.Add(1) + return 0, false + } + + if l.tat.CompareAndSwap(tat, next+l.every) { + return l.denied.Swap(0), true + } + } +} diff --git a/ratelimit/limiter_test.go b/ratelimit/limiter_test.go new file mode 100644 index 00000000..897430aa --- /dev/null +++ b/ratelimit/limiter_test.go @@ -0,0 +1,264 @@ +package ratelimit + +import ( + "math" + "sync" + "sync/atomic" + "testing" + "time" +) + +func mustLimiter(t testing.TB, burst int, every time.Duration) *Limiter { + t.Helper() + l, err := New(burst, every) + if err != nil { + t.Fatal(err) + } + return l +} + +func TestLimiterBurstAndRefill(t *testing.T) { + l, now := mustLimiter(t, 3, time.Second), int64(0) + + for i := 0; i < 3; i++ { + if n, ok := l.allow(now); !ok || n != 0 { + t.Fatalf("burst line %d: ok=%v denied=%d", i, ok, n) + } + } + for i := 0; i < 5; i++ { + if _, ok := l.allow(now); ok { + t.Fatalf("line %d past the burst logged", i) + } + } + + now += int64(time.Second) + if n, ok := l.allow(now); !ok || n != 5 { + t.Fatalf("after one refill: ok=%v denied=%d, want true 5", ok, n) + } + if _, ok := l.allow(now); ok { + t.Fatal("one refill earned more than one line") + } + + // A long quiet spell refills to burst, not past it, and the count reported above does not carry over. + now += int64(time.Hour) + for i, want := range []uint64{1, 0, 0} { + if n, ok := l.allow(now); !ok || n != want { + t.Fatalf("refilled line %d: ok=%v denied=%d, want true %d", i, ok, n, want) + } + } + if _, ok := l.allow(now); ok { + t.Fatal("bucket refilled past burst") + } +} + +func TestLimiterBounds(t *testing.T) { + for _, every := range []time.Duration{0, -time.Second} { + l := mustLimiter(t, 1, every) + for i := 0; i < 100; i++ { + if _, ok := l.allow(0); !ok { + t.Fatalf("every=%v limited line %d", every, i) + } + } + } + + l := mustLimiter(t, 0, time.Second) + if _, ok := l.allow(0); !ok { + t.Fatal("burst 0 should allow one line") + } + if _, ok := l.allow(0); ok { + t.Fatal("burst 0 should allow only one line") + } +} + +func TestLimiterOverflow(t *testing.T) { + for _, c := range []struct { + burst int + every time.Duration + }{ + {math.MaxInt, time.Second}, + {1, math.MaxInt64}, + {2, maxSpan/2 + 1}, + } { + if _, err := New(c.burst, c.every); err == nil { + t.Fatalf("burst %d every %d should not fit", c.burst, c.every) + } + } + + l := mustLimiter(t, 2, maxSpan/2) + for i := 0; i < 2; i++ { + if _, ok := l.allow(0); !ok { + t.Fatalf("the largest span should allow its burst, denied call %d", i) + } + } + if _, ok := l.allow(0); ok { + t.Fatal("the largest span allowed past its burst") + } +} + +// Concurrent callers read the clock at slightly different times, which must not limit a Limiter that never limits. +func TestLimiterUnlimitedConcurrent(t *testing.T) { + l := mustLimiter(t, 1, 0) + + var denied atomic.Int64 + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 10000; j++ { + if _, ok := l.Allow(); !ok { + denied.Add(1) + } + } + }() + } + wg.Wait() + + if got := denied.Load(); got != 0 { + t.Fatalf("denied %d lines", got) + } +} + +func TestLimiterNil(t *testing.T) { + var l *Limiter + for i := 0; i < 10; i++ { + if _, ok := l.Allow(); !ok { + t.Fatal("nil limiter limited") + } + } +} + +func TestLimiterConcurrent(t *testing.T) { + l := mustLimiter(t, 10, time.Hour) + + var allowed atomic.Int64 + var reported atomic.Uint64 + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 1000; j++ { + if n, ok := l.allow(0); ok { + allowed.Add(1) + reported.Add(n) + } + } + }() + } + wg.Wait() + + if got := allowed.Load(); got != 10 { + t.Fatalf("allowed %d lines, want the burst of 10", got) + } + // An allowed call that lands after some denials reports them, so the rest are still pending. + if got := reported.Load() + l.denied.Load(); got != 16*1000-10 { + t.Fatalf("denied %d, want %d", got, 16*1000-10) + } +} + +func BenchmarkAllowAllowed(b *testing.B) { + l := mustLimiter(b, 1, time.Nanosecond) + b.ReportAllocs() + for i := 0; i < b.N; i++ { + l.tat.Store(0) + l.Allow() + } +} + +func BenchmarkAllowDenied(b *testing.B) { + l := mustLimiter(b, 1, time.Hour) + l.Allow() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + l.Allow() + } +} + +func BenchmarkAllowDeniedParallel(b *testing.B) { + l := mustLimiter(b, 1, time.Hour) + l.Allow() + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + l.Allow() + } + }) +} + +// FuzzLimiter checks the bucket against its exact definition over arbitrary call timings. +// A call is denied if and only if some allowed call a would leave too many calls for the time since a: +// (allowed calls since a, plus this one) * every > burst * every + (now - a). +// Every denied call must also be reported exactly once, by a later allowed call or still pending. +func FuzzLimiter(f *testing.F) { + f.Add(uint8(3), uint16(1000), []byte{0, 0, 0, 0, 16, 0, 0, 200, 1, 1, 1, 15, 17, 0}) + f.Add(uint8(1), uint16(1), []byte{0, 16, 15, 16, 0, 255, 0, 0}) + f.Fuzz(func(t *testing.T, burst uint8, everyUs uint16, steps []byte) { + if len(steps) > 512 { + steps = steps[:512] + } + b := int64(burst%16) + 1 + every := int64(everyUs%1000+1) * int64(time.Microsecond) + l := mustLimiter(t, int(b), time.Duration(every)) + + var now int64 + var allowedAt []int64 + var denied, reported uint64 + for _, s := range steps { + // Each step advances by a sixteenth of every, so timings land on and around the refill. + now += int64(s) * every / 16 + + over := false + for i, a := range allowedAt { + if int64(len(allowedAt)-i+1)*every > b*every+now-a { + over = true + break + } + } + + n, ok := l.allow(now) + if ok == over { + t.Fatalf("at %d with %d allowed before: ok=%v, the definition says %v", now, len(allowedAt), ok, !over) + } + if !ok { + denied++ + continue + } + reported += n + allowedAt = append(allowedAt, now) + } + + if got := reported + l.denied.Load(); got != denied { + t.Fatalf("reported %d denied calls, made %d", got, denied) + } + }) +} + +// Concurrent callers on the real clock must account for every denied call exactly once. +func TestLimiterDeniedConservedConcurrent(t *testing.T) { + l := mustLimiter(t, 4, time.Microsecond) + + var denied, reported atomic.Uint64 + var wg sync.WaitGroup + for i := 0; i < 16; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < 20000; j++ { + if n, ok := l.Allow(); ok { + reported.Add(n) + } else { + denied.Add(1) + } + } + }() + } + wg.Wait() + + if denied.Load() == 0 { + t.Fatal("nothing was denied, the test proves nothing") + } + if got := reported.Load() + l.denied.Load(); got != denied.Load() { + t.Fatalf("reported %d denied calls, made %d", got, denied.Load()) + } +} From 94be454f4f1bb1c4dd16df7438b7dca54aa99dee Mon Sep 17 00:00:00 2001 From: Nate Brown Date: Wed, 7 Oct 2026 14:06:27 -0500 Subject: [PATCH 16/31] lighthouse: Recompute relays for me when relay.am_relay changes (#1917) --- lighthouse.go | 3 ++- lighthouse_test.go | 21 +++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/lighthouse.go b/lighthouse.go index f3a6f737..619c16a0 100644 --- a/lighthouse.go +++ b/lighthouse.go @@ -359,7 +359,8 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error { } } - if initial || c.HasChanged("relay.relays") { + // am_relay decides whether relay.relays applies to us at all + if initial || c.HasChanged("relay.relays") || c.HasChanged("relay.am_relay") { var before []netip.Addr if !initial { before = lh.GetRelaysForMe() diff --git a/lighthouse_test.go b/lighthouse_test.go index fab44cb9..ace29c15 100644 --- a/lighthouse_test.go +++ b/lighthouse_test.go @@ -976,3 +976,24 @@ func TestLighthouse_reloadRelaysTriggersUpdate(t *testing.T) { assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.6")}, lh.GetRelaysForMe()) assert.Empty(t, lh.updateTrigger) } + +func TestLighthouse_reloadAmRelayAlone(t *testing.T) { + l := test.NewLogger() + c := config.NewC(l) + base := "lighthouse:\n am_lighthouse: true\nlisten:\n port: 4242\nrelay:\n relays: [10.128.0.5]\n" + require.NoError(t, c.LoadString(base+" am_relay: false\n")) + + lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil) + require.NoError(t, err) + assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.5")}, lh.GetRelaysForMe()) + + // Relays aren't allowed to have relays, flipping am_relay alone must drop them and tell the lighthouses + require.NoError(t, c.ReloadConfigString(base+" am_relay: true\n")) + assert.Empty(t, lh.GetRelaysForMe()) + require.Len(t, lh.updateTrigger, 1) + <-lh.updateTrigger + + require.NoError(t, c.ReloadConfigString(base+" am_relay: false\n")) + assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.5")}, lh.GetRelaysForMe()) + require.Len(t, lh.updateTrigger, 1) +} From f3705cbb671d694925149abd23613b7ffddc250e Mon Sep 17 00:00:00 2001 From: Nate Brown Date: Wed, 7 Oct 2026 18:01:09 -0500 Subject: [PATCH 17/31] graphite: Time out a send, and abandon it on stop (#1920) --- graphite.go | 23 +++- graphite_test.go | 315 +++++++++++++++++++++++++++++++++++++++++++++++ stats.go | 7 +- stats_test.go | 35 +----- 4 files changed, 339 insertions(+), 41 deletions(-) create mode 100644 graphite_test.go diff --git a/graphite.go b/graphite.go index 264bb106..3ee4ed21 100644 --- a/graphite.go +++ b/graphite.go @@ -31,6 +31,7 @@ package nebula import ( "bufio" + "context" "fmt" "net" "strconv" @@ -51,17 +52,29 @@ type graphiteConfigExport struct { Percentiles []float64 // Percentiles to export from timers and histograms } +// graphiteTimeout bounds one submission, the connect and the write together. An export is a few KB, a host that can't +// take that in this long isn't going to. +const graphiteTimeout = 30 * time.Second + // graphiteOnce performs a single submission to Graphite, returning a non-nil -// error on failed connections. -func graphiteOnce(c graphiteConfigExport) error { +// error if it could not connect or write all of it within timeout. It gives up +// as soon as ctx is done. +func graphiteOnce(ctx context.Context, c graphiteConfigExport, timeout time.Duration) error { now := time.Now().Unix() du := float64(c.DurationUnit) flushSeconds := float64(c.FlushInterval) / float64(time.Second) - conn, err := net.DialTCP("tcp", nil, c.Addr) + deadline := time.Now().Add(timeout) + d := net.Dialer{Deadline: deadline} + conn, err := d.DialContext(ctx, "tcp", c.Addr.String()) if err != nil { return err } defer conn.Close() + if err := conn.SetDeadline(deadline); err != nil { + return err + } + stop := context.AfterFunc(ctx, func() { _ = conn.Close() }) + defer stop() w := bufio.NewWriter(conn) c.Registry.Each(func(name string, i any) { switch metric := i.(type) { @@ -111,7 +124,7 @@ func graphiteOnce(c graphiteConfigExport) error { fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, t.Rate15(), now) fmt.Fprintf(w, "%s.%s.mean-rate %.2f %d\n", c.Prefix, name, t.RateMean(), now) } - w.Flush() }) - return nil + // A failed write sticks, so this reports the first one + return w.Flush() } diff --git a/graphite_test.go b/graphite_test.go new file mode 100644 index 00000000..0a951da5 --- /dev/null +++ b/graphite_test.go @@ -0,0 +1,315 @@ +package nebula + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net" + "strings" + "sync" + "testing" + "time" + + "github.com/rcrowley/go-metrics" + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// graphiteHost accepts connections and hands each one to the test +type graphiteHost struct { + ln net.Listener + conns chan net.Conn + + mu sync.Mutex + closed bool + open []net.Conn +} + +// newGraphiteHost accepts connections for the test. A non-zero window shrinks each connection's receive buffer, so a +// sender that is not read from blocks after a little whatever the host's TCP tuning. Only for tests that need a stall, +// a small window makes reading very slow on Linux. +func newGraphiteHost(t *testing.T, window int) *graphiteHost { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + h := &graphiteHost{ln: ln, conns: make(chan net.Conn, 8)} + done := make(chan struct{}) + t.Cleanup(func() { + _ = ln.Close() + h.mu.Lock() + h.closed = true + for _, c := range h.open { + _ = c.Close() + } + h.mu.Unlock() + close(done) + }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + h.mu.Lock() + if h.closed { + h.mu.Unlock() + _ = c.Close() + return + } + h.open = append(h.open, c) + h.mu.Unlock() + if tc, ok := c.(*net.TCPConn); ok && window > 0 { + _ = tc.SetReadBuffer(window) + } + select { + case h.conns <- c: + case <-done: + return + } + } + }() + return h +} + +func (h *graphiteHost) addr(t *testing.T) *net.TCPAddr { + a, err := net.ResolveTCPAddr("tcp", h.ln.Addr().String()) + require.NoError(t, err) + return a +} + +// readExport reads what one send wrote, bounded so a send that never finishes fails the test rather than hanging it +func readExport(t *testing.T, c net.Conn) []byte { + t.Helper() + require.NoError(t, c.SetReadDeadline(time.Now().Add(10*time.Second))) + b, err := io.ReadAll(c) + require.NoError(t, err) + return b +} + +func (h *graphiteHost) next(t *testing.T, wait time.Duration) net.Conn { + t.Helper() + select { + case c := <-h.conns: + return c + case <-time.After(wait): + t.Fatal("no connection") + return nil + } +} + +// lockedBuffer is a log destination a test can read while the code under test writes to it from another goroutine +type lockedBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func testGraphiteConfig(addr *net.TCPAddr, r metrics.Registry) graphiteConfigExport { + return graphiteConfigExport{Addr: addr, Registry: r, FlushInterval: time.Second, DurationUnit: time.Nanosecond, Prefix: "nebula"} +} + +// Enough metrics that one export overflows the socket buffers on any platform, so a send to a host that stops +// reading blocks +func bigGraphiteRegistry(r metrics.Registry, t *testing.T) { + for i := range 200000 { + name := fmt.Sprintf("big.gauge.%d", i) + metrics.GetOrRegisterGauge(name, r).Update(int64(i)) + t.Cleanup(func() { r.Unregister(name) }) + } +} + +func TestGraphiteOnce_sends(t *testing.T) { + r := metrics.NewRegistry() + metrics.GetOrRegisterGauge("g", r).Update(7) + metrics.GetOrRegisterCounter("c", r).Inc(3) + h := newGraphiteHost(t, 0) + + require.NoError(t, graphiteOnce(t.Context(), testGraphiteConfig(h.addr(t), r), time.Minute)) + b := readExport(t, h.next(t, 5*time.Second)) + assert.Contains(t, string(b), "nebula.g.value 7 ") + assert.Contains(t, string(b), "nebula.c.count 3 ") + assert.Contains(t, string(b), "nebula.c.count_ps 3.00 ") +} + +// A host that accepts and never reads is given up on, and the failed write is reported +func TestGraphiteOnce_stalledHostTimesOut(t *testing.T) { + r := metrics.NewRegistry() + bigGraphiteRegistry(r, t) + h := newGraphiteHost(t, 4096) + done := make(chan error, 1) + cfg := testGraphiteConfig(h.addr(t), r) + go func() { done <- graphiteOnce(t.Context(), cfg, 300*time.Millisecond) }() + h.next(t, 5*time.Second) // accepted, never read + + select { + case err := <-done: + require.Error(t, err) + case <-time.After(10 * time.Second): + t.Fatal("the send did not give up on a host that stopped reading") + } +} + +// The timeout covers the whole send, a host that keeps reading but too slowly is still given up on +func TestGraphiteOnce_timeoutCoversTheWholeSend(t *testing.T) { + r := metrics.NewRegistry() + bigGraphiteRegistry(r, t) + h := newGraphiteHost(t, 0) + done := make(chan error, 1) + start := time.Now() + cfg := testGraphiteConfig(h.addr(t), r) + go func() { done <- graphiteOnce(t.Context(), cfg, 500*time.Millisecond) }() + c := h.next(t, 5*time.Second) + stopReading := make(chan struct{}) + defer close(stopReading) + // 32KB every 10ms keeps every write well inside the timeout, the whole export takes seconds + go func() { + buf := make([]byte, 32<<10) + for { + select { + case <-stopReading: + return + case <-time.After(10 * time.Millisecond): + } + if _, err := c.Read(buf); err != nil { + return + } + } + }() + + select { + case err := <-done: + require.Error(t, err) + assert.Less(t, time.Since(start), 5*time.Second) + case <-time.After(10 * time.Second): + t.Fatal("a host that kept reading slowly held the send past its timeout") + } +} + +func TestGraphiteOnce_cancelledCtxDoesNotConnect(t *testing.T) { + h := newGraphiteHost(t, 0) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + require.Error(t, graphiteOnce(ctx, testGraphiteConfig(h.addr(t), metrics.NewRegistry()), time.Minute)) + select { + case <-h.conns: + t.Fatal("connected on a cancelled ctx") + case <-time.After(200 * time.Millisecond): + } +} + +// A stop or reload abandons a stuck send rather than waiting out its timeout +func TestGraphiteOnce_cancelAbandonsSend(t *testing.T) { + r := metrics.NewRegistry() + bigGraphiteRegistry(r, t) + h := newGraphiteHost(t, 4096) + ctx, cancel := context.WithCancel(t.Context()) + done := make(chan error, 1) + cfg := testGraphiteConfig(h.addr(t), r) + go func() { done <- graphiteOnce(ctx, cfg, time.Minute) }() + h.next(t, 5*time.Second) + select { + case err := <-done: + t.Fatalf("the send should be stuck on a host that is not reading, it ended with %v", err) + case <-time.After(200 * time.Millisecond): + } + + cancel() + select { + case err := <-done: + require.Error(t, err) + case <-time.After(5 * time.Second): + t.Fatal("the send is still stuck after its ctx was cancelled") + } +} + +// buildRuntime hands graphite the configured prefix and interval +func TestStatsServer_graphiteExportUsesConfig(t *testing.T) { + const name = "graphite.wiring.test" + metrics.GetOrRegisterCounter(name, nil).Inc(8) + t.Cleanup(func() { metrics.DefaultRegistry.Unregister(name) }) + h := newGraphiteHost(t, 0) + s, c := newTestStatsServer(t) + setStatsConfig(c, map[string]any{"type": "graphite", "interval": "4s", "prefix": "pfx", "host": h.ln.Addr().String()}) + cfg, err := loadStatsConfig(c) + require.NoError(t, err) + + s.runMu.Lock() + fns, _ := s.buildRuntime(t.Context(), cfg) + s.runMu.Unlock() + for _, fn := range fns { + fn() + } + b := readExport(t, h.next(t, 5*time.Second)) + assert.Contains(t, string(b), "pfx."+name+".count 8 ") + assert.Contains(t, string(b), "pfx."+name+".count_ps 2.00 ") +} + +func TestStatsServer_graphiteFailureIsLogged(t *testing.T) { + logs := &lockedBuffer{} + s, c := newTestStatsServer(t) + s.l = test.NewLoggerWithOutput(logs) + setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": "127.0.0.1:" + freeTCPPort(t)}) + cfg, err := loadStatsConfig(c) + require.NoError(t, err) + + s.runMu.Lock() + fns, _ := s.buildRuntime(t.Context(), cfg) + s.runMu.Unlock() + for _, fn := range fns { + fn() + } + assert.Contains(t, logs.String(), "Graphite export failed") +} + +// Stop abandons the running export, so it goes through Start rather than handing buildRuntime a ctx, and a stop is +// not logged as a failure +func TestStatsServer_stopAbandonsGraphiteSend(t *testing.T) { + bigGraphiteRegistry(metrics.DefaultRegistry, t) + h := newGraphiteHost(t, 4096) + logs := &lockedBuffer{} + s, c := newTestStatsServer(t) + s.l = test.NewLoggerWithOutput(logs) + setStatsConfig(c, map[string]any{"type": "graphite", "interval": "50ms", "host": h.ln.Addr().String()}) + require.NoError(t, s.reload(c, true)) + + started := make(chan struct{}) + go func() { + s.Start() + close(started) + }() + stuck := h.next(t, 30*time.Second) + s.Stop() + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("Start did not return after Stop") + } + + // A closed sender resets the connection once it sees this, one still sending just buffers it. Either way the + // read below doesn't have to drain the export through the small window to find out + _, err := stuck.Write([]byte("x")) + require.NoError(t, err) + require.NoError(t, stuck.SetReadDeadline(time.Now().Add(10*time.Second))) + n, err := io.Copy(io.Discard, stuck) + var ne net.Error + if errors.As(err, &ne) && ne.Timeout() { + t.Fatalf("the send kept the connection after Stop, read %d bytes", n) + } + assert.Less(t, n, int64(1<<20), "the send kept going after Stop") + // The capture loop logs on its own goroutine after Start has returned, give it the chance + assert.Never(t, func() bool { return strings.Contains(logs.String(), "Graphite export failed") }, 500*time.Millisecond, 10*time.Millisecond, "a stop is not a failure") +} diff --git a/stats.go b/stats.go index a82bbc2b..c6895eef 100644 --- a/stats.go +++ b/stats.go @@ -153,8 +153,8 @@ func (s *statsServer) Start() { return } cfg := *s.runCfg - captureFns, listener := s.buildRuntime(cfg) runCtx, cancel := context.WithCancel(s.ctx) + captureFns, listener := s.buildRuntime(runCtx, cfg) rt := &statsRuntime{cancel: cancel, listener: listener} s.run = rt s.runMu.Unlock() @@ -237,7 +237,8 @@ func (s *statsServer) Stop() { // buildRuntime produces the capture functions and, for prometheus, an un-served // http.Server from cfg. cfg has already been validated by loadStatsConfig. -func (s *statsServer) buildRuntime(cfg statsConfig) ([]func(), *http.Server) { +// A graphite send in progress is abandoned when ctx is done. +func (s *statsServer) buildRuntime(ctx context.Context, cfg statsConfig) ([]func(), *http.Server) { // rcrowley/go-metrics guards these registrations with a private sync.Once, // so subsequent reloads are no-ops. metrics.RegisterDebugGCStats(metrics.DefaultRegistry) @@ -262,7 +263,7 @@ func (s *statsServer) buildRuntime(cfg statsConfig) ([]func(), *http.Server) { Percentiles: []float64{0.5, 0.75, 0.95, 0.99, 0.999}, } captureFns = append(captureFns, func() { - if err := graphiteOnce(gcfg); err != nil { + if err := graphiteOnce(ctx, gcfg, graphiteTimeout); err != nil && ctx.Err() == nil { s.l.Error("Graphite export failed", "error", err) } }) diff --git a/stats_test.go b/stats_test.go index 9d1aae08..58bfe417 100644 --- a/stats_test.go +++ b/stats_test.go @@ -2,7 +2,6 @@ package nebula import ( "context" - "io" "log/slog" "net" "strconv" @@ -110,15 +109,14 @@ func TestStatsServer_reload_initial_prometheus(t *testing.T) { } func TestStatsServer_Start_graphite_blocksUntilStop(t *testing.T) { - sink := newGraphiteSink(t) - defer sink.Close() + h := newGraphiteHost(t, 0) s, c := newTestStatsServer(t) setStatsConfig(c, map[string]any{ "type": "graphite", "interval": "1s", "protocol": "tcp", - "host": sink.Addr(), + "host": h.ln.Addr().String(), "prefix": "test", }) require.NoError(t, s.reload(c, true)) @@ -371,35 +369,6 @@ func waitForListening(t *testing.T, addr string) { }) } -// graphiteSink is a minimal TCP accept-and-discard server so graphiteOnce -// calls in tests don't spam error logs or wedge on connection refused. -type graphiteSink struct { - ln net.Listener -} - -func newGraphiteSink(t *testing.T) *graphiteSink { - t.Helper() - ln, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - g := &graphiteSink{ln: ln} - go func() { - for { - conn, err := ln.Accept() - if err != nil { - return - } - go func(c net.Conn) { - _, _ = io.Copy(io.Discard, c) - _ = c.Close() - }(conn) - } - }() - return g -} - -func (g *graphiteSink) Addr() string { return g.ln.Addr().String() } -func (g *graphiteSink) Close() { _ = g.ln.Close() } - func freeTCPPort(t *testing.T) string { t.Helper() ln, err := net.Listen("tcp", "127.0.0.1:0") From 5bb0622192aea67c8cc097fcbf50ed07bebe814f Mon Sep 17 00:00:00 2001 From: Nate Brown Date: Thu, 8 Oct 2026 10:00:48 -0500 Subject: [PATCH 18/31] stats: Follow stats.interval on reload, interface gauges ride the capture loop (#1919) --- interface.go | 21 +-- interface_emit_test.go | 19 +-- main.go | 8 +- stats.go | 59 +++++--- stats_test.go | 296 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 350 insertions(+), 53 deletions(-) diff --git a/interface.go b/interface.go index 2b9ff5eb..5320b4ce 100644 --- a/interface.go +++ b/interface.go @@ -639,17 +639,15 @@ func (f *Interface) reloadMisc(c *config.C) { } } -func (f *Interface) emitStats(ctx context.Context, i time.Duration) { - ticker := time.NewTicker(i) - defer ticker.Stop() - +// statsEmitter updates the interface's gauges, the stats server runs it on its capture loop. +func (f *Interface) statsEmitter() func() { udpStats := udp.NewUDPStatsEmitter(f.writers) certExpirationGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil) certInitiatingVersion := metrics.GetOrRegisterGauge("certificate.initiating_version", nil) certMaxVersion := metrics.GetOrRegisterGauge("certificate.max_version", nil) - emit := func() { + return func() { f.firewall.EmitStats() f.handshakeManager.EmitStats() udpStats() @@ -666,19 +664,6 @@ func (f *Interface) emitStats(ctx context.Context, i time.Duration) { certMaxVersion.Update(int64(certState.v1Cert.Version())) } } - - // Prime gauges so a Prometheus scrape that lands before the first tick - // sees real values instead of the zero defaults (issue #907). - emit() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - emit() - } - } } func (f *Interface) GetHostInfo(vpnIp netip.Addr) *HostInfo { diff --git a/interface_emit_test.go b/interface_emit_test.go index b0a9d025..f78e02ca 100644 --- a/interface_emit_test.go +++ b/interface_emit_test.go @@ -3,7 +3,6 @@ package nebula import ( - "context" "net/netip" "testing" "time" @@ -18,11 +17,8 @@ import ( "github.com/stretchr/testify/require" ) -// Test_emitStats_primesGauges covers issue #907: a Prometheus scrape that -// landed before the first ticker fire used to read 0 for the cert gauges. -// emitStats now primes the gauges before entering the ticker loop. We assert -// the gauge is zero before the first call and non-zero after. -func Test_emitStats_primesGauges(t *testing.T) { +// Priming before the first tick (issue #907) is Start's job, see TestStatsServer_Start_primes. +func Test_statsEmitter_setsGauges(t *testing.T) { defer metrics.DefaultRegistry.UnregisterAll() l := test.NewLogger() @@ -56,17 +52,12 @@ func Test_emitStats_primesGauges(t *testing.T) { ifce.pki.cs.Store(cs) ttlGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil) - require.Zero(t, ttlGauge.Value(), "gauge should be zero before emitStats runs") + require.Zero(t, ttlGauge.Value(), "gauge should be zero before the emitter runs") - // Pre-cancel the context so emitStats returns after priming the gauges - // without ever reading from ticker.C. The one hour interval is just a - // belt-and-suspenders, the test does not expect the ticker to fire. - ctx, cancel := context.WithCancel(context.Background()) - cancel() - ifce.emitStats(ctx, time.Hour) + ifce.statsEmitter()() ttl := ttlGauge.Value() - assert.Positive(t, ttl, "ttl gauge should be primed by emitStats before its first tick") + assert.Positive(t, ttl, "ttl gauge should be set by one run of the emitter") assert.LessOrEqual(t, ttl, int64(3600)) assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.initiating_version", nil).Value()) assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.max_version", nil).Value()) diff --git a/main.go b/main.go index 52f7bea8..a8d01779 100644 --- a/main.go +++ b/main.go @@ -300,6 +300,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev } var ifce *Interface + var emitStats func() if !configTest { ifce, err = NewInterface(ctx, ifConfig) if err != nil { @@ -318,9 +319,12 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev go handshakeManager.Run(ctx) punchy.Start(ctx, ifce, hostMap, lightHouse) + + // On the stats server's loop, so the gauges follow the interval it accepted + emitStats = ifce.statsEmitter() } - stats, err := newStatsServerFromConfig(ctx, l, c, buildVersion, configTest) + stats, err := newStatsServerFromConfig(ctx, l, c, buildVersion, configTest, emitStats) if err != nil { return nil, util.ContextualizeIfNeeded("Failed to start stats emitter", err) } @@ -329,8 +333,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev return nil, nil } - go ifce.emitStats(ctx, c.GetDuration("stats.interval", time.Second*10)) - attachCommands(l, c, ssh, ifce) networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c) diff --git a/stats.go b/stats.go index c6895eef..7e42d44b 100644 --- a/stats.go +++ b/stats.go @@ -38,6 +38,11 @@ type statsServer struct { runMu sync.Mutex runCfg *statsConfig run *statsRuntime // non-nil while a runtime is live + // Closed when the last runtime's capture loop has returned. Stop waits on it so nothing captures once it returns, + // and a reload's new runtime never overlaps the old one, go-metrics keeps its GC capture state in unguarded globals + loopDone chan struct{} + // See newStatsServerFromConfig, never changes after it + emitStats func() } // statsRuntime is the live state owned by a single Start invocation. Start @@ -82,12 +87,16 @@ type promConfig struct { // // Start is safe to call unconditionally: it no-ops when stats are disabled. // The returned pointer is always non-nil, even on error. -func newStatsServerFromConfig(ctx context.Context, l *slog.Logger, c *config.C, buildVersion string, configTest bool) (*statsServer, error) { +// +// emitStats, if set, updates metrics on every capture, ahead of the exporters so each export sees what it just set. It +// is fixed here, before the reload callback exists, so every runtime has it. +func newStatsServerFromConfig(ctx context.Context, l *slog.Logger, c *config.C, buildVersion string, configTest bool, emitStats func()) (*statsServer, error) { s := &statsServer{ l: l, ctx: ctx, buildVersion: buildVersion, configTest: configTest, + emitStats: emitStats, } c.RegisterReloadCallback(func(c *config.C) { @@ -120,7 +129,6 @@ func (s *statsServer) reload(c *config.C, initial bool) error { s.runMu.Lock() sameCfg := s.runCfg != nil && *s.runCfg == newCfg s.runCfg = &newCfg - running := s.run != nil s.runMu.Unlock() s.enabled.Store(enabled) @@ -129,9 +137,8 @@ func (s *statsServer) reload(c *config.C, initial bool) error { return nil } - if running { - s.Stop() - } + // Even with nothing running, a runtime whose listener failed can still be finishing its last pass + s.Stop() if enabled && !s.configTest { go s.Start() } @@ -157,9 +164,15 @@ func (s *statsServer) Start() { captureFns, listener := s.buildRuntime(runCtx, cfg) rt := &statsRuntime{cancel: cancel, listener: listener} s.run = rt + done := make(chan struct{}) + s.loopDone = done s.runMu.Unlock() - go captureStatsLoop(runCtx, cfg.interval, captureFns) + // Prime before serving, so a scrape that lands before the first tick sees real values instead of zeros (issue #907) + for _, fn := range captureFns { + fn() + } + go captureStatsLoop(runCtx, cfg.interval, captureFns, done) cleanExit := true if listener == nil { @@ -218,20 +231,25 @@ func (s *statsServer) serveListener(listener *http.Server) bool { // Stop tears down the active runtime, if any. Idempotent. func (s *statsServer) Stop() { + // Held until the last pass is done, a Start that slipped in after s.run cleared would capture alongside it s.runMu.Lock() + defer s.runMu.Unlock() rt := s.run s.run = nil - s.runMu.Unlock() - if rt == nil { - return - } - rt.cancel() - if rt.listener != nil { - shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - if err := rt.listener.Shutdown(shutdownCtx); err != nil { - s.l.Warn("Failed to shut down prometheus stats listener", "error", err) + if rt != nil { + rt.cancel() + if rt.listener != nil { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + if err := rt.listener.Shutdown(shutdownCtx); err != nil { + s.l.Warn("Failed to shut down prometheus stats listener", "error", err) + } + cancel() } - cancel() + } + // Nothing captures once Stop returns. The last runtime is cancelled by now, whoever cleared it, so this is at most + // the pass it was in. Nothing in a pass takes runMu + if s.loopDone != nil { + <-s.loopDone } } @@ -248,6 +266,9 @@ func (s *statsServer) buildRuntime(ctx context.Context, cfg statsConfig) ([]func func() { metrics.CaptureDebugGCStatsOnce(metrics.DefaultRegistry) }, func() { metrics.CaptureRuntimeMemStatsOnce(metrics.DefaultRegistry) }, } + if s.emitStats != nil { + captureFns = append(captureFns, s.emitStats) + } switch cfg.typ { case "graphite": @@ -310,8 +331,10 @@ func (s *statsServer) buildRuntime(ctx context.Context, cfg statsConfig) ([]func return captureFns, nil } -// captureStatsLoop runs each fn on every tick of d until ctx is cancelled. -func captureStatsLoop(ctx context.Context, d time.Duration, fns []func()) { +// captureStatsLoop runs each fn on every tick of d until ctx is cancelled, then +// closes done. +func captureStatsLoop(ctx context.Context, d time.Duration, fns []func(), done chan struct{}) { + defer close(done) t := time.NewTicker(d) defer t.Stop() for { diff --git a/stats_test.go b/stats_test.go index 58bfe417..a89ac9bb 100644 --- a/stats_test.go +++ b/stats_test.go @@ -2,12 +2,18 @@ package nebula import ( "context" + "fmt" + "io" "log/slog" "net" + "net/http" + "net/http/httptest" "strconv" + "sync/atomic" "testing" "time" + "github.com/rcrowley/go-metrics" "github.com/slackhq/nebula/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -369,6 +375,19 @@ func waitForListening(t *testing.T, addr string) { }) } +// stopAtCleanup returns the ctx to build *s with, and stops it when the test ends. Cancelling first turns away a Start +// a reload spawned that hasn't run yet, then Stop waits out the last pass, so nothing captures into the next test +func stopAtCleanup(t *testing.T, s **statsServer) context.Context { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(func() { + cancel() + if *s != nil { + (*s).Stop() + } + }) + return ctx +} + func freeTCPPort(t *testing.T) string { t.Helper() ln, err := net.Listen("tcp", "127.0.0.1:0") @@ -377,3 +396,280 @@ func freeTCPPort(t *testing.T) string { require.NoError(t, ln.Close()) return strconv.Itoa(port) } + +// Each export has to see what emitStats set in the same pass +func TestStatsServer_emitStatsRunsBeforeExport(t *testing.T) { + defer metrics.DefaultRegistry.Unregister("emitter.order.test") + + l := slog.New(slog.DiscardHandler) + c := config.NewC(l) + setStatsConfig(c, map[string]any{ + "type": "prometheus", + "interval": "1s", + "listen": "127.0.0.1:0", + "path": "/metrics", + }) + gauge := metrics.GetOrRegisterGauge("emitter.order.test", nil) + s, err := newStatsServerFromConfig(t.Context(), l, c, "", false, func() { gauge.Update(42) }) + require.NoError(t, err) + + s.runMu.Lock() + fns, srv := s.buildRuntime(t.Context(), *s.runCfg) + s.runMu.Unlock() + require.NotNil(t, srv) + for _, fn := range fns { + fn() + } + + rec := httptest.NewRecorder() + srv.Handler.ServeHTTP(rec, httptest.NewRequest("GET", "/metrics", nil)) + assert.Contains(t, rec.Body.String(), "emitter_order_test 42") +} + +// A scrape that lands the moment the listener is up already sees what emitStats set (issue #907) +func TestStatsServer_Start_primesBeforeServing(t *testing.T) { + defer metrics.DefaultRegistry.Unregister("prime.before.serving") + + port := freeTCPPort(t) + l := slog.New(slog.DiscardHandler) + c := config.NewC(l) + setStatsConfig(c, map[string]any{ + "type": "prometheus", + "interval": "1h", + "listen": "127.0.0.1:" + port, + "path": "/metrics", + }) + gauge := metrics.GetOrRegisterGauge("prime.before.serving", nil) + // The pause would let a listener that came up before the prime serve the zero value + var s *statsServer + ctx := stopAtCleanup(t, &s) + s, err := newStatsServerFromConfig(ctx, l, c, "", false, func() { + time.Sleep(200 * time.Millisecond) + gauge.Update(42) + }) + require.NoError(t, err) + + go s.Start() + waitForListening(t, "127.0.0.1:"+port) + + resp, err := http.Get("http://127.0.0.1:" + port + "/metrics") + require.NoError(t, err) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + assert.Contains(t, string(body), "prime_before_serving 42") +} + +// Start captures once before the first tick, the interval here is an hour +func TestStatsServer_Start_primes(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer ln.Close() + + l := slog.New(slog.DiscardHandler) + c := config.NewC(l) + setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": ln.Addr().String()}) + primed := make(chan struct{}, 1) + var s *statsServer + ctx := stopAtCleanup(t, &s) + s, err = newStatsServerFromConfig(ctx, l, c, "", false, func() { + select { + case primed <- struct{}{}: + default: + } + }) + require.NoError(t, err) + + go s.Start() + select { + case <-primed: + case <-time.After(5 * time.Second): + t.Fatal("nothing was captured before the first tick") + } +} + +// Graphite has to ship what emitStats set in the same pass, not the pass before +func TestStatsServer_emitStatsRunsBeforeGraphiteExport(t *testing.T) { + defer metrics.DefaultRegistry.Unregister("emitter.graphite.test") + + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer ln.Close() + received := make(chan string, 1) + go func() { + conn, err := ln.Accept() + if err != nil { + return + } + defer conn.Close() + b, _ := io.ReadAll(conn) + received <- string(b) + }() + + l := slog.New(slog.DiscardHandler) + c := config.NewC(l) + setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": ln.Addr().String()}) + gauge := metrics.GetOrRegisterGauge("emitter.graphite.test", nil) + s, err := newStatsServerFromConfig(t.Context(), l, c, "", false, func() { gauge.Update(42) }) + require.NoError(t, err) + + s.runMu.Lock() + fns, _ := s.buildRuntime(t.Context(), *s.runCfg) + s.runMu.Unlock() + for _, fn := range fns { + fn() + } + + select { + case body := <-received: + assert.Contains(t, body, ".emitter.graphite.test.value 42 ") + case <-time.After(5 * time.Second): + t.Fatal("graphite sent nothing") + } +} + +// stats.interval reloads, and the runtime a reload starts still has emitStats +func TestStatsServer_reloadMovesTheInterval(t *testing.T) { + var passes atomic.Int64 + l := slog.New(slog.DiscardHandler) + c := config.NewC(l) + // Nothing listens here, each export fails at once + host := "127.0.0.1:" + freeTCPPort(t) + setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": host}) + var s *statsServer + ctx := stopAtCleanup(t, &s) + s, err := newStatsServerFromConfig(ctx, l, c, "", false, func() { passes.Add(1) }) + require.NoError(t, err) + go s.Start() + waitFor(t, func() bool { return passes.Load() == 1 }) + + setStatsConfig(c, map[string]any{"type": "graphite", "interval": "20ms", "host": host}) + require.NoError(t, s.reload(c, false)) + waitFor(t, func() bool { return passes.Load() >= 6 }) + + setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": host}) + require.NoError(t, s.reload(c, false)) + // The reload's Stop waited out the 20ms loop. The new runtime's prime may land either side of this read, it is the + // only pass left to come + stopped := passes.Load() + time.Sleep(300 * time.Millisecond) + assert.LessOrEqual(t, passes.Load()-stopped, int64(1), "the 20ms loop kept running after the reload") +} + +// A reload's new runtime never captures while the old one is still in a pass, go-metrics keeps its GC capture state +// in unguarded globals +func TestStatsServer_reloadWaitsForTheLastPass(t *testing.T) { + var active, most atomic.Int64 + l := slog.New(slog.DiscardHandler) + c := config.NewC(l) + host := "127.0.0.1:" + freeTCPPort(t) + setStatsConfig(c, map[string]any{"type": "graphite", "interval": "10ms", "host": host}) + var s *statsServer + ctx := stopAtCleanup(t, &s) + s, err := newStatsServerFromConfig(ctx, l, c, "", false, func() { + n := active.Add(1) + for { + m := most.Load() + if n <= m || most.CompareAndSwap(m, n) { + break + } + } + // Long enough that a reload almost always lands mid pass + time.Sleep(50 * time.Millisecond) + active.Add(-1) + }) + require.NoError(t, err) + go s.Start() + + for i := range 6 { + setStatsConfig(c, map[string]any{"type": "graphite", "interval": fmt.Sprintf("%dms", 11-i%2), "host": host}) + require.NoError(t, s.reload(c, false)) + time.Sleep(30 * time.Millisecond) + } + assert.Equal(t, int64(1), most.Load()) +} + +// Nothing captures once Stop returns, a pass in progress is waited out +func TestStatsServer_Stop_waitsForThePass(t *testing.T) { + var inPass atomic.Bool + entered := make(chan struct{}, 1) + l := slog.New(slog.DiscardHandler) + c := config.NewC(l) + setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": "127.0.0.1:" + freeTCPPort(t)}) + var s *statsServer + ctx := stopAtCleanup(t, &s) + s, err := newStatsServerFromConfig(ctx, l, c, "", false, func() { + inPass.Store(true) + select { + case entered <- struct{}{}: + default: + } + time.Sleep(100 * time.Millisecond) + inPass.Store(false) + }) + require.NoError(t, err) + + go s.Start() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("never captured") + } + s.Stop() + assert.False(t, inPass.Load(), "Stop returned with a pass still running") +} + +// A Start that arrives while Stop waits out the last pass doesn't capture alongside it +func TestStatsServer_Start_duringStopWaitsForThePass(t *testing.T) { + var active, most atomic.Int64 + entered := make(chan struct{}, 1) + l := slog.New(slog.DiscardHandler) + c := config.NewC(l) + setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": "127.0.0.1:" + freeTCPPort(t)}) + var s *statsServer + ctx := stopAtCleanup(t, &s) + s, err := newStatsServerFromConfig(ctx, l, c, "", false, func() { + n := active.Add(1) + for { + m := most.Load() + if n <= m || most.CompareAndSwap(m, n) { + break + } + } + select { + case entered <- struct{}{}: + default: + } + time.Sleep(200 * time.Millisecond) + active.Add(-1) + }) + require.NoError(t, err) + + go s.Start() + select { + case <-entered: + case <-time.After(5 * time.Second): + t.Fatal("never captured") + } + stopped := make(chan struct{}) + go func() { + s.Stop() + close(stopped) + }() + // Stop holds runMu for its whole wait, the next Start has to arrive after it took it + waitFor(t, func() bool { + if s.runMu.TryLock() { + s.runMu.Unlock() + return false + } + return true + }) + go s.Start() + select { + case <-stopped: + case <-time.After(5 * time.Second): + t.Fatal("Stop did not return") + } + waitFor(t, func() bool { return len(entered) == 1 }) + assert.Equal(t, int64(1), most.Load(), "a Start captured alongside the pass Stop was waiting out") +} From 13af8dd6f3534d21b4492062c3194c1709f6e337 Mon Sep 17 00:00:00 2001 From: Jack Doan Date: Thu, 8 Oct 2026 15:41:08 -0500 Subject: [PATCH 19/31] Cap configured MTUs at 8937 so relayed sends never outgrow their buffers (#1899) --- connection_state_test.go | 119 +++++++++++++++++++++++++++++++++++++++ examples/config.yml | 1 + header/header.go | 5 ++ inside.go | 26 ++++++--- interface.go | 6 ++ outside.go | 4 +- overlay/route_test.go | 43 ++++++++++++++ overlay/tun.go | 24 +++++++- overlay/tun_android.go | 2 +- overlay/tun_darwin.go | 6 +- overlay/tun_freebsd.go | 4 +- overlay/tun_ios.go | 2 +- overlay/tun_linux.go | 6 +- overlay/tun_netbsd.go | 4 +- overlay/tun_openbsd.go | 4 +- overlay/tun_tester.go | 2 +- overlay/tun_windows.go | 4 +- udp/udp_rio_windows.go | 2 +- 18 files changed, 234 insertions(+), 30 deletions(-) diff --git a/connection_state_test.go b/connection_state_test.go index 310a26a4..a946343d 100644 --- a/connection_state_test.go +++ b/connection_state_test.go @@ -2,6 +2,8 @@ package nebula import ( "net/netip" + "strconv" + "sync" "testing" "time" @@ -11,6 +13,7 @@ import ( ct "github.com/slackhq/nebula/cert_test" "github.com/slackhq/nebula/handshake" "github.com/slackhq/nebula/header" + "github.com/slackhq/nebula/overlay" "github.com/slackhq/nebula/test" "github.com/slackhq/nebula/udp" "github.com/stretchr/testify/assert" @@ -147,6 +150,122 @@ func TestSendNoMetricsCloseTunnelKeepsRebindEpoch(t *testing.T) { assert.True(t, hostinfo.markOut(1), "a CloseTunnel send must not consume the rebind epoch") } +// capturingConn is a udp.Conn that records every WriteTo. +type capturingConn struct { + udp.NoopConn + writes [][]byte + addrs []netip.AddrPort +} + +func (c *capturingConn) WriteTo(b []byte, addr netip.AddrPort) error { + c.writes = append(c.writes, append([]byte(nil), b...)) + c.addrs = append(c.addrs, addr) + return nil +} + +// TestSendNoMetricsViaRelay sends through a relay from the mtu sized buffer the cached packet flush uses. Payloads up +// to overlay.MaxMTU go out intact, anything bigger is dropped rather than outgrowing the buffer, which once panicked. +func TestSendNoMetricsViaRelay(t *testing.T) { + tests := []struct { + n int + sent bool + }{ + {0, true}, + {1300, true}, + {overlay.MaxMTU, true}, + {overlay.MaxMTU + 1, false}, // too big for the relay's tag + {mtu - 47, false}, // too big for the inner tag, this panicked + {9000, false}, // a full packet at tun.mtu 9000 + } + for _, tt := range tests { + t.Run(strconv.Itoa(tt.n), func(t *testing.T) { + // One tunnel to the target, carried end to end, and one to the relay that wraps it + targetInit, targetResp := runTestHandshake(t) + relayInit, relayResp := runTestHandshake(t) + ci, err := newConnectionStateFromResult(targetInit) + require.NoError(t, err) + relayCI, err := newConnectionStateFromResult(relayInit) + require.NoError(t, err) + + targetAddr := netip.MustParseAddr("10.0.0.2") + relayAddr := netip.MustParseAddr("10.0.0.3") + relayRemote := netip.MustParseAddrPort("192.0.2.1:4242") + + relay := &Relay{Type: TerminalType, State: Established, LocalIndex: 5, RemoteIndex: 6, PeerAddr: targetAddr} + relayHI := &HostInfo{ + vpnAddrs: []netip.Addr{relayAddr}, + ConnectionState: relayCI, + relayState: RelayState{ + relayForByAddr: map[netip.Addr]*Relay{targetAddr: relay}, + relayForByIdx: map[uint32]*Relay{relay.LocalIndex: relay}, + }, + } + relayHI.remote.Store(&relayRemote) + + // No remote of its own, so the send has to go via the relay + hostinfo := &HostInfo{ + vpnAddrs: []netip.Addr{targetAddr}, + ConnectionState: ci, + remoteIndexId: 7, + relayState: RelayState{ + relays: []netip.Addr{relayAddr}, + relayForByAddr: map[netip.Addr]*Relay{}, + relayForByIdx: map[uint32]*Relay{}, + }, + } + + l := test.NewLogger() + hm := newHostMap(l) + hm.Hosts[relayAddr] = relayHI + conn := &capturingConn{} + f := &Interface{ + l: l, + hostMap: hm, + messageMetrics: &MessageMetrics{txExhausted: metrics.NewCounter()}, + writers: []udp.Conn{conn}, + } + f.connectionManager = &connectionManager{intf: f, relayUsed: map[uint32]struct{}{}, relayUsedLock: &sync.RWMutex{}} + + payload := make([]byte, tt.n) + for i := range payload { + payload[i] = byte(i) + } + counter := ci.messageCounter.Load() + f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, payload, make([]byte, 12), make([]byte, mtu), 0) + + if !tt.sent { + assert.Empty(t, conn.writes) + assert.Equal(t, counter, ci.messageCounter.Load(), "a dropped packet must not spend a counter") + return + } + require.Len(t, conn.writes, 1) + assert.Equal(t, relayRemote, conn.addrs[0]) + pkt := conn.writes[0] + + // The relay authenticates the whole packet under its own header + outer := &header.H{} + require.NoError(t, outer.Parse(pkt)) + assert.Equal(t, header.MessageRelay, outer.Subtype) + assert.Equal(t, relay.RemoteIndex, outer.RemoteIndex) + relayPeer, err := newConnectionStateFromResult(relayResp) + require.NoError(t, err) + require.NoError(t, relayPeer.VerifyRelay(l, outer.MessageCounter, pkt, make([]byte, 12))) + + // Inside it, the target decrypts the end to end packet back to the payload + inner := pkt[header.Len : len(pkt)-relayPeer.dKey.Overhead()] + h := &header.H{} + require.NoError(t, h.Parse(inner)) + assert.Equal(t, header.Message, h.Type) + assert.Equal(t, hostinfo.remoteIndexId, h.RemoteIndex) + targetPeer, err := newConnectionStateFromResult(targetResp) + require.NoError(t, err) + got, err := targetPeer.Decrypt(l, h.MessageCounter, inner, make([]byte, 12)) + require.NoError(t, err) + assert.Equal(t, payload, got) + }) + } +} + func TestNewConnectionStateFromResult(t *testing.T) { initR, respR := runTestHandshake(t) diff --git a/examples/config.yml b/examples/config.yml index 0edb4dc6..987ef90e 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -265,6 +265,7 @@ tun: # Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 500 tx_queue: 500 # Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic + # Anything above 8937, here or on a route below, is capped to 8937 with a warning mtu: 1300 # the use_offloads setting controls if Nebula will attempt to enable GSO and GRO for the tun device. Linux only, not reloadable. diff --git a/header/header.go b/header/header.go index d0848c55..7d970e51 100644 --- a/header/header.go +++ b/header/header.go @@ -24,6 +24,11 @@ type m = map[string]any const ( Version uint8 = 1 Len = 16 + + // MaxOverhead is the most a payload grows on its way to the underlay. + // A relayed packet has two headers and two AEAD tags, one for the inner tunnel and another for the relay tunnel. + // Every supported cipher (today) has a 16 byte tag. + MaxOverhead = 2 * (Len + 16) ) type MessageType uint8 diff --git a/inside.go b/inside.go index 4a0ac422..696b9d97 100644 --- a/inside.go +++ b/inside.go @@ -199,8 +199,8 @@ func (f *Interface) sendInsideMessage(hostinfo *HostInfo, pkt tio.Packet, nb []b } err = tio.SegmentSuperpacket(pkt, func(seg []byte) error { - //relay header + header + plaintext + AEAD tag (16 bytes for both AES-GCM and ChaCha20-Poly1305) + relay tag - scratch := sendBatch.Reserve(header.Len + header.Len + len(seg) + 16 + 16) + //relay header + header + plaintext + AEAD tag + relay tag + scratch := sendBatch.Reserve(len(seg) + header.MaxOverhead) innerPacket := f.sendInsideEncrypt(hostinfo, ci, seg, scratch[header.Len:], nb) if innerPacket == nil { @@ -529,16 +529,26 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType return } useRelay := !remote.IsValid() && !hostinfo.GetRemote().IsValid() - fullOut := out + // The encrypt appends to out in place and a relayed packet is then wrapped in place, + // so out has to fit the whole packet up front or the encrypt reallocates out from under SendVia. + // We cap the MTU of tun devices and routes we create ourselves, so only a host that set a bigger MTU than we allow gets here. + need := header.Len + len(p) + ci.eKey.Overhead() if useRelay { - if len(out) < header.Len { - // out always has a capacity of mtu, but not always a length greater than the header.Len. - // Grow it to make sure the next operation works. - out = out[:header.Len] + // The relay tunnel runs the same cipher, and adds its own header and tag + need += header.Len + ci.eKey.Overhead() + } + if cap(out) < need { + if l, ok := logging.RateLimited(f.l, f.txOversizeLimiter); ok { + hostinfo.logger(l).Error("Dropping outbound packet, out buffer not large enough", "outCap", cap(out), "payloadLen", len(p), "relayed", useRelay) } + return + } + + fullOut := out + if useRelay { // Save a header's worth of data at the front of the 'out' buffer. - out = out[header.Len:] + out = out[header.Len:header.Len] } if noiseutil.EncryptLockNeeded { diff --git a/interface.go b/interface.go index 5320b4ce..29051be4 100644 --- a/interface.go +++ b/interface.go @@ -127,6 +127,7 @@ type Interface struct { udpTxErrLimiter *ratelimit.Limiter // udp direct transmit error log limiter udpRelayTxErrLimiter *ratelimit.Limiter // udp relay transmit error log limiter + txOversizeLimiter *ratelimit.Limiter // oversized outbound packet drop log limiter // fatalErr holds the first unexpected reader error that caused shutdown. // nil means "no fatal error" (yet) @@ -210,6 +211,10 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) { if err != nil { return nil, err } + txOversizeLimiter, err := ratelimit.New(1, time.Second) + if err != nil { + return nil, err + } if c.routines <= 1 { c.PinThreads = false //pinning is not useful unless there's more than one tun reader @@ -245,6 +250,7 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) { pinThreads: c.PinThreads, udpTxErrLimiter: udpTxErrLimiter, udpRelayTxErrLimiter: udpRelayTxErrLimiter, + txOversizeLimiter: txOversizeLimiter, metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)), metricTxDropped: metrics.GetOrRegisterCounter("udp.tx.dropped", nil), diff --git a/outside.go b/outside.go index 12f2160c..dc019e36 100644 --- a/outside.go +++ b/outside.go @@ -160,9 +160,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont case header.TestReply: // No-op, useful for the Roaming and connectionManager side-effects above case header.TestRequest: - const maxCipherOverhead = 16 //todo we use this too often, needs a real importable const - const maxOverhead = header.Len + header.Len + maxCipherOverhead + maxCipherOverhead - if maxOverhead+len(out) > len(rxc.scratch) { + if header.MaxOverhead+len(out) > len(rxc.scratch) { // A reply that cannot fit in scratch is dropped no matter the log level. if f.l.Enabled(context.Background(), slog.LevelDebug) { hostinfo.logger(f.l).Debug("dropping oversized test request", "payloadLen", len(out), "from", via) diff --git a/overlay/route_test.go b/overlay/route_test.go index f9d9dcd9..8d62f0b0 100644 --- a/overlay/route_test.go +++ b/overlay/route_test.go @@ -6,8 +6,10 @@ import ( "testing" "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/header" "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/test" + "github.com/slackhq/nebula/udp" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -402,3 +404,44 @@ func Test_makeMultipathUnsafeRouteTree(t *testing.T) { routing.CalculateBucketsForGateways(expectedGateways) assert.ElementsMatch(t, expectedGateways, r) } + +func Test_getMTU(t *testing.T) { + l := test.NewLogger() + c := config.NewC(l) + + // unset + assert.Equal(t, DefaultMTU, getMTU(l, c)) + + // the largest a relayed packet can carry in one underlay buffer + require.Equal(t, udp.MTU, MaxMTU+header.MaxOverhead) + c.Settings["tun"] = map[string]any{"mtu": MaxMTU} + assert.Equal(t, MaxMTU, getMTU(l, c)) + + // too big is capped + c.Settings["tun"] = map[string]any{"mtu": 9000} + assert.Equal(t, MaxMTU, getMTU(l, c)) +} + +func Test_getAllRoutesFromConfigCapsMTU(t *testing.T) { + l := test.NewLogger() + c := config.NewC(l) + c.Settings["tun"] = map[string]any{ + "routes": []any{ + map[string]any{"mtu": "9000", "route": "10.0.0.0/29"}, + map[string]any{"mtu": "1300", "route": "10.0.0.8/29"}, + }, + "unsafe_routes": []any{ + map[string]any{"via": "127.0.0.1", "mtu": "9000", "route": "1.0.0.0/29"}, + }, + } + + changed, routes, err := getAllRoutesFromConfig(l, c, []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")}, true) + require.NoError(t, err) + assert.True(t, changed) + + mtus := map[string]int{} + for _, r := range routes { + mtus[r.Cidr.String()] = r.MTU + } + assert.Equal(t, map[string]int{"10.0.0.0/29": MaxMTU, "10.0.0.8/29": 1300, "1.0.0.0/29": MaxMTU}, mtus) +} diff --git a/overlay/tun.go b/overlay/tun.go index 39f27dd4..eb994341 100644 --- a/overlay/tun.go +++ b/overlay/tun.go @@ -7,11 +7,27 @@ import ( "net/netip" "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/header" + "github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/util" ) const DefaultMTU = 1300 +// MaxMTU is the largest tun.mtu, or route mtu, the underlay can carry. +// Every underlay buffer is udp.MTU bytes, and a relayed packet adds header.MaxOverhead to its plaintext. +const MaxMTU = udp.MTU - header.MaxOverhead + +// getMTU reads tun.mtu, capping a value too big for the underlay to carry. +func getMTU(l *slog.Logger, c *config.C) int { + mtu := c.GetInt("tun.mtu", DefaultMTU) + if mtu > MaxMTU { + l.Warn("tun.mtu is too big for the underlay to carry, capping it", "mtu", mtu, "maxMTU", MaxMTU) + return MaxMTU + } + return mtu +} + type NameError struct { Name string Underlying error @@ -41,7 +57,7 @@ func NewFdDeviceFromConfig(fd *int) DeviceFactory { } } -func getAllRoutesFromConfig(c *config.C, vpnNetworks []netip.Prefix, initial bool) (bool, []Route, error) { +func getAllRoutesFromConfig(l *slog.Logger, c *config.C, vpnNetworks []netip.Prefix, initial bool) (bool, []Route, error) { if !initial && !c.HasChanged("tun.routes") && !c.HasChanged("tun.unsafe_routes") { return false, nil, nil } @@ -57,6 +73,12 @@ func getAllRoutesFromConfig(c *config.C, vpnNetworks []netip.Prefix, initial boo } routes = append(routes, unsafeRoutes...) + for i, r := range routes { + if r.MTU > MaxMTU { + l.Warn("route mtu is too big for the underlay to carry, capping it", "route", r.Cidr, "mtu", r.MTU, "maxMTU", MaxMTU) + routes[i].MTU = MaxMTU + } + } return true, routes, nil } diff --git a/overlay/tun_android.go b/overlay/tun_android.go index c7d5829b..819aa1f7 100644 --- a/overlay/tun_android.go +++ b/overlay/tun_android.go @@ -68,7 +68,7 @@ func (t *tun) Activate() error { } func (t *tun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) + change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) if err != nil { return err } diff --git a/overlay/tun_darwin.go b/overlay/tun_darwin.go index c086eda2..b95a93cf 100644 --- a/overlay/tun_darwin.go +++ b/overlay/tun_darwin.go @@ -126,7 +126,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t f: os.NewFile(uintptr(fd), ""), Device: name, vpnNetworks: vpnNetworks, - DefaultMTU: c.GetInt("tun.mtu", DefaultMTU), + DefaultMTU: getMTU(l, c), l: l, } @@ -167,7 +167,7 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip f: file, Device: utunNameFromFd(deviceFd), vpnNetworks: vpnNetworks, - DefaultMTU: c.GetInt("tun.mtu", DefaultMTU), + DefaultMTU: getMTU(l, c), hostOwned: true, l: l, } @@ -350,7 +350,7 @@ func (t *tun) activate6(network netip.Prefix) error { } func (t *tun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) + change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) if err != nil { return err } diff --git a/overlay/tun_freebsd.go b/overlay/tun_freebsd.go index e2d6df10..cebf959d 100644 --- a/overlay/tun_freebsd.go +++ b/overlay/tun_freebsd.go @@ -370,7 +370,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t t := &tun{ Device: deviceName, vpnNetworks: vpnNetworks, - MTU: c.GetInt("tun.mtu", DefaultMTU), + MTU: getMTU(l, c), l: l, fd: fd, shutdownR: shutdownR, @@ -510,7 +510,7 @@ func (t *tun) setMTU() error { } func (t *tun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) + change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) if err != nil { return err } diff --git a/overlay/tun_ios.go b/overlay/tun_ios.go index 23005185..9ee30f77 100644 --- a/overlay/tun_ios.go +++ b/overlay/tun_ios.go @@ -68,7 +68,7 @@ func (t *tun) Activate() error { } func (t *tun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) + change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) if err != nil { return err } diff --git a/overlay/tun_linux.go b/overlay/tun_linux.go index 10904450..b5e0a675 100644 --- a/overlay/tun_linux.go +++ b/overlay/tun_linux.go @@ -244,7 +244,7 @@ func newTunGeneric(c *config.C, l *slog.Logger, fd int, vnetHdr bool, offloadFla } func (t *tun) reload(c *config.C, initial bool) error { - routeChange, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) + routeChange, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) if err != nil { return err } @@ -256,7 +256,7 @@ func (t *tun) reload(c *config.C, initial bool) error { // Only the mtu changed. The routes are the same, parse them again so the ones without their own mtu pick up the // new default. Without this an mtu change would remove them all if !routeChange { - _, routes, err = getAllRoutesFromConfig(c, t.vpnNetworks, true) + _, routes, err = getAllRoutesFromConfig(t.l, c, t.vpnNetworks, true) if err != nil { return err } @@ -276,7 +276,7 @@ func (t *tun) reload(c *config.C, initial bool) error { oldDefaultMTU := t.DefaultMTU oldMaxMTU := t.MaxMTU - newDefaultMTU := c.GetInt("tun.mtu", DefaultMTU) + newDefaultMTU := getMTU(t.l, c) newMaxMTU := newDefaultMTU for i, r := range routes { if r.MTU == 0 { diff --git a/overlay/tun_netbsd.go b/overlay/tun_netbsd.go index d69ea67a..c6f17848 100644 --- a/overlay/tun_netbsd.go +++ b/overlay/tun_netbsd.go @@ -99,7 +99,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t fd: fd, Device: deviceName, vpnNetworks: vpnNetworks, - MTU: c.GetInt("tun.mtu", DefaultMTU), + MTU: getMTU(l, c), l: l, } @@ -340,7 +340,7 @@ func (t *tun) doIoctlByName(ctl uintptr, value uint32) error { } func (t *tun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) + change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) if err != nil { return err } diff --git a/overlay/tun_openbsd.go b/overlay/tun_openbsd.go index da752e50..aa90b5fe 100644 --- a/overlay/tun_openbsd.go +++ b/overlay/tun_openbsd.go @@ -90,7 +90,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t fd: fd, Device: deviceName, vpnNetworks: vpnNetworks, - MTU: c.GetInt("tun.mtu", DefaultMTU), + MTU: getMTU(l, c), l: l, } @@ -319,7 +319,7 @@ func (t *tun) doIoctlByName(ctl uintptr, value uint32) error { } func (t *tun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) + change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) if err != nil { return err } diff --git a/overlay/tun_tester.go b/overlay/tun_tester.go index 39d911ec..11be1aaf 100644 --- a/overlay/tun_tester.go +++ b/overlay/tun_tester.go @@ -31,7 +31,7 @@ type TestTun struct { } func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*TestTun, error) { - _, routes, err := getAllRoutesFromConfig(c, vpnNetworks, true) + _, routes, err := getAllRoutesFromConfig(l, c, vpnNetworks, true) if err != nil { return nil, err } diff --git a/overlay/tun_windows.go b/overlay/tun_windows.go index b8d49521..982b8fa0 100644 --- a/overlay/tun_windows.go +++ b/overlay/tun_windows.go @@ -75,7 +75,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w t := &winTun{ Device: deviceName, vpnNetworks: vpnNetworks, - MTU: c.GetInt("tun.mtu", DefaultMTU), + MTU: getMTU(l, c), guid: *guid, networkCategory: cat, setCategory: setCat, @@ -118,7 +118,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w } func (t *winTun) reload(c *config.C, initial bool) error { - change, routes, err := getAllRoutesFromConfig(c, t.vpnNetworks, initial) + change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) if err != nil { return err } diff --git a/udp/udp_rio_windows.go b/udp/udp_rio_windows.go index f4b96b75..435d71a6 100644 --- a/udp/udp_rio_windows.go +++ b/udp/udp_rio_windows.go @@ -30,7 +30,7 @@ func procyield(cycles uint32) const ( packetsPerRing = 1024 - // Caps tun.mtu at MTU-32 direct, MTU-64 relayed, unenforced anywhere else. 17.6MB page locked per socket. + // Fits a relayed packet at overlay.MaxMTU, which tun.mtu and route mtus are capped to. 17.6MB page locked per socket. bytesPerPacket = MTU receiveSpins = 15 ) From a8c52f5c13e89b6ac66581ca289ec1c51cc6e9c9 Mon Sep 17 00:00:00 2001 From: Nate Brown Date: Thu, 8 Oct 2026 18:52:53 -0500 Subject: [PATCH 20/31] overlay: Handle routes Windows already has, and parse quoted route numbers (#1924) --- overlay/route.go | 32 +++++- overlay/route_test.go | 73 +++++++++++++ overlay/tun_windows.go | 79 +++++++++++++- overlay/tun_windows_route_test.go | 168 ++++++++++++++++++++++++++++++ 4 files changed, 344 insertions(+), 8 deletions(-) create mode 100644 overlay/tun_windows_route_test.go diff --git a/overlay/route.go b/overlay/route.go index c6403f91..6d32c54b 100644 --- a/overlay/route.go +++ b/overlay/route.go @@ -50,7 +50,13 @@ func (r Route) String() string { func makeRouteTree(l *slog.Logger, routes []Route, allowMTU bool) (*bart.Table[routing.Gateways], error) { routeTree := new(bart.Table[routing.Gateways]) + seen := make(map[netip.Prefix]bool, len(routes)) for _, r := range routes { + if seen[r.Cidr] { + l.Warn("route is listed more than once, only the last entry's via is used", "route", r) + } + seen[r.Cidr] = true + if !allowMTU && r.MTU > 0 { l.Warn("route MTU is not supported on this platform", "goos", runtime.GOOS, @@ -98,7 +104,11 @@ func parseRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) { mtu, ok := rMtu.(int) if !ok { - mtu, err = strconv.Atoi(rMtu.(string)) + s, ok := rMtu.(string) + if !ok { + return nil, fmt.Errorf("entry %v.mtu in tun.routes is not an integer: %v", i+1, rMtu) + } + mtu, err = strconv.Atoi(s) if err != nil { return nil, fmt.Errorf("entry %v.mtu in tun.routes is not an integer: %v", i+1, err) } @@ -174,7 +184,11 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) { if rMtu, ok := m["mtu"]; ok { mtu, ok = rMtu.(int) if !ok { - mtu, err = strconv.Atoi(rMtu.(string)) + s, ok := rMtu.(string) + if !ok { + return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, rMtu) + } + mtu, err = strconv.Atoi(s) if err != nil { return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, err) } @@ -192,10 +206,15 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) { metric, ok := rMetric.(int) if !ok { - _, err = strconv.ParseInt(rMetric.(string), 10, 32) + s, ok := rMetric.(string) + if !ok { + return nil, fmt.Errorf("entry %v.metric in tun.unsafe_routes is not an integer: %v", i+1, rMetric) + } + n, err := strconv.ParseInt(s, 10, 32) if err != nil { return nil, fmt.Errorf("entry %v.metric in tun.unsafe_routes is not an integer: %v", i+1, err) } + metric = int(n) } if metric < 0 || metric > math.MaxInt32 { @@ -248,10 +267,15 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) { gatewayWeight, ok := rGatewayWeight.(int) if !ok { - _, err = strconv.ParseInt(rGatewayWeight.(string), 10, 32) + s, ok := rGatewayWeight.(string) + if !ok { + return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1) + } + w, err := strconv.ParseInt(s, 10, 32) if err != nil { return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1) } + gatewayWeight = int(w) } if gatewayWeight < 1 || gatewayWeight > math.MaxInt32 { diff --git a/overlay/route_test.go b/overlay/route_test.go index 8d62f0b0..8a8cb4cf 100644 --- a/overlay/route_test.go +++ b/overlay/route_test.go @@ -1,8 +1,10 @@ package overlay import ( + "bytes" "fmt" "net/netip" + "strings" "testing" "github.com/slackhq/nebula/config" @@ -445,3 +447,74 @@ func Test_getAllRoutesFromConfigCapsMTU(t *testing.T) { } assert.Equal(t, map[string]int{"10.0.0.0/29": MaxMTU, "10.0.0.8/29": 1300, "1.0.0.0/29": MaxMTU}, mtus) } + +// The route tree keeps one entry per CIDR, so a CIDR listed twice only routes by the last one. Say so instead of +// letting the earlier via, or on Windows the earlier metric, quietly lose +func Test_makeRouteTree_WarnsOnDuplicate(t *testing.T) { + buf := &bytes.Buffer{} + l := test.NewLoggerWithOutput(buf) + via := func(s string) routing.Gateways { + return routing.Gateways{routing.NewGateway(netip.MustParseAddr(s), 1)} + } + cidr := netip.MustParsePrefix("10.251.1.0/24") + + _, err := makeRouteTree(l, []Route{ + {Cidr: cidr, Via: via("10.250.0.2"), Install: true, Metric: 100}, + {Cidr: netip.MustParsePrefix("10.251.2.0/24"), Via: via("10.250.0.2"), Install: true}, + }, false) + require.NoError(t, err) + assert.Empty(t, buf.String()) + + tree, err := makeRouteTree(l, []Route{ + {Cidr: cidr, Via: via("10.250.0.2"), Install: true, Metric: 100}, + {Cidr: cidr, Via: via("10.250.0.3"), Install: true, Metric: 200}, + }, false) + require.NoError(t, err) + assert.Equal(t, 1, strings.Count(buf.String(), "route is listed more than once, only the last entry's via is used")) + gw, ok := tree.Lookup(netip.MustParseAddr("10.251.1.1")) + require.True(t, ok) + assert.Equal(t, netip.MustParseAddr("10.250.0.3"), gw[0].Addr(), "the last entry wins") +} + +// A quoted metric or weight is a number written as a string, it means that number. Anything else is an error, not a +// panic +func Test_parseUnsafeRoutes_QuotedNumbers(t *testing.T) { + c := config.NewC(test.NewLogger()) + n := netip.MustParsePrefix("10.0.0.1/24") + route := func(metric, weight any) { + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{ + "route": "1.0.0.0/8", + "metric": metric, + "via": []any{map[string]any{"gateway": "10.0.0.2", "weight": weight}}, + }}} + } + + route("100", "5") + routes, err := parseUnsafeRoutes(c, []netip.Prefix{n}) + require.NoError(t, err) + require.Len(t, routes, 1) + assert.Equal(t, 100, routes[0].Metric) + assert.Equal(t, "{addr: 10.0.0.2, weight: 5}", routes[0].Via[0].String()) + + route(1.5, 1) + _, err = parseUnsafeRoutes(c, []netip.Prefix{n}) + require.EqualError(t, err, "entry 1.metric in tun.unsafe_routes is not an integer: 1.5") + + route(1, true) + _, err = parseUnsafeRoutes(c, []netip.Prefix{n}) + require.EqualError(t, err, "entry .weight in tun.unsafe_routes[1].via[1] is not an integer") +} + +// An mtu that is neither a number nor a quoted one is an error, not a panic +func Test_parseRoutes_MTUNotANumber(t *testing.T) { + c := config.NewC(test.NewLogger()) + n := netip.MustParsePrefix("10.0.0.1/24") + + c.Settings["tun"] = map[string]any{"routes": []any{map[string]any{"route": "1.0.0.0/8", "mtu": 1500.5}}} + _, err := parseRoutes(c, []netip.Prefix{n}) + require.EqualError(t, err, "entry 1.mtu in tun.routes is not an integer: 1500.5") + + c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"route": "1.0.0.0/8", "via": "10.0.0.2", "mtu": 1500.5}}} + _, err = parseUnsafeRoutes(c, []netip.Prefix{n}) + require.EqualError(t, err, "entry 1.mtu in tun.unsafe_routes is not an integer: 1500.5") +} diff --git a/overlay/tun_windows.go b/overlay/tun_windows.go index 982b8fa0..18680f2b 100644 --- a/overlay/tun_windows.go +++ b/overlay/tun_windows.go @@ -4,6 +4,7 @@ package overlay import ( "crypto" + "errors" "fmt" "log/slog" "net/netip" @@ -138,7 +139,7 @@ func (t *winTun) reload(c *config.C, initial bool) error { if !initial { // Remove first, if the system removes a wanted route hopefully it will be re-added next - err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) + err := t.removeRoutes(findRemovedSystemRoutes(routes, *oldRoutes)) if err != nil { util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) } @@ -187,17 +188,27 @@ func (t *winTun) addRoutes(logErrors bool) error { foundDefault4 := false carriesV6 := slices.ContainsFunc(t.vpnNetworks, func(p netip.Prefix) bool { return p.Addr().Is6() }) - for _, r := range routes { + // A CIDR listed twice routes by its last entry, install only that one or each reload would flip between them. The + // same rule as systemRoutes + last := make(map[netip.Prefix]int, len(routes)) + for i, r := range routes { + last[r.Cidr] = i + } + + for i, r := range routes { if len(r.Via) == 0 || !r.Install { // We don't allow route MTUs so only install routes with a via continue } + if last[r.Cidr] != i { + continue + } // A v6 unsafe_route is legal under a v4-only cert; uninstalled ones put nothing on the adapter. carriesV6 = carriesV6 || r.Cidr.Addr().Is6() // Add our unsafe route as an on-link route to the nebula tun device. - err := luid.AddRoute(r.Cidr, unspecifiedNextHop(r.Cidr), uint32(r.Metric)) + result, err := addRoute(luid, r) if err != nil { retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { @@ -207,7 +218,12 @@ func (t *winTun) addRoutes(logErrors bool) error { return retErr } } else { - t.l.Info("Added route", "route", r) + switch result { + case routeAdded: + t.l.Info("Added route", "route", r) + case routeMetricUpdated: + t.l.Info("Updated route metric", "route", r) + } } if !foundDefault4 { @@ -319,6 +335,61 @@ func (t *winTun) Close() error { return t.tun.Close() } +// systemRoutes maps each CIDR to the route that puts it in the system table. A CIDR's last entry decides, including +// one that says not to install it +func systemRoutes(routes []Route) map[netip.Prefix]Route { + out := make(map[netip.Prefix]Route, len(routes)) + for _, r := range routes { + if len(r.Via) > 0 && r.Install { + out[r.Cidr] = r + } else { + delete(out, r.Cidr) + } + } + return out +} + +// findRemovedSystemRoutes is the routes to delete from the system table. Windows deletes by destination and next hop, +// so only a CIDR nothing installs anymore goes, a changed metric is set on the route in place by addRoute +func findRemovedSystemRoutes(newRoutes, oldRoutes []Route) []Route { + keep := systemRoutes(newRoutes) + var removed []Route + for _, r := range systemRoutes(oldRoutes) { + if _, ok := keep[r.Cidr]; !ok { + removed = append(removed, r) + } + } + return removed +} + +type routeResult int + +const ( + routeAdded routeResult = iota + routeMetricUpdated + routeAlreadyInstalled +) + +// addRoute installs r on the interface. A reload re-adds every route to catch any the system dropped, and Windows +// refuses one it has with the same destination and next hop whatever its metric. One that is there with another +// metric, ours from before a metric change or someone else's, is brought to ours in place +func addRoute(luid winipcfg.LUID, r Route) (routeResult, error) { + nextHop := unspecifiedNextHop(r.Cidr) + err := luid.AddRoute(r.Cidr, nextHop, uint32(r.Metric)) + if !errors.Is(err, windows.ERROR_OBJECT_ALREADY_EXISTS) { + return routeAdded, err + } + row, err := luid.Route(r.Cidr, nextHop) + if err != nil { + return routeAlreadyInstalled, err + } + if row.Metric != uint32(r.Metric) { + row.Metric = uint32(r.Metric) + return routeMetricUpdated, row.Set() + } + return routeAlreadyInstalled, nil +} + func unspecifiedNextHop(p netip.Prefix) netip.Addr { if p.Addr().Is4() { return netip.IPv4Unspecified() diff --git a/overlay/tun_windows_route_test.go b/overlay/tun_windows_route_test.go new file mode 100644 index 00000000..71c5c232 --- /dev/null +++ b/overlay/tun_windows_route_test.go @@ -0,0 +1,168 @@ +//go:build windows && !e2e_testing + +package overlay + +import ( + "bytes" + "fmt" + "net/netip" + "strings" + "testing" + + "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/windows" + "golang.zx2c4.com/wireguard/windows/tunnel/winipcfg" +) + +type routeTest struct { + c *config.C + tun *winTun + luid winipcfg.LUID + log *bytes.Buffer +} + +// newRouteTest makes a wintun device of its own, before installing tun.unsafe_routes from cfg +func newRouteTest(t *testing.T, cfg string) *routeTest { + t.Helper() + if !windows.GetCurrentProcessToken().IsElevated() { + t.Skip("creating a wintun device needs an elevated process") + } + if err := checkWinTunExists(); err != nil { + t.Skipf("wintun.dll is not where nebula looks for it: %v", err) + } + rt := &routeTest{log: &bytes.Buffer{}} + l := test.NewLoggerWithOutput(rt.log) + rt.c = config.NewC(l) + require.NoError(t, rt.c.LoadString(cfg)) + var err error + rt.tun, err = newTun(rt.c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false) + require.NoError(t, err) + t.Cleanup(func() { _ = rt.tun.Close() }) + rt.luid = winipcfg.LUID(rt.tun.tun.LUID()) + return rt +} + +func (rt *routeTest) metric(t *testing.T, cidr string) uint32 { + t.Helper() + p := netip.MustParsePrefix(cidr) + row, err := rt.luid.Route(p, unspecifiedNextHop(p)) + require.NoError(t, err) + return row.Metric +} + +func unsafeRoutes(routes ...string) string { + var b strings.Builder + b.WriteString("tun:\n dev: nebroutetest\n unsafe_routes:\n") + // cidr[@metric][!], the ! for install: false + for _, r := range routes { + r, noInstall := strings.CutSuffix(r, "!") + cidr, metric, _ := strings.Cut(r, "@") + fmt.Fprintf(&b, " - route: %s\n via: 10.250.0.2\n", cidr) + if metric != "" { + fmt.Fprintf(&b, " metric: %s\n", metric) + } + if noInstall { + b.WriteString(" install: false\n") + } + } + return b.String() +} + +// A reload that adds a route re-adds the ones already installed, Windows refusing those as already there isn't a failure +func TestWinTun_ReloadKeepsInstalledRoutes(t *testing.T) { + forFamilies(t, func(t *testing.T, a, b string) { + rt := newRouteTest(t, unsafeRoutes(a)) + require.NoError(t, rt.tun.Activate()) + + require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a, b))) + assert.NotContains(t, rt.log.String(), "Failed to add route") + rt.metric(t, a) + rt.metric(t, b) + }) +} + +// Windows calls a route the same by destination and next hop, so a changed metric is set on the installed route in +// place rather than deleted and added, which would leave a moment with no route +func TestWinTun_ReloadMetricChangeApplies(t *testing.T) { + forFamilies(t, func(t *testing.T, a, b string) { + rt := newRouteTest(t, unsafeRoutes(a+"@100")) + require.NoError(t, rt.tun.Activate()) + require.Equal(t, uint32(100), rt.metric(t, a)) + + require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@200"))) + assert.NotContains(t, rt.log.String(), "Failed to add route") + assert.NotContains(t, rt.log.String(), "Removed route") + assert.Equal(t, uint32(200), rt.metric(t, a)) + }) +} + +// A route someone else put on the device with another metric is brought to ours, at startup too +func TestWinTun_ForeignRouteGetsOurMetric(t *testing.T) { + forFamilies(t, func(t *testing.T, a, b string) { + rt := newRouteTest(t, unsafeRoutes(a+"@100")) + p := netip.MustParsePrefix(a) + require.NoError(t, rt.luid.AddRoute(p, unspecifiedNextHop(p), 50)) + + require.NoError(t, rt.tun.Activate()) + assert.Equal(t, uint32(100), rt.metric(t, a)) + assert.Contains(t, rt.log.String(), "level=INFO msg=\"Updated route metric\"") + }) +} + +// A CIDR listed twice is installed once, by its last entry, and reloads leave it there instead of flipping between the +// two metrics +func TestWinTun_DuplicateCidrInstallsTheLastEntry(t *testing.T) { + forFamilies(t, func(t *testing.T, a, b string) { + rt := newRouteTest(t, unsafeRoutes(a+"@100", a+"@200")) + require.NoError(t, rt.tun.Activate()) + assert.Equal(t, uint32(200), rt.metric(t, a)) + + require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@100", a+"@200", b))) + assert.Equal(t, uint32(200), rt.metric(t, a)) + assert.NotContains(t, rt.log.String(), "Updated route metric") + assert.NotContains(t, rt.log.String(), "Failed to add route") + }) +} + +// Dropping the duplicate that wasn't installed leaves the installed route alone. Windows deletes by destination and +// next hop, deleting the dropped one would take the live route with it +func TestWinTun_DroppingAnUninstalledDuplicateKeepsTheRoute(t *testing.T) { + forFamilies(t, func(t *testing.T, a, b string) { + rt := newRouteTest(t, unsafeRoutes(a+"@100", a+"@200")) + require.NoError(t, rt.tun.Activate()) + + require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@200"))) + assert.NotContains(t, rt.log.String(), "Removed route") + assert.Equal(t, uint32(200), rt.metric(t, a)) + }) +} + +// A reload ends where a fresh start with the same config would, a CIDR whose last entry says not to install it is +// removed +func TestWinTun_ReloadMatchesAFreshStart(t *testing.T) { + forFamilies(t, func(t *testing.T, a, b string) { + cidr := netip.MustParsePrefix(a) + routes := unsafeRoutes(a+"@100", a+"@100!") + + fresh := newRouteTest(t, routes) + require.NoError(t, fresh.tun.Activate()) + _, err := fresh.luid.Route(cidr, unspecifiedNextHop(cidr)) + require.Error(t, err, "a fresh start installed it") + require.NoError(t, fresh.tun.Close()) + + rt := newRouteTest(t, unsafeRoutes(a+"@100")) + require.NoError(t, rt.tun.Activate()) + require.NoError(t, rt.c.ReloadConfigString(routes)) + _, err = rt.luid.Route(cidr, unspecifiedNextHop(cidr)) + assert.Error(t, err, "the reload left it installed") + }) +} + +// forFamilies runs f with two unsafe_routes CIDRs of each address family, the tun's own network stays IPv4 +func forFamilies(t *testing.T, f func(t *testing.T, a, b string)) { + t.Run("ipv4", func(t *testing.T) { f(t, "10.251.1.0/24", "10.251.2.0/24") }) + t.Run("ipv6", func(t *testing.T) { f(t, "fd00:251:1::/64", "fd00:251:2::/64") }) +} From 8a2de9b1b58d102d1967f62552f3194596236154 Mon Sep 17 00:00:00 2001 From: Ryan Huber Date: Sun, 11 Oct 2026 00:30:07 -0400 Subject: [PATCH 21/31] examples: set socket buffers and raise tun tx_queue (#1928) * examples: set socket buffers and raise tun tx_queue * examples: use the default tcp max window for socket buffers * examples: note buffers are linux only, clarify tx_queue default * tun: default tx_queue to 5000 --- examples/config.yml | 10 +++++----- overlay/tun.go | 2 +- overlay/tun_linux.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/config.yml b/examples/config.yml index 987ef90e..0d1a9180 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -135,12 +135,12 @@ listen: # superpacket, so the receive scratch is batch * 64KiB per listening socket (~4MiB per routine at the # default of 64). Lower this to trade peak per-syscall throughput for memory on constrained hosts. #batch: 64 - # Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel + # Linux only. Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel # Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default) # Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide # max, net.core.rmem_max and net.core.wmem_max - #read_buffer: 10485760 - #write_buffer: 10485760 + read_buffer: 3412512 + write_buffer: 3412512 # On Windows only # When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port. @@ -262,8 +262,8 @@ tun: drop_local_broadcast: false # Toggles forwarding of multicast packets drop_multicast: false - # Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 500 - tx_queue: 500 + # Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 5000 + tx_queue: 5000 # Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic # Anything above 8937, here or on a route below, is capped to 8937 with a warning mtu: 1300 diff --git a/overlay/tun.go b/overlay/tun.go index eb994341..0ba130b6 100644 --- a/overlay/tun.go +++ b/overlay/tun.go @@ -43,7 +43,7 @@ type DeviceFactory func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) { switch { case c.GetBool("tun.disabled", false): - tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l) + tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 5000), c.GetBool("stats.message_metrics", false), l) return tun, nil default: diff --git a/overlay/tun_linux.go b/overlay/tun_linux.go index b5e0a675..5e27ad9b 100644 --- a/overlay/tun_linux.go +++ b/overlay/tun_linux.go @@ -221,7 +221,7 @@ func newTunGeneric(c *config.C, l *slog.Logger, fd int, vnetHdr bool, offloadFla vnetHdr: vnetHdr, offloadFlags: offloadFlags, vpnNetworks: vpnNetworks, - TXQueueLen: c.GetInt("tun.tx_queue", 500), + TXQueueLen: c.GetInt("tun.tx_queue", 5000), useSystemRoutes: c.GetBool("tun.use_system_route_table", false), useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0), routesFromSystem: map[netip.Prefix]routing.Gateways{}, From 5ba7aff4b3996179e9f9d7cd8187ecb9940bfb0b Mon Sep 17 00:00:00 2001 From: Nate Brown Date: Mon, 12 Oct 2026 17:56:51 -0400 Subject: [PATCH 22/31] ssh: Only restart sshd on reload when its config or host key changed (#1923) --- ssh.go | 46 ++++++++--- sshd/server.go | 61 +++++++++++--- sshd_reload_test.go | 196 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 277 insertions(+), 26 deletions(-) create mode 100644 sshd_reload_test.go diff --git a/ssh.go b/ssh.go index 2a440d98..34b5c3a7 100644 --- a/ssh.go +++ b/ssh.go @@ -60,6 +60,20 @@ type sshDeviceInfoFlags struct { func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) { c.RegisterReloadCallback(func(c *config.C) { if c.GetBool("sshd.enabled", false) { + // Restarting the server drops every open session, the one that asked for this reload included. Only restart + // when it isn't running or its config changed. The host key file can change under the same config, new + // handshakes use the new key without a restart + if ssh.Running() && !c.HasChanged("sshd") { + key, err := loadSSHHostKey(c) + if err == nil { + err = ssh.SetHostKey(key) + } + if err != nil { + l.Error("Failed to reconfigure the sshd", "error", err) + ssh.Stop() + } + return + } sshRun, err := configSSH(l, ssh, c) if err != nil { l.Error("Failed to reconfigure the sshd", "error", err) @@ -74,6 +88,22 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) { }) } +// loadSSHHostKey returns sshd.host_key, read from its file when it names one. +func loadSSHHostKey(c *config.C) ([]byte, error) { + hostKeyPathOrKey := c.GetString("sshd.host_key", "") + if hostKeyPathOrKey == "" { + return nil, fmt.Errorf("sshd.host_key must be provided") + } + if strings.Contains(hostKeyPathOrKey, "-----BEGIN") { + return []byte(hostKeyPathOrKey), nil + } + b, err := os.ReadFile(hostKeyPathOrKey) + if err != nil { + return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err) + } + return b, nil +} + // configSSH reads the ssh info out of the passed-in Config and // updates the passed-in SSHServer. On success, it returns a function // that callers may invoke to run the configured ssh server. On @@ -92,19 +122,9 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error) return nil, fmt.Errorf("sshd.listen can not use port 22") } - hostKeyPathOrKey := c.GetString("sshd.host_key", "") - if hostKeyPathOrKey == "" { - return nil, fmt.Errorf("sshd.host_key must be provided") - } - - var hostKeyBytes []byte - if strings.Contains(hostKeyPathOrKey, "-----BEGIN") { - hostKeyBytes = []byte(hostKeyPathOrKey) - } else { - hostKeyBytes, err = os.ReadFile(hostKeyPathOrKey) - if err != nil { - return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err) - } + hostKeyBytes, err := loadSSHHostKey(c) + if err != nil { + return nil, err } err = ssh.SetHostKey(hostKeyBytes) diff --git a/sshd/server.go b/sshd/server.go index e0ee9364..b5e1aa8f 100644 --- a/sshd/server.go +++ b/sshd/server.go @@ -8,14 +8,18 @@ import ( "log/slog" "net" "sync" + "sync/atomic" "github.com/armon/go-radix" "golang.org/x/crypto/ssh" ) type SSHServer struct { - config *ssh.ServerConfig - l *slog.Logger + // config is replaced whole when the host key changes, ssh.ServerConfig.AddHostKey only replaces a key of the same + // algorithm and appends any other + config atomic.Pointer[ssh.ServerConfig] + authenticate func(ssh.ConnMetadata, ssh.PublicKey) (*ssh.Permissions, error) + l *slog.Logger certChecker *ssh.CertChecker @@ -28,7 +32,12 @@ type SSHServer struct { // List of available commands helpCommand *Command commands *radix.Tree - listener net.Listener + listener atomic.Pointer[net.Listener] + + // runs counts the Runs that are listening. A reload's Run can start before the one it replaced has returned + runs atomic.Int32 + // addr is where the newest Run listens, nil when none is + addr atomic.Pointer[net.Addr] // ctx parents per-Run contexts. Cancelling it (e.g. via Control.Stop) tears the server down even // across reloads, since each Run derives a fresh child rather than reusing this one directly. @@ -85,10 +94,8 @@ func NewSSHServer(ctx context.Context, l *slog.Logger) (*SSHServer, error) { }, } - s.config = &ssh.ServerConfig{ - PublicKeyCallback: cc.Authenticate, - ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"), - } + s.authenticate = cc.Authenticate + s.config.Store(s.newConfig()) s.RegisterCommand(&Command{ Name: "help", @@ -107,10 +114,32 @@ func (s *SSHServer) SetHostKey(hostPrivateKey []byte) error { return fmt.Errorf("failed to parse private key: %s", err) } - s.config.AddHostKey(private) + config := s.newConfig() + config.AddHostKey(private) + s.config.Store(config) return nil } +func (s *SSHServer) newConfig() *ssh.ServerConfig { + return &ssh.ServerConfig{ + PublicKeyCallback: s.authenticate, + ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"), + } +} + +// Addr is the address the newest Run listens on, nil when none is. +func (s *SSHServer) Addr() net.Addr { + if a := s.addr.Load(); a != nil { + return *a + } + return nil +} + +// Running is true while a Run is listening. +func (s *SSHServer) Running() bool { + return s.runs.Load() > 0 +} + func (s *SSHServer) ClearTrustedCAs() { s.authLock.Lock() s.trustedCAs = []ssh.PublicKey{} @@ -181,10 +210,16 @@ func (s *SSHServer) Run(addr string) error { // this run owns. They start equal but a fast reload may overwrite s.listener with the next run's // listener before this run's watcher fires, so each run must close its own listener via the local // reference. - s.listener = listener + s.listener.Store(&listener) + s.runs.Add(1) + bound := listener.Addr() + s.addr.Store(&bound) runCtx, cancel := context.WithCancel(s.ctx) defer cancel() + // Deferred after cancel so they run first, a run on its way out no longer counts as listening + defer s.runs.Add(-1) + defer s.addr.CompareAndSwap(&bound, nil) // Close the listener when this run's context is cancelled. That can come from the parent // (Control.Stop), from Run returning normally (defer cancel above), or transitively when a sibling @@ -197,7 +232,7 @@ func (s *SSHServer) Run(addr string) error { } }() - s.l.Info("SSH server is listening", "sshListener", addr) + s.l.Info("SSH server is listening", "sshListener", bound) // Run loops until there is an error s.run(runCtx, listener) @@ -226,7 +261,7 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) { <-sessionContext.Done() c.Close() }() - conn, chans, reqs, err := ssh.NewServerConn(c, s.config) + conn, chans, reqs, err := ssh.NewServerConn(c, s.config.Load()) fp := "" if conn != nil { fp = conn.Permissions.Extensions["fp"] @@ -264,8 +299,8 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) { } func (s *SSHServer) Stop() { - if s.listener != nil { - if err := s.listener.Close(); err != nil { + if l := s.listener.Load(); l != nil { + if err := (*l).Close(); err != nil && !errors.Is(err, net.ErrClosed) { s.l.Warn("Failed to close the sshd listener", "error", err) } } diff --git a/sshd_reload_test.go b/sshd_reload_test.go new file mode 100644 index 00000000..8226fd09 --- /dev/null +++ b/sshd_reload_test.go @@ -0,0 +1,196 @@ +package nebula + +import ( + "crypto" + "crypto/ed25519" + "crypto/rand" + "crypto/rsa" + "encoding/pem" + "fmt" + "net" + "os" + "path/filepath" + "testing" + "time" + + "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/sshd" + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/crypto/ssh" +) + +type sshdReloadHarness struct { + c *config.C + server *sshd.SSHServer + keyFile string + signer ssh.Signer + extra string +} + +func newSSHDReloadHarness(t *testing.T) *sshdReloadHarness { + t.Helper() + l := test.NewLogger() + h := &sshdReloadHarness{keyFile: filepath.Join(t.TempDir(), "host_key")} + h.writeHostKey(t, newEd25519Key(t)) + + _, userKey, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + h.signer, err = ssh.NewSignerFromKey(userKey) + require.NoError(t, err) + + h.c = config.NewC(l) + require.NoError(t, h.c.LoadString(h.config())) + h.server, err = sshd.NewSSHServer(t.Context(), l) + require.NoError(t, err) + wireSSHReload(l, h.server, h.c) + run, err := configSSH(l, h.server, h.c) + require.NoError(t, err) + go run() + t.Cleanup(h.server.Stop) + require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond) + return h +} + +func newEd25519Key(t *testing.T) crypto.Signer { + _, key, err := ed25519.GenerateKey(rand.Reader) + require.NoError(t, err) + return key +} + +// writeHostKey replaces the host key file and returns the public key a client should now see +func (h *sshdReloadHarness) writeHostKey(t *testing.T, key crypto.Signer) ssh.PublicKey { + t.Helper() + block, err := ssh.MarshalPrivateKey(key, "") + require.NoError(t, err) + require.NoError(t, os.WriteFile(h.keyFile, pem.EncodeToMemory(block), 0o600)) + pub, err := ssh.NewPublicKey(key.Public()) + require.NoError(t, err) + return pub +} + +func (h *sshdReloadHarness) config() string { + return fmt.Sprintf("sshd:\n enabled: true\n listen: 127.0.0.1:0\n host_key: %s\n authorized_users:\n - user: test\n keys: [%q]\n%s", + h.keyFile, string(ssh.MarshalAuthorizedKey(h.signer.PublicKey())), h.extra) +} + +// dial connects to wherever the server listens now, a restart on :0 moves it. hostKeyAlgorithms limits which host keys +// the client accepts, none means any +func (h *sshdReloadHarness) dial(hostKeyAlgorithms ...string) (*ssh.Client, ssh.PublicKey, error) { + addr := h.server.Addr() + if addr == nil { + return nil, nil, fmt.Errorf("not listening") + } + var hostKey ssh.PublicKey + c, err := ssh.Dial("tcp", addr.String(), &ssh.ClientConfig{ + User: "test", + Auth: []ssh.AuthMethod{ssh.PublicKeys(h.signer)}, + HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error { + hostKey = key + return nil + }, + HostKeyAlgorithms: hostKeyAlgorithms, + Timeout: time.Second, + }) + return c, hostKey, err +} + +func (h *sshdReloadHarness) connect(t *testing.T) (*ssh.Client, ssh.PublicKey) { + t.Helper() + var client *ssh.Client + var hostKey ssh.PublicKey + require.Eventually(t, func() bool { + var err error + client, hostKey, err = h.dial() + return err == nil + }, 5*time.Second, 20*time.Millisecond) + t.Cleanup(func() { _ = client.Close() }) + return client, hostKey +} + +// dropped is true when the server closed the client's connection within wait +func dropped(client *ssh.Client, wait time.Duration) bool { + done := make(chan struct{}) + go func() { + _ = client.Wait() + close(done) + }() + select { + case <-done: + return true + case <-time.After(wait): + return false + } +} + +// Restarting sshd drops every open session, the one that asked for the reload included. A reload that touches nothing +// sshd reads leaves it alone +func TestSSHDReload_UnchangedKeepsSessions(t *testing.T) { + h := newSSHDReloadHarness(t) + client, _ := h.connect(t) + + require.NoError(t, h.c.ReloadConfigString(h.config()+"punchy:\n punch: true\n")) + assert.False(t, dropped(client, 500*time.Millisecond), "an unchanged sshd was restarted") +} + +func TestSSHDReload_ChangedConfigRestarts(t *testing.T) { + h := newSSHDReloadHarness(t) + client, _ := h.connect(t) + + h.extra = " trusted_cas: []\n" + require.NoError(t, h.c.ReloadConfigString(h.config())) + assert.True(t, dropped(client, 5*time.Second), "a changed sshd config was not applied") + require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond) + h.connect(t) +} + +// The config can stay the same while the host key file it names changes. New connections get the new key, open +// sessions are left alone +func TestSSHDReload_ChangedHostKeyFile(t *testing.T) { + h := newSSHDReloadHarness(t) + client, _ := h.connect(t) + + want := h.writeHostKey(t, newEd25519Key(t)) + require.NoError(t, h.c.ReloadConfigString(h.config())) + _, got := h.connect(t) + assert.Equal(t, want.Marshal(), got.Marshal(), "a new host key was not applied") + assert.False(t, dropped(client, 500*time.Millisecond), "a new host key restarted sshd") +} + +// A host key of another algorithm replaces the old one rather than being offered next to it +func TestSSHDReload_HostKeyAlgorithmChange(t *testing.T) { + h := newSSHDReloadHarness(t) + h.connect(t) + + rsaKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + want := h.writeHostKey(t, rsaKey) + require.NoError(t, h.c.ReloadConfigString(h.config())) + + _, got := h.connect(t) + assert.Equal(t, want.Marshal(), got.Marshal()) + _, _, err = h.dial(ssh.KeyAlgoED25519) + assert.Error(t, err, "the old ed25519 host key is still offered") +} + +// A server that isn't running, say its port was busy, is tried again on every reload as before +func TestSSHDReload_NotRunningRestarts(t *testing.T) { + h := newSSHDReloadHarness(t) + h.server.Stop() + require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond) + + require.NoError(t, h.c.ReloadConfigString(h.config())) + require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond) + h.connect(t) +} + +// A host key that no longer loads stops sshd, the same as a config that doesn't +func TestSSHDReload_BadHostKeyFileStops(t *testing.T) { + h := newSSHDReloadHarness(t) + h.connect(t) + + require.NoError(t, os.WriteFile(h.keyFile, []byte("not a key"), 0o600)) + require.NoError(t, h.c.ReloadConfigString(h.config())) + require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond) +} From 11c1917387d251580667764261a708c3dd7fef34 Mon Sep 17 00:00:00 2001 From: Nate Brown Date: Mon, 12 Oct 2026 17:57:14 -0400 Subject: [PATCH 23/31] Warn when an unsafe_route is already routed through another interface on darwin (#1925) --- overlay/tun_darwin.go | 82 +++++++++++++++++++++++++++++--- overlay/tun_darwin_route_test.go | 75 +++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+), 7 deletions(-) create mode 100644 overlay/tun_darwin_route_test.go diff --git a/overlay/tun_darwin.go b/overlay/tun_darwin.go index b95a93cf..a0834590 100644 --- a/overlay/tun_darwin.go +++ b/overlay/tun_darwin.go @@ -6,8 +6,10 @@ import ( "errors" "fmt" "log/slog" + "math/bits" "net/netip" "os" + "sync" "sync/atomic" "syscall" "unsafe" @@ -28,7 +30,7 @@ type tun struct { DefaultMTU int Routes atomic.Pointer[[]Route] routeTree atomic.Pointer[bart.Table[routing.Gateways]] - linkAddr *netroute.LinkAddr + linkAddr atomic.Pointer[netroute.LinkAddr] // hostOwned means the fd arrived from the OS, which has already configured addressing, mtu // and routes for it. NEPacketTunnelProvider on darwin does this. hostOwned bool @@ -243,7 +245,7 @@ func (t *tun) Activate() error { if linkAddr == nil { return fmt.Errorf("unable to discover link_addr for tun interface") } - t.linkAddr = linkAddr + t.linkAddr.Store(linkAddr) for _, network := range t.vpnNetworks { if network.Addr().Is4() { @@ -303,7 +305,7 @@ func (t *tun) activate4(network netip.Prefix) error { return fmt.Errorf("failed to set tun v4 address: %s", err) } - err = addRoute(network, t.linkAddr) + err = addRoute(network, t.linkAddr.Load()) if err != nil { return err } @@ -394,6 +396,56 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways { return routing.Gateways{} } +func fetchRoutes() ([]netroute.Message, error) { + rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_DUMP, 0) + if err != nil { + return nil, err + } + return netroute.ParseRIB(unix.NET_RT_DUMP, rib) +} + +// routeInterfaceIn returns the index of the interface prefix is routed through, or 0 if msgs has no route for exactly +// that prefix. Routes scoped to an interface are skipped, they don't stop us adding ours. +func routeInterfaceIn(msgs []netroute.Message, prefix netip.Prefix) int { + for _, msg := range msgs { + m, ok := msg.(*netroute.RouteMessage) + if !ok || len(m.Addrs) <= unix.RTAX_NETMASK || m.Flags&unix.RTF_IFSCOPE != 0 { + continue + } + if p, ok := routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK]); ok && p == prefix.Masked() { + return m.Index + } + } + return 0 +} + +// routePrefix returns the prefix a route covers. Host routes have no netmask. +func routePrefix(dst, mask netroute.Addr) (netip.Prefix, bool) { + switch d := dst.(type) { + case *netroute.Inet4Addr: + bits := 32 + if m, ok := mask.(*netroute.Inet4Addr); ok { + bits = maskBits(m.IP[:]) + } + return netip.PrefixFrom(netip.AddrFrom4(d.IP), bits), true + case *netroute.Inet6Addr: + bits := 128 + if m, ok := mask.(*netroute.Inet6Addr); ok { + bits = maskBits(m.IP[:]) + } + return netip.PrefixFrom(netip.AddrFrom16(d.IP), bits), true + } + return netip.Prefix{}, false +} + +func maskBits(mask []byte) int { + n := 0 + for _, b := range mask { + n += bits.OnesCount8(b) + } + return n +} + // Get the LinkAddr for the interface of the given name // Is there an easier way to fetch this when we create the interface? // Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers. @@ -428,7 +480,14 @@ func (t *tun) addRoutes(logErrors bool) error { return nil } + // Activate hasn't run yet, it will install the routes + linkAddr := t.linkAddr.Load() + if linkAddr == nil { + return nil + } + routes := *t.Routes.Load() + systemRoutes := sync.OnceValues(fetchRoutes) for _, r := range routes { if len(r.Via) == 0 || !r.Install { @@ -436,10 +495,16 @@ func (t *tun) addRoutes(logErrors bool) error { continue } - err := addRoute(r.Cidr, t.linkAddr) + err := addRoute(r.Cidr, linkAddr) if err != nil { if errors.Is(err, unix.EEXIST) { - t.l.Warn("unable to add unsafe_route, identical route already exists", "route", r.Cidr) + // Every route we added already exists on a reload, only warn about someone else's + msgs, err := systemRoutes() + if err != nil { + t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err) + } else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index { + t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr) + } } else { retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { @@ -450,6 +515,8 @@ func (t *tun) addRoutes(logErrors bool) error { } } else { t.l.Info("Added route", "route", r) + // A cidr listed twice would otherwise find our own route missing from an older fetch + systemRoutes = sync.OnceValues(fetchRoutes) } } @@ -457,7 +524,8 @@ func (t *tun) addRoutes(logErrors bool) error { } func (t *tun) removeRoutes(routes []Route) error { - if t.hostOwned { + linkAddr := t.linkAddr.Load() + if t.hostOwned || linkAddr == nil { return nil } @@ -466,7 +534,7 @@ func (t *tun) removeRoutes(routes []Route) error { continue } - err := delRoute(r.Cidr, t.linkAddr) + err := delRoute(r.Cidr, linkAddr) if err != nil { t.l.Error("Failed to remove route", "error", err, "route", r) } else { diff --git a/overlay/tun_darwin_route_test.go b/overlay/tun_darwin_route_test.go new file mode 100644 index 00000000..9df527bd --- /dev/null +++ b/overlay/tun_darwin_route_test.go @@ -0,0 +1,75 @@ +//go:build darwin && !ios && !e2e_testing + +package overlay + +import ( + "net" + "net/netip" + "os/exec" + "strings" + "testing" + + "github.com/slackhq/nebula/routing" + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + netroute "golang.org/x/net/route" + "golang.org/x/sys/unix" +) + +// Every mac routes loopback through lo0 +func TestRouteInterfaceIn(t *testing.T) { + msgs, err := fetchRoutes() + require.NoError(t, err) + routeInterface := func(s string) int { return routeInterfaceIn(msgs, netip.MustParsePrefix(s)) } + + lo0, err := net.InterfaceByName("lo0") + require.NoError(t, err) + + assert.Equal(t, lo0.Index, routeInterface("127.0.0.0/8")) + assert.Equal(t, lo0.Index, routeInterface("127.0.0.1/32"), "a host route has no netmask") + assert.Equal(t, lo0.Index, routeInterface("::1/128")) + assert.Equal(t, 0, routeInterface("127.0.0.0/16"), "only an exact prefix counts") + + // Most macs also list the default route scoped to their other interfaces + out, err := exec.Command("route", "-n", "get", "default").Output() + require.NoError(t, err) + var name string + for _, line := range strings.Split(string(out), "\n") { + if v, ok := strings.CutPrefix(strings.TrimSpace(line), "interface: "); ok { + name = v + } + } + if name == "" { + t.Skip("no default route") + } + def, err := net.InterfaceByName(name) + require.NoError(t, err) + assert.Equal(t, def.Index, routeInterface("0.0.0.0/0")) +} + +func TestRouteInterfaceIn_SkipsScopedCopies(t *testing.T) { + route := func(index, flags int) netroute.Message { + addrs := make([]netroute.Addr, unix.RTAX_MAX) + addrs[unix.RTAX_DST] = &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}} + addrs[unix.RTAX_NETMASK] = &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}} + return &netroute.RouteMessage{Index: index, Flags: flags, Addrs: addrs} + } + p := netip.MustParsePrefix("192.168.1.0/24") + assert.Equal(t, 7, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE), route(7, unix.RTF_UP)}, p)) + assert.Equal(t, 0, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE)}, p)) +} + +// A reload can happen before Activate +func TestTunRoutesBeforeActivate(t *testing.T) { + tn := &tun{l: test.NewLogger()} + routes := []Route{{ + Cidr: netip.MustParsePrefix("10.253.1.0/24"), + Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}, + Install: true, + }} + tn.Routes.Store(&routes) + + assert.NoError(t, tn.addRoutes(true)) + assert.NoError(t, tn.removeRoutes(routes)) +} From 42280f04cedaf1d5192a91b57c2ec9c595f5e90c Mon Sep 17 00:00:00 2001 From: John Maguire Date: Tue, 13 Oct 2026 13:50:37 -0400 Subject: [PATCH 24/31] darwin: Assign host address to the utun and route vpn explicitly (#1931) Tailscale with an exit node treats every interface address prefix as an attached LAN and either claims it or excludes it, and configd then rewrites the matching route away from nebula's utun. Interfaces holding a single address are skipped, so assign /32 and /128 and add the network route ourselves for both families. --- overlay/tun_darwin.go | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/overlay/tun_darwin.go b/overlay/tun_darwin.go index a0834590..3a7b85cd 100644 --- a/overlay/tun_darwin.go +++ b/overlay/tun_darwin.go @@ -271,6 +271,7 @@ func (t *tun) Activate() error { return t.addRoutes(false) } +// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly. func (t *tun) activate4(network netip.Prefix) error { s, err := unix.Socket( unix.AF_INET, @@ -297,22 +298,32 @@ func (t *tun) activate4(network netip.Prefix) error { MaskAddr: unix.RawSockaddrInet4{ Len: unix.SizeofSockaddrInet4, Family: unix.AF_INET, - Addr: prefixToMask(network).As4(), + Addr: [4]byte{255, 255, 255, 255}, //always make the address a /32 }, } + // On Darwin, VPNs don't edit the routing table directly. + // A network extension hands its routes to the system, and configd installs them. + // configd replaces any route already holding the same prefix, including ours. + // As an example, Tailscale with an exit node treats any interface prefix wider than one address as a LAN. + // With LAN access off, it routes each LAN into Tailscale's utun. + // With LAN access on, it excludes each LAN from its tunnel. + // configd routes an excluded prefix like its own routes would, ignoring ours, so it goes out the default route. + // Either way our route is replaced, and traffic for the vpn network no longer reaches nebula. + // A single address isn't a LAN to Tailscale, so our route is left alone. + // We route the vpn network to the tun explicitly instead. + if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil { return fmt.Errorf("failed to set tun v4 address: %s", err) } - err = addRoute(network, t.linkAddr.Load()) - if err != nil { - return err + if err := addRoute(network, t.linkAddr.Load()); err != nil { + return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err) } - return nil } +// activate6 is activate4 for IPv6 func (t *tun) activate6(network netip.Prefix) error { s, err := unix.Socket( unix.AF_INET6, @@ -334,20 +345,24 @@ func (t *tun) activate6(network netip.Prefix) error { PrefixMask: unix.RawSockaddrInet6{ Len: unix.SizeofSockaddrInet6, Family: unix.AF_INET6, - Addr: prefixToMask(network).As16(), + // Always set the netmask to be a /128. See activate4 for why we do this. + Addr: [16]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, }, Lifetime: addrLifetime{ // never expires Vltime: 0xffffffff, Pltime: 0xffffffff, }, - Flags: _IN6_IFF_NODAD, + Flags: _IN6_IFF_NODAD, // No DAD, a utun has no neighbors. } if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil { return fmt.Errorf("failed to set tun address: %s", err) } + if err := addRoute(network, t.linkAddr.Load()); err != nil { + return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err) + } return nil } From 94372f4f5159134c5e09593c55221a8ca8f76b6e Mon Sep 17 00:00:00 2001 From: Nate Brown Date: Tue, 13 Oct 2026 17:34:38 -0400 Subject: [PATCH 25/31] Don't delete routes we never installed on darwin and the BSDs (#1933) --- overlay/tun_darwin.go | 3 ++- overlay/tun_darwin_route_test.go | 28 ++++++++++++++++++++++++++++ overlay/tun_freebsd.go | 3 ++- overlay/tun_netbsd.go | 3 ++- overlay/tun_openbsd.go | 3 ++- 5 files changed, 36 insertions(+), 4 deletions(-) diff --git a/overlay/tun_darwin.go b/overlay/tun_darwin.go index 3a7b85cd..0824d8bf 100644 --- a/overlay/tun_darwin.go +++ b/overlay/tun_darwin.go @@ -545,7 +545,8 @@ func (t *tun) removeRoutes(routes []Route) error { } for _, r := range routes { - if !r.Install { + if len(r.Via) == 0 || !r.Install { + // addRoutes never installs these continue } diff --git a/overlay/tun_darwin_route_test.go b/overlay/tun_darwin_route_test.go index 9df527bd..72fe008b 100644 --- a/overlay/tun_darwin_route_test.go +++ b/overlay/tun_darwin_route_test.go @@ -5,10 +5,12 @@ package overlay import ( "net" "net/netip" + "os" "os/exec" "strings" "testing" + "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/test" "github.com/stretchr/testify/assert" @@ -73,3 +75,29 @@ func TestTunRoutesBeforeActivate(t *testing.T) { assert.NoError(t, tn.addRoutes(true)) assert.NoError(t, tn.removeRoutes(routes)) } + +// A tun.routes entry with no via is never installed, so removing it must not delete a route for the same cidr that +// belongs to someone else +func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("creating a utun and adding routes needs root") + } + lo0, err := net.InterfaceByName("lo0") + require.NoError(t, err) + cidr := "10.250.0.128/25" + require.NoError(t, exec.Command("route", "-q", "-n", "add", "-net", cidr, "-interface", "lo0").Run()) + t.Cleanup(func() { _ = exec.Command("route", "-q", "-n", "delete", "-net", cidr).Run() }) + + l := test.NewLogger() + c := config.NewC(l) + require.NoError(t, c.LoadString("tun:\n routes:\n - route: "+cidr+"\n mtu: 1300\n")) + tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false) + require.NoError(t, err) + t.Cleanup(func() { _ = tn.Close() }) + require.NoError(t, tn.Activate()) + + require.NoError(t, c.ReloadConfigString("tun: {}\n")) + msgs, err := fetchRoutes() + require.NoError(t, err) + assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))) +} diff --git a/overlay/tun_freebsd.go b/overlay/tun_freebsd.go index cebf959d..8ed90daf 100644 --- a/overlay/tun_freebsd.go +++ b/overlay/tun_freebsd.go @@ -589,7 +589,8 @@ func (t *tun) addRoutes(logErrors bool) error { func (t *tun) removeRoutes(routes []Route) error { for _, r := range routes { - if !r.Install { + if len(r.Via) == 0 || !r.Install { + // addRoutes never installs these continue } diff --git a/overlay/tun_netbsd.go b/overlay/tun_netbsd.go index c6f17848..d9852ba5 100644 --- a/overlay/tun_netbsd.go +++ b/overlay/tun_netbsd.go @@ -420,7 +420,8 @@ func (t *tun) addRoutes(logErrors bool) error { func (t *tun) removeRoutes(routes []Route) error { for _, r := range routes { - if !r.Install { + if len(r.Via) == 0 || !r.Install { + // addRoutes never installs these continue } diff --git a/overlay/tun_openbsd.go b/overlay/tun_openbsd.go index aa90b5fe..b310a0d1 100644 --- a/overlay/tun_openbsd.go +++ b/overlay/tun_openbsd.go @@ -399,7 +399,8 @@ func (t *tun) addRoutes(logErrors bool) error { func (t *tun) removeRoutes(routes []Route) error { for _, r := range routes { - if !r.Install { + if len(r.Via) == 0 || !r.Install { + // addRoutes never installs these continue } From afbdc87ed93213d2baa2a9d85b5c06bb57c05806 Mon Sep 17 00:00:00 2001 From: John Maguire Date: Wed, 14 Oct 2026 21:30:59 -0400 Subject: [PATCH 26/31] Update gvisor and bump to go1.27 (#1940) gvisor's arm64 race-build compare-and-swap (pkg/sync/race_arm64.s) loaded the new value into the register meant for the old one (R1 instead of R2), so it never succeeded. Sleepers could not commit to parking and spun: under -race on arm64 every idle service netstack busied about 14 cores, and TestService's static host lookup of localhost missed its 250ms timeout and stalled for the 30s static_map cadence. Fixed in gvisor 1b2946c4fa. Go 1.27.2 is now the minimum because gvisor's go branch requires it. golangci-lint is bumped to v2.14, the first release that can read go1.27.2 export data. --- .github/workflows/release.yml | 6 +++--- .github/workflows/smoke-extra.yml | 6 +++--- .github/workflows/smoke.yml | 4 ++-- .github/workflows/test.yml | 8 ++++---- go.mod | 7 ++++--- go.sum | 10 ++++++---- 6 files changed, 22 insertions(+), 19 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e9d363c4..0ac04b26 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,7 @@ jobs: - uses: actions/setup-go@v7 with: - go-version: '1.26' + go-version: '1.27' check-latest: true - name: Build @@ -40,7 +40,7 @@ jobs: - uses: actions/setup-go@v7 with: - go-version: '1.26' + go-version: '1.27' check-latest: true - name: Build @@ -83,7 +83,7 @@ jobs: - uses: actions/setup-go@v7 with: - go-version: '1.26' + go-version: '1.27' check-latest: true # GitHub holds ARNs, not credentials, and ARNs outlive a rotation diff --git a/.github/workflows/smoke-extra.yml b/.github/workflows/smoke-extra.yml index 8a388f9f..de59a74f 100644 --- a/.github/workflows/smoke-extra.yml +++ b/.github/workflows/smoke-extra.yml @@ -34,7 +34,7 @@ jobs: - uses: actions/setup-go@v7 with: - go-version: '1.26' + go-version: '1.27' check-latest: true - name: add hashicorp source @@ -66,7 +66,7 @@ jobs: - uses: actions/setup-go@v7 with: - go-version: '1.26' + go-version: '1.27' check-latest: true - name: add hashicorp source @@ -92,7 +92,7 @@ jobs: - uses: actions/setup-go@v7 with: - go-version: '1.26' + go-version: '1.27' check-latest: true # WSL2 + Ubuntu so the smoke can run a real linux peer with its own diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 4937d53c..68f69b87 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-go@v7 with: - go-version: '1.26' + go-version: '1.27' check-latest: true - name: Smoke Docker @@ -51,7 +51,7 @@ jobs: - uses: actions/setup-go@v7 with: - go-version: '1.26' + go-version: '1.27' check-latest: true - name: build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6bde71e8..62566fa4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -22,7 +22,7 @@ jobs: - uses: actions/setup-go@v7 with: - go-version: '1.26' + go-version: '1.27' check-latest: true - name: Install goimports @@ -42,7 +42,7 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v9 with: - version: v2.12 + version: v2.14 test: name: Test ${{ matrix.name }} @@ -87,7 +87,7 @@ jobs: - uses: actions/setup-go@v7 with: - go-version: '1.26' + go-version: '1.27' check-latest: true - name: Build @@ -132,7 +132,7 @@ jobs: - uses: actions/setup-go@v7 with: - go-version: '1.26' + go-version: '1.27' check-latest: true - name: Build ${{ matrix.name }} diff --git a/go.mod b/go.mod index a110b8a2..caa3b502 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/slackhq/nebula -go 1.26.0 +go 1.27.2 require ( dario.cat/mergo v1.0.2 @@ -33,7 +33,7 @@ require ( golang.zx2c4.com/wireguard/windows v1.0.1 google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 - gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe + gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980 ) require ( @@ -45,5 +45,6 @@ require ( github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect github.com/vishvananda/netns v0.0.5 // indirect - golang.org/x/time v0.5.0 // indirect + golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc // indirect + golang.org/x/time v0.15.0 // indirect ) diff --git a/go.sum b/go.sum index 78189af1..a1a0533f 100644 --- a/go.sum +++ b/go.sum @@ -157,6 +157,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= +golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc h1:TS73t7x3KarrNd5qAipmspBDS1rkMcgVG/fS1aRb4Rc= +golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= @@ -203,8 +205,8 @@ golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= -golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -243,5 +245,5 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe h1:fre4i6mv4iBuz5lCMOzHD1rH1ljqHWSICFmZRbbgp3g= -gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU= +gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980 h1:ribyrTYQ/QYeUIs8x8bVLYwGyepa7RnHLqJzegT5YYk= +gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980/go.mod h1:pK/dapF5Bf3KGW7/kd5CI5IC4lYh9FMFY1qETAIGLXk= From 42597dda7da57f3e4b191a3ee62077abdfe5cdfb Mon Sep 17 00:00:00 2001 From: John Maguire Date: Thu, 15 Oct 2026 15:14:39 -0400 Subject: [PATCH 27/31] service: Don't hold listener lock while handing off a connection (#1939) --- service/listener.go | 35 +++++++--- service/service.go | 12 +++- service/service_test.go | 139 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 173 insertions(+), 13 deletions(-) diff --git a/service/listener.go b/service/listener.go index 6d5c8a43..c092c245 100644 --- a/service/listener.go +++ b/service/listener.go @@ -1,8 +1,8 @@ package service import ( - "io" "net" + "sync" ) type tcpListener struct { @@ -10,23 +10,38 @@ type tcpListener struct { s *Service addr *net.TCPAddr accept chan net.Conn + + // closed is closed instead of accept so a handler blocked handing off a + // connection can't send on a closed channel. + closed chan struct{} + closeOnce sync.Once } func (l *tcpListener) Accept() (net.Conn, error) { - conn, ok := <-l.accept - if !ok { - return nil, io.EOF + // select picks randomly among ready cases, so without this a handler still + // waiting to hand off could win over closed after Close has returned. + select { + case <-l.closed: + return nil, net.ErrClosed + default: + } + + select { + case conn := <-l.accept: + return conn, nil + case <-l.closed: + return nil, net.ErrClosed } - return conn, nil } func (l *tcpListener) Close() error { - l.s.mu.Lock() - defer l.s.mu.Unlock() - delete(l.s.mu.listeners, uint16(l.addr.Port)) - - close(l.accept) + l.closeOnce.Do(func() { + l.s.mu.Lock() + delete(l.s.mu.listeners, l.port) + l.s.mu.Unlock() + close(l.closed) + }) return nil } diff --git a/service/service.go b/service/service.go index 6610800d..a51dc461 100644 --- a/service/service.go +++ b/service/service.go @@ -231,6 +231,7 @@ func (s *Service) Listen(network, address string) (net.Listener, error) { s: s, addr: addr, accept: make(chan net.Conn), + closed: make(chan struct{}), } s.mu.Lock() @@ -257,9 +258,8 @@ func (s *Service) tcpHandler(r *tcp.ForwarderRequest) { endpointID := r.ID() s.mu.Lock() - defer s.mu.Unlock() - l, ok := s.mu.listeners[endpointID.LocalPort] + s.mu.Unlock() if !ok { r.Complete(true) return @@ -275,6 +275,12 @@ func (s *Service) tcpHandler(r *tcp.ForwarderRequest) { r.Complete(false) ep.SocketOptions().SetKeepAlive(true) + // The handoff blocks until Accept is called, so it must not hold s.mu or a + // listener nobody accepts from stalls every other port. conn := gonet.NewTCPConn(&wq, ep) - l.accept <- conn + select { + case l.accept <- conn: + case <-l.closed: + _ = conn.Close() + } } diff --git a/service/service_test.go b/service/service_test.go index add09057..e0ac695c 100644 --- a/service/service_test.go +++ b/service/service_test.go @@ -188,3 +188,142 @@ func TestService(t *testing.T) { t.Fatal(err) } } + +// A listener that is never accepted from must not stall connections to other +// ports, and closing it must not block. +func TestServiceUnacceptedListener(t *testing.T) { + lighthousePort := ephemeralUDPPort(t) + + ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{}) + a := newSimpleService(ca, caKey, "a", netip.MustParseAddr("10.0.0.1"), m{ + "static_host_map": m{}, + "lighthouse": m{ + "am_lighthouse": true, + }, + "listen": m{ + "host": "0.0.0.0", + "port": lighthousePort, + }, + }) + t.Cleanup(func() { _ = a.Close() }) + b := newSimpleService(ca, caKey, "b", netip.MustParseAddr("10.0.0.2"), m{ + "static_host_map": m{ + "10.0.0.1": []string{fmt.Sprintf("127.0.0.1:%d", lighthousePort)}, + }, + "lighthouse": m{ + "hosts": []string{"10.0.0.1"}, + "interval": 1, + }, + }) + t.Cleanup(func() { _ = b.Close() }) + + idle, err := a.Listen("tcp", ":1234") + if err != nil { + t.Fatal(err) + } + active, err := a.Listen("tcp", ":1235") + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + idleConn, err := b.DialContext(ctx, "tcp", "10.0.0.1:1234") + if err != nil { + t.Fatal(err) + } + defer idleConn.Close() + + accepted := make(chan error, 1) + go func() { + conn, err := active.Accept() + if err == nil { + err = conn.Close() + } + accepted <- err + }() + + ctx, cancel = context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + activeConn, err := b.DialContext(ctx, "tcp", "10.0.0.1:1235") + if err != nil { + t.Fatalf("dial to accepted port stalled behind idle listener: %v", err) + } + defer activeConn.Close() + + select { + case err := <-accepted: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("accept stalled behind idle listener") + } + + closed := make(chan error, 1) + go func() { closed <- idle.Close() }() + select { + case err := <-closed: + if err != nil { + t.Fatal(err) + } + case <-time.After(5 * time.Second): + t.Fatal("closing idle listener blocked") + } + + if _, err := idle.Accept(); !errors.Is(err, net.ErrClosed) { + t.Fatalf("expected net.ErrClosed from closed listener, got %v", err) + } + if err := idle.Close(); err != nil { + t.Fatalf("second close: %v", err) + } +} + +// Accept must report net.ErrClosed after Close even while a handler is still +// trying to hand off a connection. +func TestListenerAcceptAfterClose(t *testing.T) { + s := &Service{} + s.mu.listeners = map[uint16]*tcpListener{} + l := &tcpListener{ + port: 1234, + s: s, + accept: make(chan net.Conn), + closed: make(chan struct{}), + } + s.mu.listeners[l.port] = l + if err := l.Close(); err != nil { + t.Fatal(err) + } + + // Keep a send pending so every Accept has a connection ready alongside closed. + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + for { + server, client := net.Pipe() + _ = client.Close() + select { + case l.accept <- server: + case <-stop: + _ = server.Close() + return + } + } + }() + defer func() { + close(stop) + <-done + }() + + for range 100 { + time.Sleep(time.Millisecond) + conn, err := l.Accept() + if conn != nil { + _ = conn.Close() + } + if !errors.Is(err, net.ErrClosed) { + t.Fatalf("Accept after Close: got err %v, want net.ErrClosed", err) + } + } +} From 1a0ac25db053e5073970bbf25949ee5f6058fc0c Mon Sep 17 00:00:00 2001 From: Nate Brown Date: Thu, 15 Oct 2026 16:36:39 -0400 Subject: [PATCH 28/31] Report how a config reload went: failures, restarts required, and the keys that need one (#1935) --- cmd/nebula-service/main.go | 3 +- cmd/nebula-service/service.go | 3 +- cmd/nebula/main.go | 3 +- config/config.go | 53 +++++++++------ config/config_test.go | 4 +- config/reload.go | 98 +++++++++++++++++++++++++++ config/reload_test.go | 108 ++++++++++++++++++++++++++++++ connection_manager.go | 3 +- control_lifecycle_test.go | 6 +- dns_server.go | 3 +- e2e/handshakes_test.go | 2 +- e2e/tunnels_test.go | 4 +- examples/config.yml | 26 +++++-- hostmap.go | 2 +- interface.go | 12 ++-- interface_test.go | 30 +++++++-- lighthouse.go | 5 +- lighthouse_test.go | 43 ++++++++---- main.go | 12 +++- main_restart_keys_linux.go | 6 ++ main_restart_keys_other.go | 6 ++ overlay/tun.go | 7 ++ overlay/tun_android.go | 9 +-- overlay/tun_bypass_windows.go | 3 + overlay/tun_bypass_windows_386.go | 3 + overlay/tun_darwin.go | 36 ++++++---- overlay/tun_darwin_route_test.go | 8 ++- overlay/tun_freebsd.go | 29 +++++--- overlay/tun_ios.go | 9 +-- overlay/tun_linux.go | 42 ++++++++---- overlay/tun_netbsd.go | 29 +++++--- overlay/tun_openbsd.go | 29 +++++--- overlay/tun_reload_linux_test.go | 59 ++++++++++++++-- overlay/tun_test.go | 24 +++++++ overlay/tun_windows.go | 30 ++++++--- overlay/tun_windows_route_test.go | 30 +++++++-- pki.go | 39 ++++++++--- pki_test.go | 86 ++++++++++++++++++++++++ punchy.go | 3 +- punchy_test.go | 22 ++++-- relay_manager.go | 3 +- ssh.go | 21 ++++-- ssh_reload_test.go | 83 +++++++++++++++++++++++ sshd_reload_test.go | 12 ++-- stats.go | 3 +- udp/conn.go | 5 +- udp/netchange.go | 6 ++ udp/netchange_darwin.go | 3 + udp/netchange_generic.go | 3 + udp/netchange_test.go | 16 +++++ udp/udp_bypass_windows.go | 6 +- udp/udp_darwin.go | 2 +- udp/udp_generic.go | 2 +- udp/udp_linux.go | 5 +- udp/udp_linux_reload_test.go | 31 +++++++++ udp/udp_rio_windows.go | 2 +- udp/udp_tester.go | 2 +- 57 files changed, 941 insertions(+), 193 deletions(-) create mode 100644 config/reload.go create mode 100644 config/reload_test.go create mode 100644 main_restart_keys_linux.go create mode 100644 main_restart_keys_other.go create mode 100644 overlay/tun_test.go create mode 100644 ssh_reload_test.go create mode 100644 udp/udp_linux_reload_test.go diff --git a/cmd/nebula-service/main.go b/cmd/nebula-service/main.go index e0b335f5..d385bb8c 100644 --- a/cmd/nebula-service/main.go +++ b/cmd/nebula-service/main.go @@ -85,9 +85,10 @@ func main() { fmt.Printf("failed to apply logging config: %s", err) os.Exit(1) } - c.RegisterReloadCallback(func(c *config.C) { + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { if err := logging.ApplyConfig(l, c); err != nil { l.Error("Failed to reconfigure logger on reload", "error", err) + r.Failed() } }) diff --git a/cmd/nebula-service/service.go b/cmd/nebula-service/service.go index abe9abe0..9ef8cd5a 100644 --- a/cmd/nebula-service/service.go +++ b/cmd/nebula-service/service.go @@ -34,9 +34,10 @@ func (p *program) Start(s service.Service) error { if err := logging.ApplyConfig(l, c); err != nil { return fmt.Errorf("failed to apply logging config: %s", err) } - c.RegisterReloadCallback(func(c *config.C) { + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { if err := logging.ApplyConfig(l, c); err != nil { l.Error("Failed to reconfigure logger on reload", "error", err) + r.Failed() } }) diff --git a/cmd/nebula/main.go b/cmd/nebula/main.go index 3c786b84..31da63e4 100644 --- a/cmd/nebula/main.go +++ b/cmd/nebula/main.go @@ -71,9 +71,10 @@ func main() { fmt.Printf("failed to apply logging config: %s", err) os.Exit(1) } - c.RegisterReloadCallback(func(c *config.C) { + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { if err := logging.ApplyConfig(l, c); err != nil { l.Error("Failed to reconfigure logger on reload", "error", err) + r.Failed() } }) diff --git a/config/config.go b/config/config.go index 79cdce0f..ffa039be 100644 --- a/config/config.go +++ b/config/config.go @@ -26,9 +26,12 @@ type C struct { files []string Settings map[string]any oldSettings map[string]any - callbacks []func(*C) + callbacks []func(*C, *Reload) l *slog.Logger reloadLock sync.Mutex + + // Set by the first reload that needs a restart and never cleared. Guarded by reloadLock + needsRestart bool } func NewC(l *slog.Logger) *C { @@ -72,8 +75,9 @@ func (c *C) LoadString(raw string) error { // RegisterReloadCallback stores a function to be called when a config reload is triggered. The functions registered // here should decide if they need to make a change to the current process before making the change. HasChanged can be // used to help decide if a change is necessary. -// These functions should return quickly or spawn their own go routine if they will take a while -func (c *C) RegisterReloadCallback(f func(*C)) { +// These functions should return quickly or spawn their own go routine if they will take a while. +// Use r to report anything that failed or needs a restart to apply. +func (c *C) RegisterReloadCallback(f func(c *C, r *Reload)) { c.callbacks = append(c.callbacks, f) } @@ -144,50 +148,55 @@ func (c *C) CatchHUP(ctx context.Context) { return case <-ch: c.l.Info("Caught HUP, reloading config") - c.ReloadConfig() + c.ReloadConfig().Log(c.l) } } }() } -func (c *C) ReloadConfig() { +// ReloadConfig reloads the config from the path provided to Load and runs every registered callback. A config that +// fails to load is logged here, like any other failure, and the result carries the error for the summary. +func (c *C) ReloadConfig() ReloadResult { c.reloadLock.Lock() defer c.reloadLock.Unlock() c.oldSettings = make(map[string]any) maps.Copy(c.oldSettings, c.Settings) - err := c.Load(c.path) - if err != nil { - c.l.Error("Error occurred while reloading config", - "config_path", c.path, - "error", err, - ) - return + if err := c.Load(c.path); err != nil { + c.l.Error("Error occurred while reloading config", "config_path", c.path, "error", err) + return ReloadResult{Err: err, RestartRequired: c.needsRestart} } - for _, v := range c.callbacks { - v(c) - } + return c.runCallbacks() } -func (c *C) ReloadConfigString(raw string) error { +// ReloadConfigString is ReloadConfig for a config held in memory rather than on disk. +func (c *C) ReloadConfigString(raw string) ReloadResult { c.reloadLock.Lock() defer c.reloadLock.Unlock() c.oldSettings = make(map[string]any) maps.Copy(c.oldSettings, c.Settings) - err := c.LoadString(raw) - if err != nil { - return err + if err := c.LoadString(raw); err != nil { + c.l.Error("Error occurred while reloading config", "error", err) + return ReloadResult{Err: err, RestartRequired: c.needsRestart} } + return c.runCallbacks() +} + +func (c *C) runCallbacks() ReloadResult { + r := &Reload{c: c} for _, v := range c.callbacks { - v(c) + v(c, r) } - return nil + if r.restart { + c.needsRestart = true + } + return ReloadResult{Failed: r.failed, RestartRequired: c.needsRestart, RestartKeys: r.keys} } // GetString will get the string for k or return the default d if not found or invalid @@ -391,7 +400,7 @@ func (c *C) parse() error { var nm map[string]any err = yaml.Unmarshal(b, &nm) if err != nil { - return err + return fmt.Errorf("%s: %w", path, err) } // We need to use WithAppendSlice so that firewall rules in separate diff --git a/config/config_test.go b/config/config_test.go index aba7b2a8..e5d21e5b 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -19,7 +19,7 @@ func TestConfig_Load(t *testing.T) { // invalid yaml c := NewC(l) os.WriteFile(filepath.Join(dir, "01.yaml"), []byte(" invalid yaml"), 0644) - require.EqualError(t, c.Load(dir), "yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `invalid...` into map[string]interface {}") + require.EqualError(t, c.Load(dir), filepath.Join(dir, "01.yaml")+": yaml: unmarshal errors:\n line 1: cannot unmarshal !!str `invalid...` into map[string]interface {}") // simple multi config merge c = NewC(l) @@ -129,7 +129,7 @@ func TestConfig_ReloadConfig(t *testing.T) { os.WriteFile(filepath.Join(dir, "01.yaml"), []byte("outer:\n inner: ho"), 0644) - c.RegisterReloadCallback(func(c *C) { + c.RegisterReloadCallback(func(c *C, _ *Reload) { done <- true }) diff --git a/config/reload.go b/config/reload.go new file mode 100644 index 00000000..f0fbd01e --- /dev/null +++ b/config/reload.go @@ -0,0 +1,98 @@ +package config + +import ( + "context" + "fmt" + "log/slog" + "slices" +) + +// ReloadResult describes how a reload went. Problems are logged where they happen, Log writes a one line summary. +type ReloadResult struct { + // Err is set when the new config could not be read or parsed, no callbacks ran + Err error + // Failed is true when a callback could not apply part of the new config + Failed bool + // RestartRequired is true once any reload has needed a restart, and stays true until the process restarts + RestartRequired bool + // RestartKeys are the keys this reload changed that only apply on a restart + RestartKeys []string +} + +// Summary returns a one line summary of the reload and the level to log it at. +func (r ReloadResult) Summary() (slog.Level, string) { + switch { + case r.Err != nil && r.RestartRequired: + return slog.LevelError, "Config reload failed, the new config did not load and a restart is still required" + case r.Err != nil: + return slog.LevelError, "Config reload failed, the new config did not load" + case r.Failed && r.RestartRequired: + return slog.LevelError, "Config reload completed with errors and requires a restart to fully apply, review the logs" + case r.Failed: + return slog.LevelError, "Config reload completed with errors, review the logs" + case r.RestartRequired: + return slog.LevelWarn, "Config reload was completed but requires a restart to fully apply" + default: + return slog.LevelInfo, "Config reload complete" + } +} + +// String is the summary with Err and RestartKeys included, for output other than the log. +func (r ReloadResult) String() string { + _, msg := r.Summary() + if r.Err != nil { + msg = fmt.Sprintf("%s: %v", msg, r.Err) + } + if len(r.RestartKeys) > 0 { + msg = fmt.Sprintf("%s, keys %v", msg, r.RestartKeys) + } + return msg +} + +// Log writes Summary to l. +func (r ReloadResult) Log(l *slog.Logger) { + level, msg := r.Summary() + if len(r.RestartKeys) > 0 { + l.Log(context.Background(), level, msg, "keys", r.RestartKeys) + return + } + l.Log(context.Background(), level, msg) +} + +// Reload is passed to each reload callback to report how applying the new config went. +type Reload struct { + c *C + failed bool + restart bool + // Changed keys that only apply on a restart, from every callback + keys []string +} + +// Failed marks the reload as failed, the caller logs why. Safe to call on a nil Reload, which is what the initial load +// passes. +func (r *Reload) Failed() { + if r != nil { + r.failed = true + } +} + +// RestartRequired marks the reload as needing a restart, the caller logs why. Safe to call on a nil Reload. +func (r *Reload) RestartRequired() { + if r != nil { + r.restart = true + } +} + +// RestartRequiredIfChanged marks the reload as needing a restart if any of keys changed, and adds them to RestartKeys. +// Use it for keys that are only read at startup. Safe to call on a nil Reload. +func (r *Reload) RestartRequiredIfChanged(keys ...string) { + if r == nil { + return + } + for _, k := range keys { + if r.c.HasChanged(k) && !slices.Contains(r.keys, k) { + r.keys = append(r.keys, k) + r.restart = true + } + } +} diff --git a/config/reload_test.go b/config/reload_test.go new file mode 100644 index 00000000..86d16025 --- /dev/null +++ b/config/reload_test.go @@ -0,0 +1,108 @@ +package config + +import ( + "errors" + "log/slog" + "testing" + + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestReloadConfigString(t *testing.T) { + c := NewC(test.NewLogger()) + require.NoError(t, c.LoadString("a: 1")) + fail := false + ran := 0 + c.RegisterReloadCallback(func(c *C, r *Reload) { + ran++ + if fail { + r.Failed() + } + }) + + assert.Equal(t, ReloadResult{}, c.ReloadConfigString("a: 2")) + fail = true + assert.Equal(t, ReloadResult{Failed: true}, c.ReloadConfigString("a: 3")) + assert.Equal(t, 2, ran) + + // A config that fails to load runs no callbacks and returns the error + res := c.ReloadConfigString("a: [") + require.Error(t, res.Err) + assert.False(t, res.Failed) + assert.Equal(t, 2, ran) + assert.Equal(t, 3, c.GetInt("a", 0)) +} + +func TestReloadRestartRequiredLatches(t *testing.T) { + c := NewC(test.NewLogger()) + require.NoError(t, c.LoadString("a: 1\nb: 1")) + c.RegisterReloadCallback(func(c *C, r *Reload) { r.RestartRequiredIfChanged("a", "b") }) + + // Nothing that needs a restart changed + assert.Equal(t, ReloadResult{}, c.ReloadConfigString("a: 1\nb: 1\nc: 1")) + assert.Equal(t, ReloadResult{RestartRequired: true, RestartKeys: []string{"a"}}, c.ReloadConfigString("a: 2\nb: 1\nc: 1")) + + // The restart sticks through a reload with no changes, reverting the key and a config that fails to load. Only the + // reload that changed the key lists it + assert.Equal(t, ReloadResult{RestartRequired: true}, c.ReloadConfigString("a: 2\nb: 1\nc: 1")) + assert.Equal(t, ReloadResult{RestartRequired: true, RestartKeys: []string{"a"}}, c.ReloadConfigString("a: 1\nb: 1\nc: 1")) + + res := c.ReloadConfigString("a: [") + require.Error(t, res.Err) + assert.True(t, res.RestartRequired) +} + +// A key reported by more than one callback is listed once +func TestReloadRestartKeysDeduped(t *testing.T) { + c := NewC(test.NewLogger()) + require.NoError(t, c.LoadString("a: 1\nb: 1")) + c.RegisterReloadCallback(func(c *C, r *Reload) { r.RestartRequiredIfChanged("a") }) + c.RegisterReloadCallback(func(c *C, r *Reload) { r.RestartRequiredIfChanged("a", "b") }) + + assert.Equal(t, []string{"a", "b"}, c.ReloadConfigString("a: 2\nb: 2").RestartKeys) +} + +// A callback can require a restart without a key changing, a file the config points at may have +func TestReloadRestartRequiredWithoutAChange(t *testing.T) { + c := NewC(test.NewLogger()) + require.NoError(t, c.LoadString("a: 1")) + c.RegisterReloadCallback(func(c *C, r *Reload) { r.RestartRequired() }) + + assert.Equal(t, ReloadResult{RestartRequired: true}, c.ReloadConfigString("a: 1")) +} + +func TestReloadResultSummary(t *testing.T) { + cases := []struct { + res ReloadResult + level slog.Level + msg string + }{ + {ReloadResult{}, slog.LevelInfo, "Config reload complete"}, + {ReloadResult{RestartRequired: true}, slog.LevelWarn, "Config reload was completed but requires a restart to fully apply"}, + {ReloadResult{Failed: true}, slog.LevelError, "Config reload completed with errors, review the logs"}, + {ReloadResult{Failed: true, RestartRequired: true}, slog.LevelError, "Config reload completed with errors and requires a restart to fully apply, review the logs"}, + {ReloadResult{Err: errors.New("bad yaml")}, slog.LevelError, "Config reload failed, the new config did not load"}, + {ReloadResult{Err: errors.New("bad yaml"), RestartRequired: true}, slog.LevelError, "Config reload failed, the new config did not load and a restart is still required"}, + } + for _, tc := range cases { + level, msg := tc.res.Summary() + assert.Equal(t, tc.level, level, tc.msg) + assert.Equal(t, tc.msg, msg) + } + + res := ReloadResult{RestartRequired: true, RestartKeys: []string{"a", "b"}} + assert.Equal(t, "Config reload was completed but requires a restart to fully apply, keys [a b]", res.String()) + + // String includes the error + assert.Equal(t, "Config reload failed, the new config did not load: bad yaml", ReloadResult{Err: errors.New("bad yaml")}.String()) +} + +// The initial load passes a nil Reload +func TestReloadNilIsSafe(t *testing.T) { + var r *Reload + r.Failed() + r.RestartRequired() + r.RestartRequiredIfChanged("a") +} diff --git a/connection_manager.go b/connection_manager.go index 4ec9f614..af6d7d06 100644 --- a/connection_manager.go +++ b/connection_manager.go @@ -57,7 +57,8 @@ func newConnectionManagerFromConfig(l *slog.Logger, c *config.C, hm *HostMap, p } cm.reload(c, true) - c.RegisterReloadCallback(func(c *config.C) { + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + r.RestartRequiredIfChanged("timers.connection_alive_interval", "timers.pending_deletion_interval") cm.reload(c, false) }) diff --git a/control_lifecycle_test.go b/control_lifecycle_test.go index 4bc75273..6c0387b6 100644 --- a/control_lifecycle_test.go +++ b/control_lifecycle_test.go @@ -151,9 +151,9 @@ func (c *fakeConn) WriteTo(_ []byte, _ netip.AddrPort) error { return nil } func (c *fakeConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) { return len(bufs), nil } -func (c *fakeConn) ReloadConfig(_ *config.C) {} -func (c *fakeConn) SupportsMultipleReaders() bool { return true } -func (c *fakeConn) Close() error { c.closed = true; return nil } +func (c *fakeConn) ReloadConfig(_ *config.C, _ *config.Reload) {} +func (c *fakeConn) SupportsMultipleReaders() bool { return true } +func (c *fakeConn) Close() error { c.closed = true; return nil } type multiqueueDevice struct { *fakeDevice diff --git a/dns_server.go b/dns_server.go index 9339b068..6a3b9e4d 100644 --- a/dns_server.go +++ b/dns_server.go @@ -69,9 +69,10 @@ func newDnsServerFromConfig(ctx context.Context, l *slog.Logger, pki *PKI, hostM ds.mux = dns.NewServeMux() ds.mux.HandleFunc(".", ds.handleDnsRequest) - c.RegisterReloadCallback(func(c *config.C) { + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { if err := ds.reload(c, false); err != nil { ds.l.Error("Failed to reload DNS responder from config", "error", err) + r.Failed() } }) diff --git a/e2e/handshakes_test.go b/e2e/handshakes_test.go index aef2e8d1..770ab0f4 100644 --- a/e2e/handshakes_test.go +++ b/e2e/handshakes_test.go @@ -1513,7 +1513,7 @@ func TestLighthouseUpdateOnReload(t *testing.T) { // Reload the config. The lighthouse.hosts change triggers TriggerUpdate, // which wakes the update worker. It calls SendUpdate, initiating a // handshake to the new lighthouse and caching the HostUpdateNotification. - require.NoError(t, myConfig.ReloadConfigString(string(newCfg))) + require.NoError(t, myConfig.ReloadConfigString(string(newCfg)).Err) // Route until the lighthouse receives the HostUpdateNotification. // This covers: handshake stage 1, stage 2, then the cached update. diff --git a/e2e/tunnels_test.go b/e2e/tunnels_test.go index 7831b7dd..e2272ef8 100644 --- a/e2e/tunnels_test.go +++ b/e2e/tunnels_test.go @@ -129,7 +129,7 @@ func TestCertUpgrade(t *testing.T) { } r.Logf("reload new v2-only config") - err = myC.ReloadConfigString(string(cb)) + err = myC.ReloadConfigString(string(cb)).Err assert.NoError(t, err) r.Log("yay, spin until their sees it") waitStart := time.Now() @@ -222,7 +222,7 @@ func TestCertDowngrade(t *testing.T) { } r.Logf("reload new v1-only config") - err = myC.ReloadConfigString(string(cb)) + err = myC.ReloadConfigString(string(cb)).Err assert.NoError(t, err) r.Log("yay, spin until their sees it") waitStart := time.Now() diff --git a/examples/config.yml b/examples/config.yml index 0d1a9180..ef29812e 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -43,7 +43,7 @@ static_host_map: lighthouse: # am_lighthouse is used to enable lighthouse functionality for a node. This should ONLY be true on nodes - # you have configured to be lighthouses in your network + # you have configured to be lighthouses in your network. Not reloadable. am_lighthouse: false # serve_dns optionally starts a dns listener that responds to various queries and can even be # delegated to for resolution @@ -126,8 +126,9 @@ lighthouse: # Port Nebula will be listening on. The default here is 4242. For a lighthouse node, the port should be defined, # however using port 0 will dynamically assign a port and is recommended for roaming nodes. listen: - # To listen on only ipv4, use "0.0.0.0" + # To listen on only ipv4, use "0.0.0.0". Not reloadable. host: "::" + # Not reloadable. port: 4242 # Sets the max number of packets to pull from the kernel for each syscall (under systems that support recvmmsg) # default is 64, does not support reload @@ -135,7 +136,8 @@ listen: # superpacket, so the receive scratch is batch * 64KiB per listening socket (~4MiB per routine at the # default of 64). Lower this to trade peak per-syscall throughput for memory on constrained hosts. #batch: 64 - # Linux only. Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel + # Linux only. Reloadable, but removing a value leaves the socket at the old size until Nebula restarts. + # Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel # Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default) # Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide # max, net.core.rmem_max and net.core.wmem_max @@ -179,7 +181,7 @@ listen: # Currently, this defaults to 1 which means we have 1 tun queue reader and 1 # UDP queue reader. Setting this above one will set IFF_MULTI_QUEUE on the tun # device and SO_REUSEPORT on the UDP socket to allow multiple queues. -# This option is only supported on Linux. +# This option is only supported on Linux. Not reloadable. #routines: 1 punchy: @@ -203,6 +205,7 @@ punchy: # Cipher allows you to choose between the available ciphers for your network. Options are chachapoly or aes # IMPORTANT: this value must be identical on ALL NODES/LIGHTHOUSES. We do not/will not support use of different ciphers simultaneously! +# Not reloadable. #cipher: aes # Preferred ranges is used to define a hint about the local network ranges, which speeds up discovering the fastest @@ -234,6 +237,7 @@ punchy: # and absolute paths outside of it will be rejected. Default is $TMP/nebula-debug. # The directory is NOT automatically created. # Overriding this to "" is the same as "/" and will allow overwriting any path on the host. + # Not reloadable. #sandbox_dir: /var/tmp/nebula-debug # EXPERIMENTAL: relay support for networks that can't establish direct connections. @@ -253,19 +257,25 @@ relay: # Configure the private interface. Note: addr is baked into the nebula certificate tun: # When tun is disabled, a lighthouse can be started without a local tun interface (and therefore without root) + # Not reloadable. disabled: false # Name of the device. If not set, a default will be chosen by the OS. # For macOS: if set, must be in the form `utun[0-9]+`. # For NetBSD: Required to be set, must be in the form `tun[0-9]+` + # Not reloadable. dev: nebula1 # Toggles forwarding of local broadcast packets, the address of which depends on the ip/mask encoded in pki.cert + # Not reloadable. drop_local_broadcast: false - # Toggles forwarding of multicast packets + # Toggles forwarding of multicast packets. Not reloadable. drop_multicast: false # Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 5000 + # Not reloadable. tx_queue: 5000 # Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic # Anything above 8937, here or on a route below, is capped to 8937 with a warning + # Reloadable on Linux only. Where the tun is handed to nebula by a host app (iOS, Android, macOS network extensions) + # the host app owns the MTU. mtu: 1300 # the use_offloads setting controls if Nebula will attempt to enable GSO and GRO for the tun device. Linux only, not reloadable. @@ -358,7 +368,7 @@ tun: #use_system_route_table: false # Buffer size for reading routes updates. 0 means default system buffer size. (/proc/sys/net/core/rmem_default). # If using massive routes updates, for example BGP, you may need to increase this value to avoid packet loss. - # SO_RCVBUFFORCE is used to avoid having to raise the system wide max + # SO_RCVBUFFORCE is used to avoid having to raise the system wide max. Not reloadable. #use_system_route_table_buffer_size: 0 # Configure logging level @@ -408,14 +418,16 @@ logging: # Handshakes are sent to all known addresses at each interval with a linear backoff, # Wait try_interval after the 1st attempt, 2 * try_interval after the 2nd, etc, until the handshake is older than timeout # A 100ms interval with the default 10 retries will give a handshake 5.5 seconds to resolve before timing out + # Not reloadable. #try_interval: 100ms #retries: 20 # query_buffer is the size of the buffer channel for querying lighthouses, drops are recorded in the lighthouse.query.dropped metric. + # Not reloadable. #query_buffer: 1024 # trigger_buffer is the size of the buffer channel for quickly sending handshakes - # after receiving the response for lighthouse queries + # after receiving the response for lighthouse queries. Not reloadable. #trigger_buffer: 64 # Tunnel manager settings diff --git a/hostmap.go b/hostmap.go index 71814a2f..a3eb0c29 100644 --- a/hostmap.go +++ b/hostmap.go @@ -317,7 +317,7 @@ func NewHostMapFromConfig(l *slog.Logger, c *config.C) *HostMap { hm := newHostMap(l) hm.reload(c, true) - c.RegisterReloadCallback(func(c *config.C) { + c.RegisterReloadCallback(func(c *config.C, _ *config.Reload) { hm.reload(c, false) }) diff --git a/interface.go b/interface.go index 29051be4..96d44446 100644 --- a/interface.go +++ b/interface.go @@ -509,10 +509,11 @@ func (f *Interface) flushSendBatch(sb *batch.SendBatch, q int) { func (f *Interface) RegisterConfigChangeCallbacks(c *config.C) { c.RegisterReloadCallback(f.reloadFirewall) - c.RegisterReloadCallback(f.reloadSendRecvError) - c.RegisterReloadCallback(f.reloadAcceptRecvError) - c.RegisterReloadCallback(f.reloadDisconnectInvalid) - c.RegisterReloadCallback(f.reloadMisc) + for _, reload := range []func(*config.C){ + f.reloadSendRecvError, f.reloadAcceptRecvError, f.reloadDisconnectInvalid, f.reloadMisc, + } { + c.RegisterReloadCallback(func(c *config.C, _ *config.Reload) { reload(c) }) + } for _, udpConn := range f.writers { c.RegisterReloadCallback(udpConn.ReloadConfig) @@ -529,7 +530,7 @@ func (f *Interface) reloadDisconnectInvalid(c *config.C) { } } -func (f *Interface) reloadFirewall(c *config.C) { +func (f *Interface) reloadFirewall(c *config.C, r *config.Reload) { cs := f.pki.getCertState() curCert := cs.getCertificate(cert.Version2) if curCert == nil { @@ -548,6 +549,7 @@ func (f *Interface) reloadFirewall(c *config.C) { fw, err := NewFirewallFromConfig(f.l, cs, c) if err != nil { f.l.Error("Error while creating firewall during reload", "error", err) + r.Failed() return } diff --git a/interface_test.go b/interface_test.go index 1b912bbb..b66f1e81 100644 --- a/interface_test.go +++ b/interface_test.go @@ -65,10 +65,10 @@ func TestReloadFirewall_CertUnsafeNetworksChanged(t *testing.T) { pki.cs.Store(&CertState{v2Cert: c2, initiatingVersion: cert.Version2}) // Reload with the same YAML so HasChanged("firewall") reports false. - require.NoError(t, cfg.ReloadConfigString(rawYAML)) + require.NoError(t, cfg.ReloadConfigString(rawYAML).Err) require.False(t, cfg.HasChanged("firewall")) - f.reloadFirewall(cfg) + f.reloadFirewall(cfg, nil) assert.NotSame(t, fw, f.firewall, "firewall pointer should have been replaced") assert.Equal(t, newUnsafe, f.firewall.unsafeNetworks) @@ -113,8 +113,30 @@ func TestReloadFirewall_NoChange(t *testing.T) { l: l, } - require.NoError(t, cfg.ReloadConfigString(rawYAML)) - f.reloadFirewall(cfg) + require.NoError(t, cfg.ReloadConfigString(rawYAML).Err) + f.reloadFirewall(cfg, nil) assert.Same(t, fw, f.firewall, "firewall should not have been replaced") } + +// Rules that don't parse keep the old firewall, and the reload says it could not apply them +func TestReloadFirewall_BadRulesFailTheReload(t *testing.T) { + l := test.NewLogger() + pki := &PKI{} + pki.cs.Store(&CertState{v2Cert: &dummyCert{version: cert.Version2, networks: []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}}, initiatingVersion: cert.Version2}) + + rules := func(proto string) string { + return "firewall:\n outbound:\n - port: any\n proto: any\n host: any\n inbound:\n - port: any\n proto: " + proto + "\n host: any\n" + } + cfg := config.NewC(l) + require.NoError(t, cfg.LoadString(rules("any"))) + fw, err := NewFirewallFromConfig(l, pki.getCertState(), cfg) + require.NoError(t, err) + f := &Interface{pki: pki, firewall: fw, l: l} + cfg.RegisterReloadCallback(f.reloadFirewall) + + res := cfg.ReloadConfigString(rules("nope")) + require.NoError(t, res.Err) + assert.True(t, res.Failed) + assert.Same(t, fw, f.firewall) +} diff --git a/lighthouse.go b/lighthouse.go index 619c16a0..5e610111 100644 --- a/lighthouse.go +++ b/lighthouse.go @@ -143,13 +143,16 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c return nil, err } - c.RegisterReloadCallback(func(c *config.C) { + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + r.RestartRequiredIfChanged("lighthouse.am_lighthouse", "listen.port", "handshakes.query_buffer", "stats.lighthouse_metrics") err := h.reload(c, false) switch v := err.(type) { case *util.ContextualError: v.Log(l) + r.Failed() case error: l.Error("failed to reload lighthouse", "error", err) + r.Failed() } }) diff --git a/lighthouse_test.go b/lighthouse_test.go index ace29c15..4d1c5368 100644 --- a/lighthouse_test.go +++ b/lighthouse_test.go @@ -87,15 +87,15 @@ func TestReloadLighthouseInterval(t *testing.T) { lh.ifce = &mockEncWriter{} // The first one routine is kicked off by main.go currently, lets make sure that one dies - require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 5")) + require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 5").Err) assert.Equal(t, int64(5), lh.interval.Load()) // Subsequent calls are killed off by the LightHouse.Reload function - require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 10")) + require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 10").Err) assert.Equal(t, int64(10), lh.interval.Load()) // If this completes then nothing is stealing our reload routine - require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 11")) + require.NoError(t, c.ReloadConfigString("lighthouse:\n interval: 11").Err) assert.Equal(t, int64(11), lh.interval.Load()) } @@ -334,7 +334,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) { } rc, err := yaml.Marshal(nc) require.NoError(t, err) - require.NoError(t, c.ReloadConfigString(string(rc))) + require.NoError(t, c.ReloadConfigString(string(rc)).Err) rl := lh.Query(staticHost) require.NotNil(t, rl) @@ -350,7 +350,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) { } rc, err = yaml.Marshal(nc) require.NoError(t, err) - require.NoError(t, c.ReloadConfigString(string(rc))) + require.NoError(t, c.ReloadConfigString(string(rc)).Err) rl = lh.Query(staticHost) require.NotNil(t, rl) @@ -358,7 +358,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) { assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.1.1.1:4242")}, rl.CopyAddrs([]netip.Prefix{})) // Reload with the same config. An unchanged entry must not duplicate. - require.NoError(t, c.ReloadConfigString(string(rc))) + require.NoError(t, c.ReloadConfigString(string(rc)).Err) rl = lh.Query(staticHost) require.NotNil(t, rl) @@ -373,7 +373,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) { } rc, err = yaml.Marshal(nc) require.NoError(t, err) - require.NoError(t, c.ReloadConfigString(string(rc))) + require.NoError(t, c.ReloadConfigString(string(rc)).Err) // Add a second host alongside the first. Both should be present, neither duplicated. nc = map[string]any{ @@ -384,7 +384,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) { } rc, err = yaml.Marshal(nc) require.NoError(t, err) - require.NoError(t, c.ReloadConfigString(string(rc))) + require.NoError(t, c.ReloadConfigString(string(rc)).Err) rl = lh.Query(staticHost) require.NotNil(t, rl) @@ -406,7 +406,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) { } rc, err = yaml.Marshal(nc) require.NoError(t, err) - require.NoError(t, c.ReloadConfigString(string(rc))) + require.NoError(t, c.ReloadConfigString(string(rc)).Err) _, isStatic := lh.GetStaticHostList()[staticHost] assert.False(t, isStatic) @@ -966,13 +966,13 @@ func TestLighthouse_reloadRelaysTriggersUpdate(t *testing.T) { require.NoError(t, err) assert.Empty(t, lh.updateTrigger) - require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6]\n")) + require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6]\n").Err) assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.6")}, lh.GetRelaysForMe()) require.Len(t, lh.updateTrigger, 1) <-lh.updateTrigger // The config changed but what we advertise did not, a bad entry is skipped - require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6, nope]\n")) + require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6, nope]\n").Err) assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.6")}, lh.GetRelaysForMe()) assert.Empty(t, lh.updateTrigger) } @@ -988,12 +988,29 @@ func TestLighthouse_reloadAmRelayAlone(t *testing.T) { assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.5")}, lh.GetRelaysForMe()) // Relays aren't allowed to have relays, flipping am_relay alone must drop them and tell the lighthouses - require.NoError(t, c.ReloadConfigString(base+" am_relay: true\n")) + require.NoError(t, c.ReloadConfigString(base+" am_relay: true\n").Err) assert.Empty(t, lh.GetRelaysForMe()) require.Len(t, lh.updateTrigger, 1) <-lh.updateTrigger - require.NoError(t, c.ReloadConfigString(base+" am_relay: false\n")) + require.NoError(t, c.ReloadConfigString(base+" am_relay: false\n").Err) assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.5")}, lh.GetRelaysForMe()) require.Len(t, lh.updateTrigger, 1) } + +func TestLighthouse_reloadRestartRequired(t *testing.T) { + l := test.NewLogger() + c := config.NewC(l) + base := "lighthouse:\n am_lighthouse: true\n" + require.NoError(t, c.LoadString(base+"listen:\n port: 4242\n")) + + _, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil) + require.NoError(t, err) + + res := c.ReloadConfigString(base + "listen:\n port: 4242\n") + require.NoError(t, res.Err) + assert.False(t, res.RestartRequired) + res = c.ReloadConfigString(base + "listen:\n port: 4243\n") + require.NoError(t, res.Err) + assert.True(t, res.RestartRequired) +} diff --git a/main.go b/main.go index a8d01779..e7adfd58 100644 --- a/main.go +++ b/main.go @@ -197,7 +197,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev if err != nil { return nil, util.NewContextualError("Failed to open udp listener", m{"queue": i}, err) } - udpServer.ReloadConfig(c) + udpServer.ReloadConfig(c, nil) udpConns[i] = udpServer // If port is dynamic, discover it before the next pass through the for loop @@ -311,6 +311,16 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev lightHouse.ifce = ifce ifce.RegisterConfigChangeCallbacks(c) + // These are only read at startup + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + r.RestartRequiredIfChanged( + "routines", "tun.routines", "listen.routines", "firewall.conntrack.routine_cache_timeout", + "listen.port", "listen.host", "stats.message_metrics", + "handshakes.try_interval", "handshakes.retries", "handshakes.trigger_buffer", + "tun.drop_local_broadcast", "tun.drop_multicast", + ) + r.RestartRequiredIfChanged(osStartupOnlyKeys...) + }) ifce.reloadDisconnectInvalid(c) ifce.reloadSendRecvError(c) ifce.reloadAcceptRecvError(c) diff --git a/main_restart_keys_linux.go b/main_restart_keys_linux.go new file mode 100644 index 00000000..5f7071c1 --- /dev/null +++ b/main_restart_keys_linux.go @@ -0,0 +1,6 @@ +//go:build linux && !android && !e2e_testing + +package nebula + +// osStartupOnlyKeys are startup only keys that only matter on linux: udp batch size, udp offloads and thread pinning +var osStartupOnlyKeys = []string{"listen.batch", "listen.udp_offloads", "tun.pin_threads", "tun.pin_threads_key", "tun.cpu_affinity"} diff --git a/main_restart_keys_other.go b/main_restart_keys_other.go new file mode 100644 index 00000000..faaaed80 --- /dev/null +++ b/main_restart_keys_other.go @@ -0,0 +1,6 @@ +//go:build !linux || android || e2e_testing + +package nebula + +// osStartupOnlyKeys is empty, none of the OS specific startup keys matter here +var osStartupOnlyKeys []string diff --git a/overlay/tun.go b/overlay/tun.go index 0ba130b6..dbf2e843 100644 --- a/overlay/tun.go +++ b/overlay/tun.go @@ -41,9 +41,16 @@ func (e *NameError) Error() string { type DeviceFactory func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) { + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + r.RestartRequiredIfChanged("tun.disabled") + }) + switch { case c.GetBool("tun.disabled", false): tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 5000), c.GetBool("stats.message_metrics", false), l) + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + r.RestartRequiredIfChanged("tun.tx_queue", "stats.message_metrics") + }) return tun, nil default: diff --git a/overlay/tun_android.go b/overlay/tun_android.go index 819aa1f7..4d21f526 100644 --- a/overlay/tun_android.go +++ b/overlay/tun_android.go @@ -38,16 +38,17 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip l: l, } - err := t.reload(c, true) + err := t.reload(c, true, nil) if err != nil { _ = file.Close() return nil, err } - c.RegisterReloadCallback(func(c *config.C) { - err := t.reload(c, false) + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + err := t.reload(c, false, r) if err != nil { util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + r.Failed() } }) @@ -67,7 +68,7 @@ func (t *tun) Activate() error { return nil } -func (t *tun) reload(c *config.C, initial bool) error { +func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) if err != nil { return err diff --git a/overlay/tun_bypass_windows.go b/overlay/tun_bypass_windows.go index 1f62373c..d5a9708b 100644 --- a/overlay/tun_bypass_windows.go +++ b/overlay/tun_bypass_windows.go @@ -10,6 +10,9 @@ import ( "github.com/slackhq/nebula/wfp" ) +// interfaceBypassKeys are only read when the interface bypass is installed +var interfaceBypassKeys = []string{"tun.windows_bypass_wdf"} + // installInterfaceBypass installs a WFP PERMIT filter scoped to the wintun interface LUID so inbound traffic on the // nebula adapter bypasses Windows Defender Firewall. func installInterfaceBypass(l *slog.Logger, luid uint64) closer { diff --git a/overlay/tun_bypass_windows_386.go b/overlay/tun_bypass_windows_386.go index d8878b60..4cf9fb04 100644 --- a/overlay/tun_bypass_windows_386.go +++ b/overlay/tun_bypass_windows_386.go @@ -4,6 +4,9 @@ package overlay import "log/slog" +// interfaceBypassKeys is empty, windows-386 has no bypass +var interfaceBypassKeys []string + // installInterfaceBypass is a no-op on windows-386 because we don't currently build for it. func installInterfaceBypass(_ *slog.Logger, _ uint64) closer { return nil diff --git a/overlay/tun_darwin.go b/overlay/tun_darwin.go index 0824d8bf..dcde7fae 100644 --- a/overlay/tun_darwin.go +++ b/overlay/tun_darwin.go @@ -132,15 +132,17 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t l: l, } - err = t.reload(c, true) + err = t.reload(c, true, nil) if err != nil { return nil, err } - c.RegisterReloadCallback(func(c *config.C) { - err := t.reload(c, false) + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + r.RestartRequiredIfChanged("tun.dev", "tun.mtu") + err := t.reload(c, false, r) if err != nil { util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + r.Failed() } }) @@ -174,14 +176,15 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip l: l, } - if err := t.reload(c, true); err != nil { + if err := t.reload(c, true, nil); err != nil { _ = file.Close() return nil, err } - c.RegisterReloadCallback(func(c *config.C) { - if err := t.reload(c, false); err != nil { + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + if err := t.reload(c, false, r); err != nil { util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + r.Failed() } }) @@ -268,7 +271,7 @@ func (t *tun) Activate() error { } // Unsafe path routes - return t.addRoutes(false) + return t.addRoutes(false, nil) } // activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly. @@ -366,7 +369,7 @@ func (t *tun) activate6(network netip.Prefix) error { return nil } -func (t *tun) reload(c *config.C, initial bool) error { +func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) if err != nil { return err @@ -387,16 +390,18 @@ func (t *tun) reload(c *config.C, initial bool) error { if !initial { // Remove first, if the system removes a wanted route hopefully it will be re-added next - err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) + err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r) if err != nil { util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) + r.Failed() } // Ensure any routes we actually want are installed - err = t.addRoutes(true) + err = t.addRoutes(true, r) if err != nil { // Catch any stray logs util.LogWithContextIfNeeded("Failed to add routes", err, t.l) + r.Failed() } } @@ -489,7 +494,7 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) { return nil, nil } -func (t *tun) addRoutes(logErrors bool) error { +func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error { // The route tree is still ours, the system routing table is not if t.hostOwned { return nil @@ -524,6 +529,7 @@ func (t *tun) addRoutes(logErrors bool) error { retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { retErr.Log(t.l) + rl.Failed() } else { return retErr } @@ -538,7 +544,7 @@ func (t *tun) addRoutes(logErrors bool) error { return nil } -func (t *tun) removeRoutes(routes []Route) error { +func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error { linkAddr := t.linkAddr.Load() if t.hostOwned || linkAddr == nil { return nil @@ -551,8 +557,12 @@ func (t *tun) removeRoutes(routes []Route) error { } err := delRoute(r.Cidr, linkAddr) - if err != nil { + if errors.Is(err, unix.ESRCH) { + // Already gone, which is what we wanted + t.l.Info("Route was already removed", "route", r) + } else if err != nil { t.l.Error("Failed to remove route", "error", err, "route", r) + rl.Failed() } else { t.l.Info("Removed route", "route", r) } diff --git a/overlay/tun_darwin_route_test.go b/overlay/tun_darwin_route_test.go index 72fe008b..a566dd38 100644 --- a/overlay/tun_darwin_route_test.go +++ b/overlay/tun_darwin_route_test.go @@ -72,8 +72,8 @@ func TestTunRoutesBeforeActivate(t *testing.T) { }} tn.Routes.Store(&routes) - assert.NoError(t, tn.addRoutes(true)) - assert.NoError(t, tn.removeRoutes(routes)) + assert.NoError(t, tn.addRoutes(true, nil)) + assert.NoError(t, tn.removeRoutes(routes, nil)) } // A tun.routes entry with no via is never installed, so removing it must not delete a route for the same cidr that @@ -96,7 +96,9 @@ func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) { t.Cleanup(func() { _ = tn.Close() }) require.NoError(t, tn.Activate()) - require.NoError(t, c.ReloadConfigString("tun: {}\n")) + res := c.ReloadConfigString("tun: {}\n") + require.NoError(t, res.Err) + assert.False(t, res.Failed, "removing a route we never installed failed the reload") msgs, err := fetchRoutes() require.NoError(t, err) assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))) diff --git a/overlay/tun_freebsd.go b/overlay/tun_freebsd.go index 8ed90daf..d68e3df6 100644 --- a/overlay/tun_freebsd.go +++ b/overlay/tun_freebsd.go @@ -385,16 +385,18 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t }, } - err = t.reload(c, true) + err = t.reload(c, true, nil) if err != nil { return nil, err } closeOnErr = false - c.RegisterReloadCallback(func(c *config.C) { - err := t.reload(c, false) + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + r.RestartRequiredIfChanged("tun.dev", "tun.mtu") + err := t.reload(c, false, r) if err != nil { util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + r.Failed() } }) @@ -493,7 +495,7 @@ func (t *tun) Activate() error { } } - return t.addRoutes(false) + return t.addRoutes(false, nil) } func (t *tun) setMTU() error { @@ -509,7 +511,7 @@ func (t *tun) setMTU() error { return err } -func (t *tun) reload(c *config.C, initial bool) error { +func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) if err != nil { return err @@ -530,16 +532,18 @@ func (t *tun) reload(c *config.C, initial bool) error { if !initial { // Remove first, if the system removes a wanted route hopefully it will be re-added next - err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) + err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r) if err != nil { util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) + r.Failed() } // Ensure any routes we actually want are installed - err = t.addRoutes(true) + err = t.addRoutes(true, r) if err != nil { // Catch any stray logs util.LogWithContextIfNeeded("Failed to add routes", err, t.l) + r.Failed() } } @@ -563,7 +567,7 @@ func (t *tun) Queues(int) ([]tio.Queue, error) { return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil } -func (t *tun) addRoutes(logErrors bool) error { +func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error { routes := *t.Routes.Load() for _, r := range routes { if len(r.Via) == 0 || !r.Install { @@ -576,6 +580,7 @@ func (t *tun) addRoutes(logErrors bool) error { retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { retErr.Log(t.l) + rl.Failed() } else { return retErr } @@ -587,7 +592,7 @@ func (t *tun) addRoutes(logErrors bool) error { return nil } -func (t *tun) removeRoutes(routes []Route) error { +func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error { for _, r := range routes { if len(r.Via) == 0 || !r.Install { // addRoutes never installs these @@ -595,8 +600,12 @@ func (t *tun) removeRoutes(routes []Route) error { } err := delRoute(r.Cidr, t.linkAddr) - if err != nil { + if errors.Is(err, unix.ESRCH) { + // Already gone, which is what we wanted + t.l.Info("Route was already removed", "route", r) + } else if err != nil { t.l.Error("Failed to remove route", "error", err, "route", r) + rl.Failed() } else { t.l.Info("Removed route", "route", r) } diff --git a/overlay/tun_ios.go b/overlay/tun_ios.go index 9ee30f77..cbbad940 100644 --- a/overlay/tun_ios.go +++ b/overlay/tun_ios.go @@ -47,16 +47,17 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip l: l, } - err := t.reload(c, true) + err := t.reload(c, true, nil) if err != nil { _ = file.Close() return nil, err } - c.RegisterReloadCallback(func(c *config.C) { - err := t.reload(c, false) + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + err := t.reload(c, false, r) if err != nil { util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + r.Failed() } }) @@ -67,7 +68,7 @@ func (t *tun) Activate() error { return nil } -func (t *tun) reload(c *config.C, initial bool) error { +func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) if err != nil { return err diff --git a/overlay/tun_linux.go b/overlay/tun_linux.go index 5e27ad9b..541ef6b8 100644 --- a/overlay/tun_linux.go +++ b/overlay/tun_linux.go @@ -185,6 +185,10 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue } t.Device = name + // Only read when we create the device, an fd handed to us already has its name and offloads + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + r.RestartRequiredIfChanged("tun.dev", "tun.use_offloads") + }) return t, nil } @@ -228,22 +232,24 @@ func newTunGeneric(c *config.C, l *slog.Logger, fd int, vnetHdr bool, offloadFla l: l, } - if err = t.reload(c, true); err != nil { + if err = t.reload(c, true, nil); err != nil { _ = t.Close() return nil, err } - c.RegisterReloadCallback(func(c *config.C) { - err := t.reload(c, false) + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + r.RestartRequiredIfChanged("tun.tx_queue", "tun.use_system_route_table", "tun.use_system_route_table_buffer_size") + err := t.reload(c, false, r) if err != nil { util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + r.Failed() } }) return t, nil } -func (t *tun) reload(c *config.C, initial bool) error { +func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { routeChange, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) if err != nil { return err @@ -297,7 +303,7 @@ func (t *tun) reload(c *config.C, initial bool) error { if !initial { if oldMaxMTU != newMaxMTU { - t.setMTU() + t.setMTU(r) t.l.Info("Set max MTU", "mtu", t.MaxMTU, "oldMTU", oldMaxMTU) } @@ -305,7 +311,8 @@ func (t *tun) reload(c *config.C, initial bool) error { for i := range t.vpnNetworks { err := t.setDefaultRoute(t.vpnNetworks[i]) if err != nil { - t.l.Warn(err.Error()) + t.l.Error(err.Error()) + r.Failed() } else { t.l.Info("Set default MTU", "mtu", t.DefaultMTU, "oldMTU", oldDefaultMTU) } @@ -313,13 +320,14 @@ func (t *tun) reload(c *config.C, initial bool) error { } // Remove first, if the system removes a wanted route hopefully it will be re-added next - t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) + t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r) // Ensure any routes we actually want are installed - err = t.addRoutes(true) + err = t.addRoutes(true, r) if err != nil { // This should never be called since addRoutes should log its own errors in a reload condition util.LogWithContextIfNeeded("Failed to refresh routes", err, t.l) + r.Failed() } } @@ -474,7 +482,7 @@ func (t *tun) Activate() error { t.deviceIndex = link.Attrs().Index // Setup our default MTU - t.setMTU() + t.setMTU(nil) // Set the transmit queue length ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)} @@ -506,7 +514,7 @@ func (t *tun) Activate() error { } // Set the routes - if err = t.addRoutes(false); err != nil { + if err = t.addRoutes(false, nil); err != nil { return err } @@ -519,12 +527,13 @@ func (t *tun) Activate() error { return nil } -func (t *tun) setMTU() { +func (t *tun) setMTU(r *config.Reload) { // Set the MTU on the device ifm := ifreqMTU{Name: t.deviceBytes(), MTU: int32(t.MaxMTU)} if err := ioctl(t.ioctlFd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil { // This is currently a non fatal condition because the route table must have the MTU set appropriately as well t.l.Error("Failed to set tun mtu", "error", err) + r.Failed() } } @@ -577,7 +586,7 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error { return nil } -func (t *tun) addRoutes(logErrors bool) error { +func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error { // Path routes routes := *t.Routes.Load() for _, r := range routes { @@ -607,6 +616,7 @@ func (t *tun) addRoutes(logErrors bool) error { retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { retErr.Log(t.l) + rl.Failed() } else { return retErr } @@ -618,7 +628,7 @@ func (t *tun) addRoutes(logErrors bool) error { return nil } -func (t *tun) removeRoutes(routes []Route) { +func (t *tun) removeRoutes(routes []Route, rl *config.Reload) { for _, r := range routes { if !r.Install { continue @@ -642,8 +652,12 @@ func (t *tun) removeRoutes(routes []Route) { } err := netlink.RouteDel(&nr) - if err != nil { + if errors.Is(err, unix.ESRCH) { + // Already gone, which is what we wanted + t.l.Info("Route was already removed", "route", r) + } else if err != nil { t.l.Error("Failed to remove route", "error", err, "route", r) + rl.Failed() } else { t.l.Info("Removed route", "route", r) } diff --git a/overlay/tun_netbsd.go b/overlay/tun_netbsd.go index d9852ba5..056e5f98 100644 --- a/overlay/tun_netbsd.go +++ b/overlay/tun_netbsd.go @@ -103,15 +103,17 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t l: l, } - err = t.reload(c, true) + err = t.reload(c, true, nil) if err != nil { return nil, err } - c.RegisterReloadCallback(func(c *config.C) { - err := t.reload(c, false) + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + r.RestartRequiredIfChanged("tun.dev", "tun.mtu") + err := t.reload(c, false, r) if err != nil { util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + r.Failed() } }) @@ -324,7 +326,7 @@ func (t *tun) Activate() error { } } - return t.addRoutes(false) + return t.addRoutes(false, nil) } func (t *tun) doIoctlByName(ctl uintptr, value uint32) error { @@ -339,7 +341,7 @@ func (t *tun) doIoctlByName(ctl uintptr, value uint32) error { return err } -func (t *tun) reload(c *config.C, initial bool) error { +func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) if err != nil { return err @@ -360,16 +362,18 @@ func (t *tun) reload(c *config.C, initial bool) error { if !initial { // Remove first, if the system removes a wanted route hopefully it will be re-added next - err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) + err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r) if err != nil { util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) + r.Failed() } // Ensure any routes we actually want are installed - err = t.addRoutes(true) + err = t.addRoutes(true, r) if err != nil { // Catch any stray logs util.LogWithContextIfNeeded("Failed to add routes", err, t.l) + r.Failed() } } @@ -393,7 +397,7 @@ func (t *tun) Queues(int) ([]tio.Queue, error) { return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil } -func (t *tun) addRoutes(logErrors bool) error { +func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error { routes := *t.Routes.Load() for _, r := range routes { @@ -407,6 +411,7 @@ func (t *tun) addRoutes(logErrors bool) error { retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { retErr.Log(t.l) + rl.Failed() } else { return retErr } @@ -418,7 +423,7 @@ func (t *tun) addRoutes(logErrors bool) error { return nil } -func (t *tun) removeRoutes(routes []Route) error { +func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error { for _, r := range routes { if len(r.Via) == 0 || !r.Install { // addRoutes never installs these @@ -426,8 +431,12 @@ func (t *tun) removeRoutes(routes []Route) error { } err := delRoute(r.Cidr, t.vpnNetworks) - if err != nil { + if errors.Is(err, unix.ESRCH) { + // Already gone, which is what we wanted + t.l.Info("Route was already removed", "route", r) + } else if err != nil { t.l.Error("Failed to remove route", "error", err, "route", r) + rl.Failed() } else { t.l.Info("Removed route", "route", r) } diff --git a/overlay/tun_openbsd.go b/overlay/tun_openbsd.go index b310a0d1..d9aa7a30 100644 --- a/overlay/tun_openbsd.go +++ b/overlay/tun_openbsd.go @@ -94,15 +94,17 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t l: l, } - err = t.reload(c, true) + err = t.reload(c, true, nil) if err != nil { return nil, err } - c.RegisterReloadCallback(func(c *config.C) { - err := t.reload(c, false) + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + r.RestartRequiredIfChanged("tun.dev", "tun.mtu") + err := t.reload(c, false, r) if err != nil { util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + r.Failed() } }) @@ -303,7 +305,7 @@ func (t *tun) Activate() error { } } - return t.addRoutes(false) + return t.addRoutes(false, nil) } func (t *tun) doIoctlByName(ctl uintptr, value uint32) error { @@ -318,7 +320,7 @@ func (t *tun) doIoctlByName(ctl uintptr, value uint32) error { return err } -func (t *tun) reload(c *config.C, initial bool) error { +func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) if err != nil { return err @@ -339,16 +341,18 @@ func (t *tun) reload(c *config.C, initial bool) error { if !initial { // Remove first, if the system removes a wanted route hopefully it will be re-added next - err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes)) + err := t.removeRoutes(findRemovedRoutes(routes, *oldRoutes), r) if err != nil { util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) + r.Failed() } // Ensure any routes we actually want are installed - err = t.addRoutes(true) + err = t.addRoutes(true, r) if err != nil { // Catch any stray logs util.LogWithContextIfNeeded("Failed to add routes", err, t.l) + r.Failed() } } @@ -372,7 +376,7 @@ func (t *tun) Queues(int) ([]tio.Queue, error) { return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil } -func (t *tun) addRoutes(logErrors bool) error { +func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error { routes := *t.Routes.Load() for _, r := range routes { @@ -386,6 +390,7 @@ func (t *tun) addRoutes(logErrors bool) error { retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { retErr.Log(t.l) + rl.Failed() } else { return retErr } @@ -397,7 +402,7 @@ func (t *tun) addRoutes(logErrors bool) error { return nil } -func (t *tun) removeRoutes(routes []Route) error { +func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error { for _, r := range routes { if len(r.Via) == 0 || !r.Install { // addRoutes never installs these @@ -405,8 +410,12 @@ func (t *tun) removeRoutes(routes []Route) error { } err := delRoute(r.Cidr, t.vpnNetworks) - if err != nil { + if errors.Is(err, unix.ESRCH) { + // Already gone, which is what we wanted + t.l.Info("Route was already removed", "route", r) + } else if err != nil { t.l.Error("Failed to remove route", "error", err, "route", r) + rl.Failed() } else { t.l.Info("Removed route", "route", r) } diff --git a/overlay/tun_reload_linux_test.go b/overlay/tun_reload_linux_test.go index 0fcf48f4..de358ebf 100644 --- a/overlay/tun_reload_linux_test.go +++ b/overlay/tun_reload_linux_test.go @@ -4,6 +4,7 @@ package overlay import ( "net/netip" + "os" "testing" "github.com/slackhq/nebula/config" @@ -25,10 +26,10 @@ func TestReloadMTUOnlyKeepsRoutes(t *testing.T) { ioctlFd: ^uintptr(0), routesFromSystem: map[netip.Prefix]routing.Gateways{}, } - require.NoError(t, tn.reload(c, true)) + require.NoError(t, tn.reload(c, true, nil)) - require.NoError(t, c.ReloadConfigString("tun:\n mtu: 1400\n"+routes)) - _ = tn.reload(c, false) + require.NoError(t, c.ReloadConfigString("tun:\n mtu: 1400\n"+routes).Err) + _ = tn.reload(c, false, nil) got := *tn.Routes.Load() require.Len(t, got, 1) @@ -50,7 +51,57 @@ func TestReloadMaxMTUIsTheLargestRoute(t *testing.T) { ioctlFd: ^uintptr(0), routesFromSystem: map[netip.Prefix]routing.Gateways{}, } - require.NoError(t, tn.reload(c, true)) + require.NoError(t, tn.reload(c, true, nil)) assert.Equal(t, 1500, tn.MaxMTU) assert.Equal(t, 1300, tn.DefaultMTU) } + +// There is no device behind this tun, so adding a route always fails. Removing one is refused without root, as root the +// kernel answers that the route isn't there, which is what we wanted +func TestReloadRouteFailures(t *testing.T) { + route := "tun:\n mtu: 1300\n unsafe_routes:\n - route: 192.168.100.0/24\n via: 10.1.0.1\n" + cases := []struct { + name string + from, to string + wantFails bool + }{ + {"added", "tun:\n mtu: 1300\n", route, true}, + {"removed", route, "tun:\n mtu: 1300\n", os.Geteuid() != 0}, + {"unchanged", route, route + "punchy:\n punch: true\n", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString(tc.from)) + tn := &tun{ + l: test.NewLogger(), + vpnNetworks: []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, + ioctlFd: ^uintptr(0), + routesFromSystem: map[netip.Prefix]routing.Gateways{}, + } + require.NoError(t, tn.reload(c, true, nil)) + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + require.NoError(t, tn.reload(c, false, r)) + }) + + res := c.ReloadConfigString(tc.to) + require.NoError(t, res.Err) + assert.Equal(t, tc.wantFails, res.Failed) + }) + } +} + +// Keys only read when the device is created need a restart to apply +func TestTunReloadRestartKeys(t *testing.T) { + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString("tun:\n dev: nebrk0\n")) + tn, err := NewDeviceFromConfig(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, 1) + if err != nil { + t.Skipf("can't create a tun device here: %v", err) + } + t.Cleanup(func() { _ = tn.Close() }) + + res := c.ReloadConfigString("tun:\n disabled: true\n dev: nebrk1\n tx_queue: 100\n") + require.NoError(t, res.Err) + assert.ElementsMatch(t, []string{"tun.disabled", "tun.dev", "tun.tx_queue"}, res.RestartKeys) +} diff --git a/overlay/tun_test.go b/overlay/tun_test.go new file mode 100644 index 00000000..16354fe5 --- /dev/null +++ b/overlay/tun_test.go @@ -0,0 +1,24 @@ +package overlay + +import ( + "net/netip" + "testing" + + "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// tun.disabled and the disabled tun's settings are only read when the device is made +func TestDisabledTunRestartKeys(t *testing.T) { + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString("tun:\n disabled: true\n")) + d, err := NewDeviceFromConfig(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, 1) + require.NoError(t, err) + t.Cleanup(func() { _ = d.Close() }) + + res := c.ReloadConfigString("tun:\n disabled: false\n tx_queue: 100\n") + require.NoError(t, res.Err) + assert.ElementsMatch(t, []string{"tun.disabled", "tun.tx_queue"}, res.RestartKeys) +} diff --git a/overlay/tun_windows.go b/overlay/tun_windows.go index 18680f2b..fcb6ac93 100644 --- a/overlay/tun_windows.go +++ b/overlay/tun_windows.go @@ -84,7 +84,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w l: l, } - err = t.reload(c, true) + err = t.reload(c, true, nil) if err != nil { return nil, err } @@ -108,17 +108,20 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w // Disable registration before Activate assigns addresses and triggers it. disableDNSRegistration(l, *guid) - c.RegisterReloadCallback(func(c *config.C) { - err := t.reload(c, false) + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + r.RestartRequiredIfChanged("tun.dev", "tun.mtu", "tun.network_category") + r.RestartRequiredIfChanged(interfaceBypassKeys...) + err := t.reload(c, false, r) if err != nil { util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) + r.Failed() } }) return t, nil } -func (t *winTun) reload(c *config.C, initial bool) error { +func (t *winTun) reload(c *config.C, initial bool, r *config.Reload) error { change, routes, err := getAllRoutesFromConfig(t.l, c, t.vpnNetworks, initial) if err != nil { return err @@ -139,16 +142,18 @@ func (t *winTun) reload(c *config.C, initial bool) error { if !initial { // Remove first, if the system removes a wanted route hopefully it will be re-added next - err := t.removeRoutes(findRemovedSystemRoutes(routes, *oldRoutes)) + err := t.removeRoutes(findRemovedSystemRoutes(routes, *oldRoutes), r) if err != nil { util.LogWithContextIfNeeded("Failed to remove routes", err, t.l) + r.Failed() } // Ensure any routes we actually want are installed - err = t.addRoutes(true) + err = t.addRoutes(true, r) if err != nil { // Catch any stray logs util.LogWithContextIfNeeded("Failed to add routes", err, t.l) + r.Failed() } } @@ -163,7 +168,7 @@ func (t *winTun) Activate() error { return fmt.Errorf("failed to set address: %w", err) } - err = t.addRoutes(false) + err = t.addRoutes(false, nil) if err != nil { return err } @@ -182,7 +187,7 @@ func (t *winTun) Activate() error { return nil } -func (t *winTun) addRoutes(logErrors bool) error { +func (t *winTun) addRoutes(logErrors bool, rl *config.Reload) error { luid := winipcfg.LUID(t.tun.LUID()) routes := *t.Routes.Load() foundDefault4 := false @@ -213,6 +218,7 @@ func (t *winTun) addRoutes(logErrors bool) error { retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) if logErrors { retErr.Log(t.l) + rl.Failed() continue } else { return retErr @@ -274,7 +280,7 @@ func (t *winTun) setMTU(luid winipcfg.LUID, foundDefault4, carriesV6 bool) error return nil } -func (t *winTun) removeRoutes(routes []Route) error { +func (t *winTun) removeRoutes(routes []Route, rl *config.Reload) error { luid := winipcfg.LUID(t.tun.LUID()) for _, r := range routes { @@ -284,8 +290,12 @@ func (t *winTun) removeRoutes(routes []Route) error { // See comment on luid.AddRoute err := luid.DeleteRoute(r.Cidr, unspecifiedNextHop(r.Cidr)) - if err != nil { + if errors.Is(err, windows.ERROR_NOT_FOUND) { + // Already gone, which is what we wanted + t.l.Info("Route was already removed", "route", r) + } else if err != nil { t.l.Error("Failed to remove route", "error", err, "route", r) + rl.Failed() } else { t.l.Info("Removed route", "route", r) } diff --git a/overlay/tun_windows_route_test.go b/overlay/tun_windows_route_test.go index 71c5c232..1546dfd6 100644 --- a/overlay/tun_windows_route_test.go +++ b/overlay/tun_windows_route_test.go @@ -53,6 +53,14 @@ func (rt *routeTest) metric(t *testing.T, cidr string) uint32 { return row.Metric } +// reload applies cfg and expects nothing in it to fail +func (rt *routeTest) reload(t *testing.T, cfg string) { + t.Helper() + res := rt.c.ReloadConfigString(cfg) + require.NoError(t, res.Err) + assert.False(t, res.Failed, "the reload reported a failure") +} + func unsafeRoutes(routes ...string) string { var b strings.Builder b.WriteString("tun:\n dev: nebroutetest\n unsafe_routes:\n") @@ -77,7 +85,7 @@ func TestWinTun_ReloadKeepsInstalledRoutes(t *testing.T) { rt := newRouteTest(t, unsafeRoutes(a)) require.NoError(t, rt.tun.Activate()) - require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a, b))) + rt.reload(t, unsafeRoutes(a, b)) assert.NotContains(t, rt.log.String(), "Failed to add route") rt.metric(t, a) rt.metric(t, b) @@ -92,7 +100,7 @@ func TestWinTun_ReloadMetricChangeApplies(t *testing.T) { require.NoError(t, rt.tun.Activate()) require.Equal(t, uint32(100), rt.metric(t, a)) - require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@200"))) + rt.reload(t, unsafeRoutes(a+"@200")) assert.NotContains(t, rt.log.String(), "Failed to add route") assert.NotContains(t, rt.log.String(), "Removed route") assert.Equal(t, uint32(200), rt.metric(t, a)) @@ -120,7 +128,7 @@ func TestWinTun_DuplicateCidrInstallsTheLastEntry(t *testing.T) { require.NoError(t, rt.tun.Activate()) assert.Equal(t, uint32(200), rt.metric(t, a)) - require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@100", a+"@200", b))) + rt.reload(t, unsafeRoutes(a+"@100", a+"@200", b)) assert.Equal(t, uint32(200), rt.metric(t, a)) assert.NotContains(t, rt.log.String(), "Updated route metric") assert.NotContains(t, rt.log.String(), "Failed to add route") @@ -134,7 +142,7 @@ func TestWinTun_DroppingAnUninstalledDuplicateKeepsTheRoute(t *testing.T) { rt := newRouteTest(t, unsafeRoutes(a+"@100", a+"@200")) require.NoError(t, rt.tun.Activate()) - require.NoError(t, rt.c.ReloadConfigString(unsafeRoutes(a+"@200"))) + rt.reload(t, unsafeRoutes(a+"@200")) assert.NotContains(t, rt.log.String(), "Removed route") assert.Equal(t, uint32(200), rt.metric(t, a)) }) @@ -155,12 +163,24 @@ func TestWinTun_ReloadMatchesAFreshStart(t *testing.T) { rt := newRouteTest(t, unsafeRoutes(a+"@100")) require.NoError(t, rt.tun.Activate()) - require.NoError(t, rt.c.ReloadConfigString(routes)) + rt.reload(t, routes) _, err = rt.luid.Route(cidr, unspecifiedNextHop(cidr)) assert.Error(t, err, "the reload left it installed") }) } +// Removing a route that is already gone is what we wanted, not a failure +func TestWinTun_ReloadRouteAlreadyGone(t *testing.T) { + forFamilies(t, func(t *testing.T, a, b string) { + rt := newRouteTest(t, unsafeRoutes(a, b)) + require.NoError(t, rt.tun.Activate()) + p := netip.MustParsePrefix(b) + require.NoError(t, rt.luid.DeleteRoute(p, unspecifiedNextHop(p))) + + rt.reload(t, unsafeRoutes(a)) + }) +} + // forFamilies runs f with two unsafe_routes CIDRs of each address family, the tun's own network stays IPv4 func forFamilies(t *testing.T, f func(t *testing.T, a, b string)) { t.Run("ipv4", func(t *testing.T) { f(t, "10.251.1.0/24", "10.251.2.0/24") }) diff --git a/pki.go b/pki.go index 17efe7c1..0e4deea3 100644 --- a/pki.go +++ b/pki.go @@ -50,15 +50,19 @@ type CertState struct { myVpnBroadcastAddrsTable *bart.Lite } +// errCertNeedsRestart marks a new cert that is refused on reload because only a restart can apply it +var errCertNeedsRestart = errors.New("a restart is required to use the new cert") + func NewPKIFromConfig(l *slog.Logger, c *config.C) (*PKI, error) { pki := &PKI{l: l} - err := pki.reload(c, true) + err := pki.reload(c, true, nil) if err != nil { return nil, err } - c.RegisterReloadCallback(func(c *config.C) { - rErr := pki.reload(c, false) + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + r.RestartRequiredIfChanged("cipher") + rErr := pki.reload(c, false, r) if rErr != nil { util.LogWithContextIfNeeded("Failed to reload PKI from config", rErr, l) } @@ -75,13 +79,29 @@ func (p *PKI) getCertState() *CertState { return p.cs.Load() } -func (p *PKI) reload(c *config.C, initial bool) error { +// r is nil on the initial load +func (p *PKI) reload(c *config.C, initial bool, r *config.Reload) error { + before := p.cs.Load() err := p.reloadCerts(c, initial) if err != nil { if initial { return err } err.Log(p.l) + if errors.Is(err, errCertNeedsRestart) { + r.RestartRequired() + } else { + r.Failed() + } + } else if !initial { + // A cert for a version we weren't running is accepted without comparing its networks to ours. The tun and the + // lighthouse keep the networks they started with + after := p.cs.Load() + if !slices.Equal(before.myVpnNetworks, after.myVpnNetworks) { + p.l.Warn("The new cert changes our networks, a restart is required to use them", + "old_networks", before.myVpnNetworks, "new_networks", after.myVpnNetworks) + r.RestartRequired() + } } err = p.reloadCAPool(c) @@ -90,6 +110,7 @@ func (p *PKI) reload(c *config.C, initial bool) error { return err } err.Log(p.l) + r.Failed() } return nil @@ -132,7 +153,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError { return util.NewContextualError( "Networks in new cert was different from old", m{"new_networks": newState.v1Cert.Networks(), "old_networks": currentState.v1Cert.Networks(), "cert_version": cert.Version1}, - nil, + errCertNeedsRestart, ) } @@ -140,7 +161,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError { return util.NewContextualError( "Curve in new v1 cert was different from old", m{"new_curve": newState.v1Cert.Curve(), "old_curve": currentState.v1Cert.Curve(), "cert_version": cert.Version1}, - nil, + errCertNeedsRestart, ) } } @@ -155,7 +176,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError { return util.NewContextualError( "Networks in new cert was different from old", m{"new_networks": newState.v2Cert.Networks(), "old_networks": currentState.v2Cert.Networks(), "cert_version": cert.Version2}, - nil, + errCertNeedsRestart, ) } @@ -163,7 +184,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError { return util.NewContextualError( "Curve in new cert was different from old", m{"new_curve": newState.v2Cert.Curve(), "old_curve": currentState.v2Cert.Curve(), "cert_version": cert.Version2}, - nil, + errCertNeedsRestart, ) } } @@ -178,7 +199,7 @@ func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError { return util.NewContextualError( "Removing a V2 cert is not permitted unless it has identical networks to the new V1 cert", m{"new_v1_networks": newState.v1Cert.Networks(), "old_v2_networks": currentState.v2Cert.Networks()}, - nil, + errCertNeedsRestart, ) } } diff --git a/pki_test.go b/pki_test.go index 7bd1a24e..572d4c21 100644 --- a/pki_test.go +++ b/pki_test.go @@ -1,10 +1,16 @@ package nebula import ( + "fmt" + "net/netip" "strings" "testing" + "time" "github.com/slackhq/nebula/cert" + "github.com/slackhq/nebula/cert_test" + "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -92,3 +98,83 @@ func TestNewCipherSuiteUnsupportedCurve(t *testing.T) { assert.True(t, strings.HasPrefix(err.Error(), "unsupported curve:"), "got: %v", err) assert.Nil(t, cs) } + +func TestPKIReloadRestartRequired(t *testing.T) { + ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil) + pkiConfig := func(cipher, network string) string { + _, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil) + return fmt.Sprintf("cipher: %s\npki:\n ca: %q\n cert: %q\n key: %q\n", cipher, caPEM, certPEM, keyPEM) + } + + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString(pkiConfig("aes", "10.1.0.2/24"))) + p, err := NewPKIFromConfig(test.NewLogger(), c) + require.NoError(t, err) + + // A cert that changes our networks is refused and needs a restart, it is not a failure + res := c.ReloadConfigString(pkiConfig("aes", "10.1.0.3/24")) + require.NoError(t, res.Err) + assert.True(t, res.RestartRequired) + assert.False(t, res.Failed) + assert.Equal(t, []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, p.getCertState().myVpnNetworks) + + // The cipher only applies at startup + c2 := config.NewC(test.NewLogger()) + require.NoError(t, c2.LoadString(pkiConfig("aes", "10.1.0.2/24"))) + p, err = NewPKIFromConfig(test.NewLogger(), c2) + require.NoError(t, err) + res = c2.ReloadConfigString(pkiConfig("chachapoly", "10.1.0.2/24")) + require.NoError(t, res.Err) + assert.True(t, res.RestartRequired) + assert.Equal(t, "aes", p.getCertState().cipher) + + // A cert that keeps our networks applies without a restart + c3 := config.NewC(test.NewLogger()) + require.NoError(t, c3.LoadString(pkiConfig("aes", "10.1.0.2/24"))) + _, err = NewPKIFromConfig(test.NewLogger(), c3) + require.NoError(t, err) + res = c3.ReloadConfigString(pkiConfig("aes", "10.1.0.2/24")) + require.NoError(t, res.Err) + assert.False(t, res.RestartRequired) + + // A cert that can't be read is a failure, and a restart wouldn't help + res = c3.ReloadConfigString(strings.Replace(pkiConfig("aes", "10.1.0.2/24"), "cert: \"", "cert: \"garbage", 1)) + require.NoError(t, res.Err) + assert.True(t, res.Failed) + assert.False(t, res.RestartRequired) +} + +// A cert for a version we weren't running is accepted without comparing networks, so switching to one with different +// networks needs a restart. One with the same networks doesn't +func TestPKIReloadNewVersionWithOtherNetworks(t *testing.T) { + ca1, _, ca1Key, ca1PEM := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil) + ca2, _, ca2Key, ca2PEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil) + caPEM := string(ca1PEM) + string(ca2PEM) + pkiConfig := func(v cert.Version, network string) string { + ca, caKey := ca1, ca1Key + if v == cert.Version2 { + ca, caKey = ca2, ca2Key + } + _, _, keyPEM, certPEM := cert_test.NewTestCert(v, cert.Curve_CURVE25519, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix(network)}, nil, nil) + return fmt.Sprintf("pki:\n ca: %q\n cert: %q\n key: %q\n", caPEM, certPEM, keyPEM) + } + + for _, tc := range []struct { + network string + restart bool + }{ + {"10.9.0.7/24", true}, + {"10.1.0.2/24", false}, + } { + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString(pkiConfig(cert.Version1, "10.1.0.2/24"))) + p, err := NewPKIFromConfig(test.NewLogger(), c) + require.NoError(t, err) + + res := c.ReloadConfigString(pkiConfig(cert.Version2, tc.network)) + require.NoError(t, res.Err) + assert.False(t, res.Failed, tc.network) + assert.Equal(t, tc.restart, res.RestartRequired, tc.network) + assert.Equal(t, []netip.Prefix{netip.MustParsePrefix(tc.network)}, p.getCertState().myVpnNetworks, "accepted either way, as before") + } +} diff --git a/punchy.go b/punchy.go index 4bce4392..67876269 100644 --- a/punchy.go +++ b/punchy.go @@ -67,7 +67,8 @@ func NewPunchyFromConfig(l *slog.Logger, c *config.C, punchConn udp.Conn) *Punch } p.reload(c, true) - c.RegisterReloadCallback(func(c *config.C) { + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + r.RestartRequiredIfChanged("stats.lighthouse_metrics") p.reload(c, false) }) diff --git a/punchy_test.go b/punchy_test.go index e56f3eff..25f8de79 100644 --- a/punchy_test.go +++ b/punchy_test.go @@ -76,7 +76,7 @@ punchy: punch: true delay: 10m respond: true -`)) +`).Err) p.reload(c, false) assert.True(t, p.punch.Load()) assert.Equal(t, newDelay, time.Duration(p.delay.Load())) @@ -170,7 +170,7 @@ func TestPunchy_LogFormat_ReloadPunch(t *testing.T) { NewPunchyFromConfig(l, c, nil) hook.entries = nil - require.NoError(t, c.ReloadConfigString(`punchy: {punch: true}`)) + require.NoError(t, c.ReloadConfigString(`punchy: {punch: true}`).Err) entry := findEntry(t, hook.entries, "punchy.punch changed") assert.Equal(t, slog.LevelInfo, entry.Level) @@ -184,7 +184,7 @@ func TestPunchy_LogFormat_ReloadRespond(t *testing.T) { NewPunchyFromConfig(l, c, nil) hook.entries = nil - require.NoError(t, c.ReloadConfigString(`punchy: {respond: true}`)) + require.NoError(t, c.ReloadConfigString(`punchy: {respond: true}`).Err) entry := findEntry(t, hook.entries, "punchy.respond changed") assert.Equal(t, slog.LevelInfo, entry.Level) @@ -198,7 +198,7 @@ func TestPunchy_LogFormat_ReloadDelay(t *testing.T) { NewPunchyFromConfig(l, c, nil) hook.entries = nil - require.NoError(t, c.ReloadConfigString(`punchy: {delay: 10s}`)) + require.NoError(t, c.ReloadConfigString(`punchy: {delay: 10s}`).Err) entry := findEntry(t, hook.entries, "punchy.delay changed") assert.Equal(t, slog.LevelInfo, entry.Level) @@ -212,7 +212,7 @@ func TestPunchy_LogFormat_ReloadTargetAllRemotes(t *testing.T) { NewPunchyFromConfig(l, c, nil) hook.entries = nil - require.NoError(t, c.ReloadConfigString(`punchy: {target_all_remotes: true}`)) + require.NoError(t, c.ReloadConfigString(`punchy: {target_all_remotes: true}`).Err) entry := findEntry(t, hook.entries, "punchy.target_all_remotes changed") assert.Equal(t, slog.LevelInfo, entry.Level) @@ -226,9 +226,19 @@ func TestPunchy_LogFormat_ReloadRespondDelay(t *testing.T) { NewPunchyFromConfig(l, c, nil) hook.entries = nil - require.NoError(t, c.ReloadConfigString(`punchy: {respond_delay: 15s}`)) + require.NoError(t, c.ReloadConfigString(`punchy: {respond_delay: 15s}`).Err) entry := findEntry(t, hook.entries, "punchy.respond_delay changed") assert.Equal(t, slog.LevelInfo, entry.Level) assert.Equal(t, map[string]any{"respond_delay": 15 * time.Second}, entry.Attrs) } + +func TestPunchyReloadRestartRequired(t *testing.T) { + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString("stats:\n lighthouse_metrics: false\n")) + NewPunchyFromConfig(test.NewLogger(), c, nil) + + res := c.ReloadConfigString("stats:\n lighthouse_metrics: true\n") + require.NoError(t, res.Err) + assert.True(t, res.RestartRequired) +} diff --git a/relay_manager.go b/relay_manager.go index 46d7a2bb..0c8f67eb 100644 --- a/relay_manager.go +++ b/relay_manager.go @@ -28,10 +28,11 @@ func NewRelayManager(ctx context.Context, l *slog.Logger, hostmap *HostMap, c *c hostmap: hostmap, } rm.reload(c, true) - c.RegisterReloadCallback(func(c *config.C) { + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { err := rm.reload(c, false) if err != nil { rm.l.Error("Failed to reload relay_manager", "error", err) + r.Failed() } }) return rm diff --git a/ssh.go b/ssh.go index 34b5c3a7..e59cc858 100644 --- a/ssh.go +++ b/ssh.go @@ -58,7 +58,9 @@ type sshDeviceInfoFlags struct { } func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) { - c.RegisterReloadCallback(func(c *config.C) { + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + // sshd.sandbox_dir is read once, when the commands are attached + r.RestartRequiredIfChanged("sshd.sandbox_dir") if c.GetBool("sshd.enabled", false) { // Restarting the server drops every open session, the one that asked for this reload included. Only restart // when it isn't running or its config changed. The host key file can change under the same config, new @@ -71,6 +73,7 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) { if err != nil { l.Error("Failed to reconfigure the sshd", "error", err) ssh.Stop() + r.Failed() } return } @@ -78,6 +81,7 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) { if err != nil { l.Error("Failed to reconfigure the sshd", "error", err) ssh.Stop() + r.Failed() } if sshRun != nil { go sshRun() @@ -276,7 +280,7 @@ func attachCommands(l *slog.Logger, c *config.C, ssh *sshd.SSHServer, f *Interfa Name: "reload", ShortDescription: "Reloads configuration from disk, same as sending HUP to the process", Callback: func(fs any, a []string, w sshd.StringWriter) error { - return sshReload(c, w) + return sshReload(l, c, w) }, }) @@ -1075,8 +1079,13 @@ func sshDeviceInfo(ifce *Interface, fs any, w sshd.StringWriter) error { } } -func sshReload(c *config.C, w sshd.StringWriter) error { - err := w.WriteLine("Reloading config") - c.ReloadConfig() - return err +// sshReload reloads the config and writes the result to the client. It reloads even if the client has gone away +func sshReload(l *slog.Logger, c *config.C, w sshd.StringWriter) error { + writeErr := w.WriteLine("Reloading config") + res := c.ReloadConfig() + res.Log(l) + if err := w.WriteLine(res.String()); writeErr == nil { + writeErr = err + } + return writeErr } diff --git a/ssh_reload_test.go b/ssh_reload_test.go new file mode 100644 index 00000000..2727d1c7 --- /dev/null +++ b/ssh_reload_test.go @@ -0,0 +1,83 @@ +package nebula + +import ( + "errors" + "io" + "os" + "path/filepath" + "testing" + + "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type lineWriter struct { + lines []string + err error +} + +func (w *lineWriter) WriteLine(s string) error { + w.lines = append(w.lines, s) + return w.err +} +func (w *lineWriter) Write(s string) error { return w.WriteLine(s) } +func (w *lineWriter) WriteBytes(b []byte) error { return w.WriteLine(string(b)) } +func (w *lineWriter) GetWriter() io.Writer { return io.Discard } + +func TestSSHReloadReportsTheResult(t *testing.T) { + dir := t.TempDir() + write := func(body string) { + require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(body), 0o600)) + } + write("a: 1\nb: 1") + l := test.NewLogger() + c := config.NewC(l) + require.NoError(t, c.Load(dir)) + fail := false + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + r.RestartRequiredIfChanged("b") + if fail { + r.Failed() + } + }) + + reload := func() []string { + w := &lineWriter{} + require.NoError(t, sshReload(l, c, w)) + return w.lines + } + + write("a: 2\nb: 1") + assert.Equal(t, []string{"Reloading config", "Config reload complete"}, reload()) + + fail = true + write("a: 3\nb: 1") + assert.Equal(t, []string{"Reloading config", "Config reload completed with errors, review the logs"}, reload()) + + fail = false + write("a: 3\nb: 2") + assert.Equal(t, []string{"Reloading config", "Config reload was completed but requires a restart to fully apply, keys [b]"}, reload()) + + write("a: [") + lines := reload() + require.Len(t, lines, 2) + assert.Contains(t, lines[1], "Config reload failed, the new config did not load") + assert.Contains(t, lines[1], dir, "names the config that did not load") +} + +// A client that went away still gets the reload it asked for +func TestSSHReloadReloadsWhenTheClientIsGone(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte("a: 1"), 0o600)) + l := test.NewLogger() + c := config.NewC(l) + require.NoError(t, c.Load(dir)) + reloaded := false + c.RegisterReloadCallback(func(*config.C, *config.Reload) { reloaded = true }) + + gone := errors.New("client is gone") + require.ErrorIs(t, sshReload(l, c, &lineWriter{err: gone}), gone) + assert.True(t, reloaded) +} diff --git a/sshd_reload_test.go b/sshd_reload_test.go index 8226fd09..0e81cece 100644 --- a/sshd_reload_test.go +++ b/sshd_reload_test.go @@ -130,7 +130,7 @@ func TestSSHDReload_UnchangedKeepsSessions(t *testing.T) { h := newSSHDReloadHarness(t) client, _ := h.connect(t) - require.NoError(t, h.c.ReloadConfigString(h.config()+"punchy:\n punch: true\n")) + require.NoError(t, h.c.ReloadConfigString(h.config()+"punchy:\n punch: true\n").Err) assert.False(t, dropped(client, 500*time.Millisecond), "an unchanged sshd was restarted") } @@ -139,7 +139,7 @@ func TestSSHDReload_ChangedConfigRestarts(t *testing.T) { client, _ := h.connect(t) h.extra = " trusted_cas: []\n" - require.NoError(t, h.c.ReloadConfigString(h.config())) + require.NoError(t, h.c.ReloadConfigString(h.config()).Err) assert.True(t, dropped(client, 5*time.Second), "a changed sshd config was not applied") require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond) h.connect(t) @@ -152,7 +152,7 @@ func TestSSHDReload_ChangedHostKeyFile(t *testing.T) { client, _ := h.connect(t) want := h.writeHostKey(t, newEd25519Key(t)) - require.NoError(t, h.c.ReloadConfigString(h.config())) + require.NoError(t, h.c.ReloadConfigString(h.config()).Err) _, got := h.connect(t) assert.Equal(t, want.Marshal(), got.Marshal(), "a new host key was not applied") assert.False(t, dropped(client, 500*time.Millisecond), "a new host key restarted sshd") @@ -166,7 +166,7 @@ func TestSSHDReload_HostKeyAlgorithmChange(t *testing.T) { rsaKey, err := rsa.GenerateKey(rand.Reader, 2048) require.NoError(t, err) want := h.writeHostKey(t, rsaKey) - require.NoError(t, h.c.ReloadConfigString(h.config())) + require.NoError(t, h.c.ReloadConfigString(h.config()).Err) _, got := h.connect(t) assert.Equal(t, want.Marshal(), got.Marshal()) @@ -180,7 +180,7 @@ func TestSSHDReload_NotRunningRestarts(t *testing.T) { h.server.Stop() require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond) - require.NoError(t, h.c.ReloadConfigString(h.config())) + require.NoError(t, h.c.ReloadConfigString(h.config()).Err) require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond) h.connect(t) } @@ -191,6 +191,6 @@ func TestSSHDReload_BadHostKeyFileStops(t *testing.T) { h.connect(t) require.NoError(t, os.WriteFile(h.keyFile, []byte("not a key"), 0o600)) - require.NoError(t, h.c.ReloadConfigString(h.config())) + assert.True(t, h.c.ReloadConfigString(h.config()).Failed) require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond) } diff --git a/stats.go b/stats.go index 7e42d44b..9b957650 100644 --- a/stats.go +++ b/stats.go @@ -99,9 +99,10 @@ func newStatsServerFromConfig(ctx context.Context, l *slog.Logger, c *config.C, emitStats: emitStats, } - c.RegisterReloadCallback(func(c *config.C) { + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { if err := s.reload(c, false); err != nil { s.l.Error("Failed to reload stats from config", "error", err) + r.Failed() } }) diff --git a/udp/conn.go b/udp/conn.go index 6d20cbbf..62806569 100644 --- a/udp/conn.go +++ b/udp/conn.go @@ -43,7 +43,8 @@ type Conn interface { // its own packet, so a short count means some peers were undeliverable, not that the batch failed. // Not safe for concurrent use on the same Conn. WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) - ReloadConfig(c *config.C) + // ReloadConfig applies the listen settings that can change, r is nil at startup + ReloadConfig(c *config.C, r *config.Reload) SupportsMultipleReaders() bool Close() error } @@ -68,7 +69,7 @@ func (NoopConn) WriteTo(_ []byte, _ netip.AddrPort) error { func (NoopConn) WriteBatch(bufs [][]byte, _ []netip.AddrPort) (int, error) { return len(bufs), nil } -func (NoopConn) ReloadConfig(_ *config.C) { +func (NoopConn) ReloadConfig(_ *config.C, _ *config.Reload) { return } func (NoopConn) Close() error { diff --git a/udp/netchange.go b/udp/netchange.go index e7b21e6b..86956212 100644 --- a/udp/netchange.go +++ b/udp/netchange.go @@ -24,6 +24,12 @@ type NetworkChangeMonitor struct { // NewNetworkChangeMonitor builds a monitor for local network changes. The returned monitor is always usable: Start // is safe to call unconditionally, it no-ops when disabled or on a platform that does not need it. func NewNetworkChangeMonitor(ctx context.Context, l *slog.Logger, c *config.C) *NetworkChangeMonitor { + if watchesNetworkChanges { + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + r.RestartRequiredIfChanged("listen.rebind_on_network_change") + }) + } + return &NetworkChangeMonitor{ l: l, ctx: ctx, diff --git a/udp/netchange_darwin.go b/udp/netchange_darwin.go index ac67ea79..e4bc6144 100644 --- a/udp/netchange_darwin.go +++ b/udp/netchange_darwin.go @@ -13,6 +13,9 @@ import ( "golang.org/x/sys/unix" ) +// watchesNetworkChanges is true where Start does anything +const watchesNetworkChanges = true + const ( // netChangeSettleWindow is how long we keep swallowing routing messages after the first interesting one. A // single network change is never a single message, it is a burst: the link drops, addresses go away, new ones diff --git a/udp/netchange_generic.go b/udp/netchange_generic.go index 00cc9214..a210137f 100644 --- a/udp/netchange_generic.go +++ b/udp/netchange_generic.go @@ -7,6 +7,9 @@ import ( "log/slog" ) +// watchesNetworkChanges is true where Start does anything +const watchesNetworkChanges = false + // watchNetworkChanges is a no-op outside of darwin. // // Darwin is the platform that scopes a udp socket to the interface it came up on, so it is the platform whose socket diff --git a/udp/netchange_test.go b/udp/netchange_test.go index fe57a2e2..d91c6242 100644 --- a/udp/netchange_test.go +++ b/udp/netchange_test.go @@ -37,3 +37,19 @@ func TestNetworkChangeMonitorNilRebindIsANoOp(t *testing.T) { m := newMonitor(t, context.Background(), "listen:\n rebind_on_network_change: true\n") m.Start(nil) } + +// The monitor only reads listen.rebind_on_network_change when it is made +func TestNetworkChangeMonitorRestartKey(t *testing.T) { + l := test.NewLogger() + c := config.NewC(l) + require.NoError(t, c.LoadString("listen:\n rebind_on_network_change: true\n")) + NewNetworkChangeMonitor(context.Background(), l, c) + + res := c.ReloadConfigString("listen:\n rebind_on_network_change: false\n") + require.NoError(t, res.Err) + if watchesNetworkChanges { + assert.Equal(t, []string{"listen.rebind_on_network_change"}, res.RestartKeys) + } else { + assert.Empty(t, res.RestartKeys, "the setting does nothing here") + } +} diff --git a/udp/udp_bypass_windows.go b/udp/udp_bypass_windows.go index b8b06b1e..bf261223 100644 --- a/udp/udp_bypass_windows.go +++ b/udp/udp_bypass_windows.go @@ -26,7 +26,9 @@ type bypassConn struct { session *wfp.Session } -func (b *bypassConn) ReloadConfig(c *config.C) { +func (b *bypassConn) ReloadConfig(c *config.C, r *config.Reload) { + // Only read on the first call, at startup + r.RestartRequiredIfChanged("listen.windows_bypass_wdf") b.installOnce.Do(func() { if !c.GetBool("listen.windows_bypass_wdf", true) { return @@ -45,7 +47,7 @@ func (b *bypassConn) ReloadConfig(c *config.C) { "port", addr.Port()) b.session = s }) - b.Conn.ReloadConfig(c) + b.Conn.ReloadConfig(c, r) } func (b *bypassConn) Close() error { diff --git a/udp/udp_darwin.go b/udp/udp_darwin.go index 2bb0cf0d..516e5d7b 100644 --- a/udp/udp_darwin.go +++ b/udp/udp_darwin.go @@ -171,7 +171,7 @@ func (u *StdConn) LocalAddr() (netip.AddrPort, error) { } } -func (u *StdConn) ReloadConfig(c *config.C) { +func (u *StdConn) ReloadConfig(c *config.C, r *config.Reload) { // TODO } diff --git a/udp/udp_generic.go b/udp/udp_generic.go index 0ba1d412..54f00ad2 100644 --- a/udp/udp_generic.go +++ b/udp/udp_generic.go @@ -73,7 +73,7 @@ func (u *GenericConn) LocalAddr() (netip.AddrPort, error) { } } -func (u *GenericConn) ReloadConfig(c *config.C) { +func (u *GenericConn) ReloadConfig(c *config.C, r *config.Reload) { } diff --git a/udp/udp_linux.go b/udp/udp_linux.go index cc834131..f9987341 100644 --- a/udp/udp_linux.go +++ b/udp/udp_linux.go @@ -392,7 +392,7 @@ func writeSockaddr(buf []byte, addr netip.AddrPort, isV4 bool) (int, error) { return unix.SizeofSockaddrInet6, nil } -func (u *StdConn) ReloadConfig(c *config.C) { +func (u *StdConn) ReloadConfig(c *config.C, r *config.Reload) { b := c.GetInt("listen.read_buffer", 0) if b > 0 { if err := u.SetRecvBuffer(b); err == nil { @@ -403,6 +403,7 @@ func (u *StdConn) ReloadConfig(c *config.C) { } } else { u.l.Error("Failed to set listen.read_buffer", "error", err) + r.Failed() } } @@ -416,6 +417,7 @@ func (u *StdConn) ReloadConfig(c *config.C) { } } else { u.l.Error("Failed to set listen.write_buffer", "error", err) + r.Failed() } } @@ -430,6 +432,7 @@ func (u *StdConn) ReloadConfig(c *config.C) { } } else { u.l.Error("Failed to set listen.so_mark", "error", err) + r.Failed() } } } diff --git a/udp/udp_linux_reload_test.go b/udp/udp_linux_reload_test.go new file mode 100644 index 00000000..28caefa2 --- /dev/null +++ b/udp/udp_linux_reload_test.go @@ -0,0 +1,31 @@ +//go:build linux && !android && !e2e_testing + +package udp + +import ( + "net/netip" + "os" + "testing" + + "github.com/slackhq/nebula/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A setting the kernel refuses fails the reload +func TestReloadConfig_RefusedSettingFailsTheReload(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root may set listen.so_mark, run as an unprivileged user") + } + conn, err := NewListener(testLogger(), Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Batch: 1}) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + + c := config.NewC(testLogger()) + require.NoError(t, c.LoadString("listen:\n port: 0\n")) + c.RegisterReloadCallback(conn.ReloadConfig) + + res := c.ReloadConfigString("listen:\n port: 0\n so_mark: 42\n") + require.NoError(t, res.Err) + assert.True(t, res.Failed) +} diff --git a/udp/udp_rio_windows.go b/udp/udp_rio_windows.go index 435d71a6..36ae2040 100644 --- a/udp/udp_rio_windows.go +++ b/udp/udp_rio_windows.go @@ -351,7 +351,7 @@ func (u *RIOConn) Rebind() error { return nil } -func (u *RIOConn) ReloadConfig(*config.C) {} +func (u *RIOConn) ReloadConfig(*config.C, *config.Reload) {} func (u *RIOConn) Close() error { if !u.isOpen.CompareAndSwap(true, false) { diff --git a/udp/udp_tester.go b/udp/udp_tester.go index db90b370..292111ab 100644 --- a/udp/udp_tester.go +++ b/udp/udp_tester.go @@ -197,7 +197,7 @@ func (u *TesterConn) ListenOut(r EncReader, flush func()) error { } } -func (u *TesterConn) ReloadConfig(*config.C) {} +func (u *TesterConn) ReloadConfig(*config.C, *config.Reload) {} func NewUDPStatsEmitter(_ []Conn) func() { // No UDP stats for non-linux From 316bf877808448ee2c95d1b8c6455fe5fcfdac35 Mon Sep 17 00:00:00 2001 From: Nate Brown Date: Thu, 15 Oct 2026 23:12:12 -0400 Subject: [PATCH 29/31] Don't restart sshd when only sshd.sandbox_dir changes (#1948) --- ssh.go | 7 ++++++- sshd_reload_test.go | 48 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/ssh.go b/ssh.go index e59cc858..716f105d 100644 --- a/ssh.go +++ b/ssh.go @@ -14,6 +14,7 @@ import ( "path/filepath" "runtime" "runtime/pprof" + "slices" "sort" "strconv" "strings" @@ -65,7 +66,7 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) { // Restarting the server drops every open session, the one that asked for this reload included. Only restart // when it isn't running or its config changed. The host key file can change under the same config, new // handshakes use the new key without a restart - if ssh.Running() && !c.HasChanged("sshd") { + if ssh.Running() && !slices.ContainsFunc(sshdServerKeys, c.HasChanged) { key, err := loadSSHHostKey(c) if err == nil { err = ssh.SetHostKey(key) @@ -92,6 +93,10 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) { }) } +// sshdServerKeys are what configSSH reads. sshd.sandbox_dir isn't one, it's read once at startup and restarting the +// server doesn't apply it +var sshdServerKeys = []string{"sshd.enabled", "sshd.listen", "sshd.host_key", "sshd.trusted_cas", "sshd.authorized_users"} + // loadSSHHostKey returns sshd.host_key, read from its file when it names one. func loadSSHHostKey(c *config.C) ([]byte, error) { hostKeyPathOrKey := c.GetString("sshd.host_key", "") diff --git a/sshd_reload_test.go b/sshd_reload_test.go index 0e81cece..b7f8a6fa 100644 --- a/sshd_reload_test.go +++ b/sshd_reload_test.go @@ -26,13 +26,15 @@ type sshdReloadHarness struct { server *sshd.SSHServer keyFile string signer ssh.Signer + listen string + users []string extra string } func newSSHDReloadHarness(t *testing.T) *sshdReloadHarness { t.Helper() l := test.NewLogger() - h := &sshdReloadHarness{keyFile: filepath.Join(t.TempDir(), "host_key")} + h := &sshdReloadHarness{keyFile: filepath.Join(t.TempDir(), "host_key"), listen: "127.0.0.1:0", users: []string{"test"}} h.writeHostKey(t, newEd25519Key(t)) _, userKey, err := ed25519.GenerateKey(rand.Reader) @@ -71,8 +73,12 @@ func (h *sshdReloadHarness) writeHostKey(t *testing.T, key crypto.Signer) ssh.Pu } func (h *sshdReloadHarness) config() string { - return fmt.Sprintf("sshd:\n enabled: true\n listen: 127.0.0.1:0\n host_key: %s\n authorized_users:\n - user: test\n keys: [%q]\n%s", - h.keyFile, string(ssh.MarshalAuthorizedKey(h.signer.PublicKey())), h.extra) + users := "" + for _, u := range h.users { + users += fmt.Sprintf(" - user: %s\n keys: [%q]\n", u, string(ssh.MarshalAuthorizedKey(h.signer.PublicKey()))) + } + return fmt.Sprintf("sshd:\n enabled: true\n listen: %s\n host_key: %s\n authorized_users:\n%s%s", + h.listen, h.keyFile, users, h.extra) } // dial connects to wherever the server listens now, a restart on :0 moves it. hostKeyAlgorithms limits which host keys @@ -134,15 +140,41 @@ func TestSSHDReload_UnchangedKeepsSessions(t *testing.T) { assert.False(t, dropped(client, 500*time.Millisecond), "an unchanged sshd was restarted") } +// Every key the server reads restarts it when it changes func TestSSHDReload_ChangedConfigRestarts(t *testing.T) { + cases := map[string]func(t *testing.T, h *sshdReloadHarness){ + "listen": func(t *testing.T, h *sshdReloadHarness) { h.listen = "localhost:0" }, + "authorized_users": func(t *testing.T, h *sshdReloadHarness) { h.users = append(h.users, "other") }, + "trusted_cas": func(t *testing.T, h *sshdReloadHarness) { h.extra = " trusted_cas: []\n" }, + "host_key": func(t *testing.T, h *sshdReloadHarness) { + h.keyFile = filepath.Join(t.TempDir(), "other_host_key") + h.writeHostKey(t, newEd25519Key(t)) + }, + } + for name, change := range cases { + t.Run(name, func(t *testing.T) { + h := newSSHDReloadHarness(t) + client, _ := h.connect(t) + + change(t, h) + require.NoError(t, h.c.ReloadConfigString(h.config()).Err) + assert.True(t, dropped(client, 5*time.Second), "a changed sshd config was not applied") + require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond) + h.connect(t) + }) + } +} + +// sandbox_dir is only read at startup, restarting sshd for it would drop every session and still not apply it +func TestSSHDReload_SandboxDirKeepsSessions(t *testing.T) { h := newSSHDReloadHarness(t) client, _ := h.connect(t) - h.extra = " trusted_cas: []\n" - require.NoError(t, h.c.ReloadConfigString(h.config()).Err) - assert.True(t, dropped(client, 5*time.Second), "a changed sshd config was not applied") - require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond) - h.connect(t) + h.extra = " sandbox_dir: " + t.TempDir() + "\n" + res := h.c.ReloadConfigString(h.config()) + require.NoError(t, res.Err) + assert.Equal(t, []string{"sshd.sandbox_dir"}, res.RestartKeys) + assert.False(t, dropped(client, 500*time.Millisecond), "sshd was restarted") } // The config can stay the same while the host key file it names changes. New connections get the new key, open From a203d044bba01ba2b1dcf2044eb3176d4ca3fb13 Mon Sep 17 00:00:00 2001 From: Nate Brown Date: Fri, 16 Oct 2026 00:22:47 -0400 Subject: [PATCH 30/31] Fail the reload when a config entry is skipped with a warning (#1952) --- hostmap.go | 12 +++++++----- hostmap_test.go | 13 +++++++++++++ lighthouse.go | 9 ++++++--- lighthouse_test.go | 30 +++++++++++++++++++++++++++++- main.go | 2 +- ssh.go | 12 ++++++++++-- sshd_reload_test.go | 26 +++++++++++++++++++++++++- 7 files changed, 91 insertions(+), 13 deletions(-) diff --git a/hostmap.go b/hostmap.go index a3eb0c29..24dc6411 100644 --- a/hostmap.go +++ b/hostmap.go @@ -316,9 +316,9 @@ type cachedPacketMetrics struct { func NewHostMapFromConfig(l *slog.Logger, c *config.C) *HostMap { hm := newHostMap(l) - hm.reload(c, true) - c.RegisterReloadCallback(func(c *config.C, _ *config.Reload) { - hm.reload(c, false) + hm.reload(c, true, nil) + c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { + hm.reload(c, false, r) }) l.Info("Main HostMap created", "preferredRanges", hm.GetPreferredRanges()) @@ -337,7 +337,8 @@ func newHostMap(l *slog.Logger) *HostMap { } } -func (hm *HostMap) reload(c *config.C, initial bool) { +// r is nil on the initial load +func (hm *HostMap) reload(c *config.C, initial bool, r *config.Reload) { if initial || c.HasChanged("preferred_ranges") { var preferredRanges []netip.Prefix rawPreferredRanges := c.GetStringSlice("preferred_ranges", []string{}) @@ -348,8 +349,9 @@ func (hm *HostMap) reload(c *config.C, initial bool) { if err != nil { hm.l.Warn("Failed to parse preferred ranges, ignoring", "error", err, - "range", rawPreferredRanges, + "range", rawPreferredRange, ) + r.Failed() continue } diff --git a/hostmap_test.go b/hostmap_test.go index b522af70..e1984b3e 100644 --- a/hostmap_test.go +++ b/hostmap_test.go @@ -447,3 +447,16 @@ func TestHostInfo_markOutOnly(t *testing.T) { assert.True(t, h.markOut(6), "a relayed send must not consume the epoch edge") assert.False(t, h.markOut(6)) } + +// A preferred range that doesn't parse is skipped and the rest are used, the reload says one was skipped +func TestHostMap_reloadBadPreferredRangeFails(t *testing.T) { + l := test.NewLogger() + c := config.NewC(l) + require.NoError(t, c.LoadString("preferred_ranges: [10.0.0.0/8]\n")) + hm := NewHostMapFromConfig(l, c) + + res := c.ReloadConfigString("preferred_ranges: [10.0.0.0/8, nope]\n") + require.NoError(t, res.Err) + assert.True(t, res.Failed) + assert.Equal(t, []netip.Prefix{netip.MustParsePrefix("10.0.0.0/8")}, hm.GetPreferredRanges()) +} diff --git a/lighthouse.go b/lighthouse.go index 5e610111..49cf8b14 100644 --- a/lighthouse.go +++ b/lighthouse.go @@ -138,14 +138,14 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c h.metrics = newLighthouseMetrics() } - err := h.reload(c, true) + err := h.reload(c, true, nil) if err != nil { return nil, err } c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { r.RestartRequiredIfChanged("lighthouse.am_lighthouse", "listen.port", "handshakes.query_buffer", "stats.lighthouse_metrics") - err := h.reload(c, false) + err := h.reload(c, false, r) switch v := err.(type) { case *util.ContextualError: v.Log(l) @@ -193,7 +193,8 @@ func (lh *LightHouse) GetUpdateInterval() int64 { return lh.interval.Load() } -func (lh *LightHouse) reload(c *config.C, initial bool) error { +// r is nil on the initial load +func (lh *LightHouse) reload(c *config.C, initial bool, r *config.Reload) error { if initial || c.HasChanged("lighthouse.advertise_addrs") { rawAdvAddrs := c.GetStringSlice("lighthouse.advertise_addrs", []string{}) advAddrs := make([]netip.AddrPort, 0) @@ -228,6 +229,7 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error { "addr", rawAddr, "entry", i+1, ) + r.Failed() continue } @@ -386,6 +388,7 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error { "relay", v, "error", err, ) + r.Failed() } else { lh.l.Info("Read relay from config", "relay", v) relaysForMe = append(relaysForMe, configRIP) diff --git a/lighthouse_test.go b/lighthouse_test.go index 4d1c5368..23d0efd5 100644 --- a/lighthouse_test.go +++ b/lighthouse_test.go @@ -296,7 +296,7 @@ func TestLighthouse_reload(t *testing.T) { require.NoError(t, err) c.ReloadConfigString(string(rc)) - err = lh.reload(c, false) + err = lh.reload(c, false, nil) require.NoError(t, err) } @@ -1014,3 +1014,31 @@ func TestLighthouse_reloadRestartRequired(t *testing.T) { require.NoError(t, res.Err) assert.True(t, res.RestartRequired) } + +// A relay that doesn't parse is skipped and the rest are used, the reload says one was skipped +func TestLighthouse_reloadBadRelayFails(t *testing.T) { + l := test.NewLogger() + c := config.NewC(l) + require.NoError(t, c.LoadString("listen:\n port: 4242\n")) + lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil) + require.NoError(t, err) + + res := c.ReloadConfigString("listen:\n port: 4242\nrelay:\n relays: [10.128.0.2, nope]\n") + require.NoError(t, res.Err) + assert.True(t, res.Failed) + assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.2")}, lh.GetRelaysForMe()) +} + +// An advertise_addrs entry inside our own network is skipped and the rest are used, the reload says one was skipped +func TestLighthouse_reloadAdvertiseAddrInNetworkFails(t *testing.T) { + l := test.NewLogger() + c := config.NewC(l) + require.NoError(t, c.LoadString("listen:\n port: 4242\n")) + lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil) + require.NoError(t, err) + + res := c.ReloadConfigString("listen:\n port: 4242\nlighthouse:\n advertise_addrs: [\"1.2.3.4:4242\", \"10.128.0.5:4242\"]\n") + require.NoError(t, res.Err) + assert.True(t, res.Failed) + assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("1.2.3.4:4242")}, lh.GetAdvertiseAddrs()) +} diff --git a/main.go b/main.go index e7adfd58..105bc6d9 100644 --- a/main.go +++ b/main.go @@ -78,7 +78,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev wireSSHReload(l, ssh, c) var sshStart func() if c.GetBool("sshd.enabled", false) { - sshStart, err = configSSH(l, ssh, c) + sshStart, err = configSSH(l, ssh, c, nil) if err != nil { l.Warn("Failed to configure sshd, ssh debugging will not be available", "error", err) sshStart = nil diff --git a/ssh.go b/ssh.go index 716f105d..ab4fd022 100644 --- a/ssh.go +++ b/ssh.go @@ -78,7 +78,7 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) { } return } - sshRun, err := configSSH(l, ssh, c) + sshRun, err := configSSH(l, ssh, c, r) if err != nil { l.Error("Failed to reconfigure the sshd", "error", err) ssh.Stop() @@ -117,7 +117,8 @@ func loadSSHHostKey(c *config.C) ([]byte, error) { // updates the passed-in SSHServer. On success, it returns a function // that callers may invoke to run the configured ssh server. On // failure, it returns nil, error. -func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error) { +// r is nil at startup, entries that are skipped with a warning fail it on a reload +func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C, r *config.Reload) (func(), error) { listen := c.GetString("sshd.listen", "") if listen == "" { return nil, fmt.Errorf("sshd.listen must be provided") @@ -150,6 +151,7 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error) err := ssh.AddTrustedCA(caAuthorizedKey) if err != nil { l.Warn("SSH CA had an error, ignoring", "error", err, "sshCA", caAuthorizedKey) + r.Failed() continue } } @@ -161,12 +163,14 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error) kDef, ok := rk.(map[string]any) if !ok { l.Warn("Authorized user had an error, ignoring", "sshKeyConfig", rk) + r.Failed() continue } user, ok := kDef["user"].(string) if !ok { l.Warn("Authorized user is missing the user field", "sshKeyConfig", rk) + r.Failed() continue } @@ -180,6 +184,7 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error) "sshKeyConfig", rk, "sshKey", v, ) + r.Failed() continue } @@ -191,6 +196,7 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error) "sshKeyConfig", rk, "sshKey", subK, ) + r.Failed() continue } @@ -200,12 +206,14 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error) "error", err, "sshKeyConfig", sk, ) + r.Failed() continue } } default: l.Warn("Authorized user is missing the keys field or was not understood", "sshKeyConfig", rk) + r.Failed() } } } else { diff --git a/sshd_reload_test.go b/sshd_reload_test.go index b7f8a6fa..4d650dc0 100644 --- a/sshd_reload_test.go +++ b/sshd_reload_test.go @@ -47,7 +47,7 @@ func newSSHDReloadHarness(t *testing.T) *sshdReloadHarness { h.server, err = sshd.NewSSHServer(t.Context(), l) require.NoError(t, err) wireSSHReload(l, h.server, h.c) - run, err := configSSH(l, h.server, h.c) + run, err := configSSH(l, h.server, h.c, nil) require.NoError(t, err) go run() t.Cleanup(h.server.Stop) @@ -226,3 +226,27 @@ func TestSSHDReload_BadHostKeyFileStops(t *testing.T) { assert.True(t, h.c.ReloadConfigString(h.config()).Failed) require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond) } + +// A trusted CA or authorized user that is skipped with a warning fails the reload, the valid ones are still used +func TestSSHDReload_SkippedEntriesFail(t *testing.T) { + cases := map[string]string{ + "bad trusted ca": " trusted_cas: [nope]\n", + "user entry not a map": " - nope\n", + "user missing": " - keys: [nope]\n", + "bad key": " - user: other\n keys: nope\n", + "key in a list not a string": " - user: other\n keys: [5]\n", + "bad key in a list": " - user: other\n keys: [nope]\n", + "keys missing": " - user: other\n", + } + for name, extra := range cases { + t.Run(name, func(t *testing.T) { + h := newSSHDReloadHarness(t) + h.extra = extra + res := h.c.ReloadConfigString(h.config()) + require.NoError(t, res.Err) + assert.True(t, res.Failed) + require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond) + h.connect(t) + }) + } +} From 915495d5f4b657834a5cd480651976c77d3d6b48 Mon Sep 17 00:00:00 2001 From: Nate Brown Date: Fri, 16 Oct 2026 00:34:09 -0400 Subject: [PATCH 31/31] Fail the reload on a startup only value that would fail a restart, and stop asking for tun.dev restarts that change nothing (#1953) --- examples/config.yml | 2 +- overlay/tun_bsd_dev.go | 45 +++++++++++++++++ overlay/tun_bsd_reload_test.go | 80 +++++++++++++++++++++++++++++++ overlay/tun_darwin.go | 41 +++++++++++----- overlay/tun_darwin_route_test.go | 13 +++++ overlay/tun_netbsd.go | 20 ++------ overlay/tun_openbsd.go | 20 ++------ overlay/tun_windows.go | 10 +++- overlay/tun_windows_route_test.go | 21 ++++++++ pki.go | 28 ++++++++--- pki_test.go | 39 +++++++++++++++ 11 files changed, 269 insertions(+), 50 deletions(-) create mode 100644 overlay/tun_bsd_dev.go create mode 100644 overlay/tun_bsd_reload_test.go diff --git a/examples/config.yml b/examples/config.yml index ef29812e..e04e9dbe 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -261,7 +261,7 @@ tun: disabled: false # Name of the device. If not set, a default will be chosen by the OS. # For macOS: if set, must be in the form `utun[0-9]+`. - # For NetBSD: Required to be set, must be in the form `tun[0-9]+` + # For OpenBSD and NetBSD: if set, must be in the form `tun[0-9]+`, otherwise the first free one is used. # Not reloadable. dev: nebula1 # Toggles forwarding of local broadcast packets, the address of which depends on the ip/mask encoded in pki.cert diff --git a/overlay/tun_bsd_dev.go b/overlay/tun_bsd_dev.go new file mode 100644 index 00000000..6a21eea9 --- /dev/null +++ b/overlay/tun_bsd_dev.go @@ -0,0 +1,45 @@ +//go:build (openbsd || netbsd) && !e2e_testing + +package overlay + +import ( + "errors" + "fmt" + "log/slog" + "os" + "regexp" + "runtime" + + "golang.org/x/sys/unix" +) + +var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`) + +// tunDevNeedsRestart is true when tun.dev asks for a particular tun that isn't the running one. Anything else takes the +// first free one, and the one we have will do +func tunDevNeedsRestart(name, running string) bool { + return deviceNameRE.MatchString(name) && name != running +} + +// openTun opens /dev/name, or the first /dev/tunN nothing else has open when name is empty or not tunN +func openTun(l *slog.Logger, name string) (int, string, error) { + if name != "" && !deviceNameRE.MatchString(name) { + // The name doesn't mean anything here, so like darwin a bad one isn't worth refusing to start over + l.Warn("interface name must be tun[0-9]+ on "+runtime.GOOS+", ignoring", "name", name) + name = "" + } + if name != "" { + fd, err := unix.Open("/dev/"+name, os.O_RDWR, 0) + return fd, name, err + } + for i := 0; ; i++ { + name = fmt.Sprintf("tun%d", i) + fd, err := unix.Open("/dev/"+name, os.O_RDWR, 0) + if err == nil { + return fd, name, nil + } + if !errors.Is(err, unix.EBUSY) { + return -1, "", fmt.Errorf("no free tun device, %s: %w", name, err) + } + } +} diff --git a/overlay/tun_bsd_reload_test.go b/overlay/tun_bsd_reload_test.go new file mode 100644 index 00000000..a6ce15b8 --- /dev/null +++ b/overlay/tun_bsd_reload_test.go @@ -0,0 +1,80 @@ +//go:build (openbsd || netbsd) && !e2e_testing + +package overlay + +import ( + "net/netip" + "testing" + + "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A tun.dev that isn't tunN is ignored like darwin does, and the first free tun is used. Only naming a different tun +// asks for a restart +func TestTunReloadDevRestart(t *testing.T) { + newWithDev := func(dev string) (*config.C, *tun, error) { + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString("tun:\n dev: "+dev+"\n")) + tn, err := newTun(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, false) + if err == nil { + t.Cleanup(func() { _ = tn.Close() }) + } + return c, tn, err + } + if _, _, err := newWithDev(""); err != nil { + t.Skipf("can't create a tun device here: %v", err) + } + c, tn, err := newWithDev("nope") + require.NoError(t, err, "a bad name should fall back to a free tun") + require.Regexp(t, `^tun[0-9]+$`, tn.Device) + + for _, dev := range []string{"nope", "", tn.Device, "nope"} { + res := c.ReloadConfigString("tun:\n dev: " + dev + "\n") + require.NoError(t, res.Err) + assert.False(t, res.Failed, dev) + assert.False(t, res.RestartRequired, dev) + } + + other := "tun0" + if tn.Device == other { + other = "tun1" + } + res := c.ReloadConfigString("tun:\n dev: " + other + "\n") + require.NoError(t, res.Err) + assert.Equal(t, []string{"tun.dev"}, res.RestartKeys) +} + +// An empty tun.dev takes the first tun nothing else has open. Reloading it empty, or naming the one we have, needs +// no restart +func TestTunEmptyDevPicksAFreeOne(t *testing.T) { + newEmpty := func() (*config.C, *tun, error) { + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString("tun: {}\n")) + tn, err := newTun(c, test.NewLogger(), []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, false) + if err == nil { + t.Cleanup(func() { _ = tn.Close() }) + } + return c, tn, err + } + c, first, err := newEmpty() + if err != nil { + t.Skipf("can't create a tun device here: %v", err) + } + _, second, err := newEmpty() + require.NoError(t, err, "the second tun should skip the one the first has open") + assert.NotEqual(t, first.Device, second.Device) + + for _, cfg := range []string{ + "tun: {}\npunchy:\n punch: true\n", + "tun:\n dev: " + first.Device + "\n", + "tun: {}\n", + } { + res := c.ReloadConfigString(cfg) + require.NoError(t, res.Err) + assert.False(t, res.Failed) + assert.False(t, res.RestartRequired, cfg) + } +} diff --git a/overlay/tun_darwin.go b/overlay/tun_darwin.go index dcde7fae..5a62f061 100644 --- a/overlay/tun_darwin.go +++ b/overlay/tun_darwin.go @@ -80,17 +80,31 @@ type ifreqAlias6 struct { Lifetime addrLifetime } +// utunIndex is the utun tun.dev asks for, or -1 for whichever is free. ok is false for a name that isn't utunN, which +// also gets whichever is free +func utunIndex(name string) (index int, ok bool) { + if name == "" || name == "utun" { + return -1, true + } + if _, err := fmt.Sscanf(name, "utun%d", &index); err != nil || index < 0 { + return -1, false + } + return index, true +} + +// tunDevNeedsRestart is true when tun.dev asks for a particular utun that isn't the running one. Anything else takes +// whichever is free, and the one we have will do +func tunDevNeedsRestart(name, running string) bool { + index, _ := utunIndex(name) + return index >= 0 && fmt.Sprintf("utun%d", index) != running +} + func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { - name := c.GetString("tun.dev", "") - ifIndex := -1 - if name != "" && name != "utun" { - _, err := fmt.Sscanf(name, "utun%d", &ifIndex) - if err != nil || ifIndex < 0 { - // NOTE: we don't make this error so we don't break existing - // configs that set a name before it was used. - l.Warn("interface name must be utun[0-9]+ on Darwin, ignoring") - ifIndex = -1 - } + ifIndex, ok := utunIndex(c.GetString("tun.dev", "")) + if !ok { + // NOTE: we don't make this error so we don't break existing + // configs that set a name before it was used. + l.Warn("interface name must be utun[0-9]+ on Darwin, ignoring") } fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL) @@ -114,7 +128,7 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t return nil, fmt.Errorf("SYS_CONNECT: %v", err) } - name, err = unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME) + name, err := unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME) if err != nil { return nil, fmt.Errorf("failed to retrieve tun name: %w", err) } @@ -138,7 +152,10 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t } c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { - r.RestartRequiredIfChanged("tun.dev", "tun.mtu") + r.RestartRequiredIfChanged("tun.mtu") + if tunDevNeedsRestart(c.GetString("tun.dev", ""), t.Device) { + r.RestartRequiredIfChanged("tun.dev") + } err := t.reload(c, false, r) if err != nil { util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) diff --git a/overlay/tun_darwin_route_test.go b/overlay/tun_darwin_route_test.go index a566dd38..5eda4813 100644 --- a/overlay/tun_darwin_route_test.go +++ b/overlay/tun_darwin_route_test.go @@ -103,3 +103,16 @@ func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) { require.NoError(t, err) assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))) } + +// Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free +func TestTunDevNeedsRestart(t *testing.T) { + for name, want := range map[string]bool{ + "": false, + "utun": false, + "nope": false, + "utun7": false, + "utun3": true, + } { + assert.Equal(t, want, tunDevNeedsRestart(name, "utun7"), name) + } +} diff --git a/overlay/tun_netbsd.go b/overlay/tun_netbsd.go index 056e5f98..720bfe39 100644 --- a/overlay/tun_netbsd.go +++ b/overlay/tun_netbsd.go @@ -8,7 +8,6 @@ import ( "log/slog" "net/netip" "os" - "regexp" "sync/atomic" "syscall" "unsafe" @@ -67,24 +66,12 @@ type tun struct { fd int } -var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`) - func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) { return nil, fmt.Errorf("newTunFromFd not supported in NetBSD") } func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { - // Try to open tun device - var err error - deviceName := c.GetString("tun.dev", "") - if deviceName == "" { - return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified") - } - if !deviceNameRE.MatchString(deviceName) { - return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified") - } - - fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0) + fd, deviceName, err := openTun(l, c.GetString("tun.dev", "")) if err != nil { return nil, err } @@ -109,7 +96,10 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t } c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { - r.RestartRequiredIfChanged("tun.dev", "tun.mtu") + r.RestartRequiredIfChanged("tun.mtu") + if tunDevNeedsRestart(c.GetString("tun.dev", ""), t.Device) { + r.RestartRequiredIfChanged("tun.dev") + } err := t.reload(c, false, r) if err != nil { util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) diff --git a/overlay/tun_openbsd.go b/overlay/tun_openbsd.go index d9aa7a30..9066d7a2 100644 --- a/overlay/tun_openbsd.go +++ b/overlay/tun_openbsd.go @@ -8,7 +8,6 @@ import ( "log/slog" "net/netip" "os" - "regexp" "sync/atomic" "syscall" "unsafe" @@ -58,24 +57,12 @@ type tun struct { fd int } -var deviceNameRE = regexp.MustCompile(`^tun[0-9]+$`) - func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) { return nil, fmt.Errorf("newTunFromFd not supported in openbsd") } func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*tun, error) { - // Try to open tun device - var err error - deviceName := c.GetString("tun.dev", "") - if deviceName == "" { - return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified") - } - if !deviceNameRE.MatchString(deviceName) { - return nil, fmt.Errorf("a device name in the format of /dev/tunN must be specified") - } - - fd, err := unix.Open("/dev/"+deviceName, os.O_RDWR, 0) + fd, deviceName, err := openTun(l, c.GetString("tun.dev", "")) if err != nil { return nil, err } @@ -100,7 +87,10 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t } c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { - r.RestartRequiredIfChanged("tun.dev", "tun.mtu") + r.RestartRequiredIfChanged("tun.mtu") + if tunDevNeedsRestart(c.GetString("tun.dev", ""), t.Device) { + r.RestartRequiredIfChanged("tun.dev") + } err := t.reload(c, false, r) if err != nil { util.LogWithContextIfNeeded("failed to reload tun device", err, t.l) diff --git a/overlay/tun_windows.go b/overlay/tun_windows.go index fcb6ac93..cca4eeed 100644 --- a/overlay/tun_windows.go +++ b/overlay/tun_windows.go @@ -109,7 +109,15 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*w disableDNSRegistration(l, *guid) c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { - r.RestartRequiredIfChanged("tun.dev", "tun.mtu", "tun.network_category") + r.RestartRequiredIfChanged("tun.dev", "tun.mtu") + // Compare with the running category, not the last config. A restart with one we can't parse would fail, so say + // that now rather than ask for one, and going back to the running one after a bad reload needs no restart + if cat, setCat, err := parseNetworkCategory(c.GetString("tun.network_category", "private")); err != nil { + t.l.Error("Invalid tun.network_category, a restart with it would fail", "error", err) + r.Failed() + } else if cat != t.networkCategory || setCat != t.setCategory { + r.RestartRequiredIfChanged("tun.network_category") + } r.RestartRequiredIfChanged(interfaceBypassKeys...) err := t.reload(c, false, r) if err != nil { diff --git a/overlay/tun_windows_route_test.go b/overlay/tun_windows_route_test.go index 1546dfd6..e9583f28 100644 --- a/overlay/tun_windows_route_test.go +++ b/overlay/tun_windows_route_test.go @@ -181,6 +181,27 @@ func TestWinTun_ReloadRouteAlreadyGone(t *testing.T) { }) } +// A network category we can't parse would fail the restart, so it fails the reload instead of asking for one +func TestWinTun_ReloadInvalidNetworkCategoryFails(t *testing.T) { + rt := newRouteTest(t, unsafeRoutes()) + for range 2 { + res := rt.c.ReloadConfigString(unsafeRoutes() + " network_category: nope\n") + require.NoError(t, res.Err) + assert.True(t, res.Failed, "a bad category that didn't change since the last reload is still bad") + assert.False(t, res.RestartRequired) + } + + // Fixing it back to the running category needs nothing + res := rt.c.ReloadConfigString(unsafeRoutes() + " network_category: private\n") + require.NoError(t, res.Err) + assert.False(t, res.Failed) + assert.False(t, res.RestartRequired) + + res = rt.c.ReloadConfigString(unsafeRoutes() + " network_category: public\n") + require.NoError(t, res.Err) + assert.Equal(t, []string{"tun.network_category"}, res.RestartKeys) +} + // forFamilies runs f with two unsafe_routes CIDRs of each address family, the tun's own network stays IPv4 func forFamilies(t *testing.T, f func(t *testing.T, a, b string)) { t.Run("ipv4", func(t *testing.T) { f(t, "10.251.1.0/24", "10.251.2.0/24") }) diff --git a/pki.go b/pki.go index 0e4deea3..7956ee4a 100644 --- a/pki.go +++ b/pki.go @@ -61,7 +61,18 @@ func NewPKIFromConfig(l *slog.Logger, c *config.C) (*PKI, error) { } c.RegisterReloadCallback(func(c *config.C, r *config.Reload) { - r.RestartRequiredIfChanged("cipher") + // Compare with the running cipher, not the last config. A restart with one we can't run would fail, so say that + // now rather than ask for one, and going back to the running one after a bad reload needs no restart + if cipher := c.GetString("cipher", "aes"); !validCipher(cipher) { + l.Error("Unknown cipher, a restart with it would fail", "cipher", cipher) + r.Failed() + } else if cipher == "chachapoly" && fipsOnly() { + // newCipherSuite refuses it at startup + l.Error("ChaChaPoly is not allowed in FIPS 140-only mode, a restart with it would fail", "cipher", cipher) + r.Failed() + } else if cipher != pki.getCertState().cipher { + r.RestartRequiredIfChanged("cipher") + } rErr := pki.reload(c, false, r) if rErr != nil { util.LogWithContextIfNeeded("Failed to reload PKI from config", rErr, l) @@ -116,16 +127,21 @@ func (p *PKI) reload(c *config.C, initial bool, r *config.Reload) error { return nil } +// fipsOnly is fips140.Enforced, a variable so a test can run the FIPS 140-only reload check without GODEBUG +var fipsOnly = fips140.Enforced + +// validCipher is true for the ciphers nebula can run. Each post-handshake CipherState in noiseutil hardcodes its own +// nonce endianness, so there's nothing to set up for either. +func validCipher(cipher string) bool { + return cipher == "aes" || cipher == "chachapoly" +} + func (p *PKI) reloadCerts(c *config.C, initial bool) *util.ContextualError { var cipher string var currentState *CertState if initial { cipher = c.GetString("cipher", "aes") - switch cipher { - case "aes", "chachapoly": - // Each post-handshake CipherState in noiseutil hardcodes its own - // nonce endianness now, so there's nothing to set up here. - default: + if !validCipher(cipher) { return util.NewContextualError( "unknown cipher", m{"cipher": cipher}, diff --git a/pki_test.go b/pki_test.go index 572d4c21..7f05b103 100644 --- a/pki_test.go +++ b/pki_test.go @@ -1,6 +1,7 @@ package nebula import ( + "crypto/fips140" "fmt" "net/netip" "strings" @@ -128,6 +129,23 @@ func TestPKIReloadRestartRequired(t *testing.T) { assert.True(t, res.RestartRequired) assert.Equal(t, "aes", p.getCertState().cipher) + // A cipher we can't run would fail the restart, so it fails the reload instead of asking for one + c4 := config.NewC(test.NewLogger()) + require.NoError(t, c4.LoadString(pkiConfig("aes", "10.1.0.2/24"))) + _, err = NewPKIFromConfig(test.NewLogger(), c4) + require.NoError(t, err) + for range 2 { + res = c4.ReloadConfigString(pkiConfig("nope", "10.1.0.2/24")) + require.NoError(t, res.Err) + assert.True(t, res.Failed, "a bad cipher that didn't change since the last reload is still bad") + assert.False(t, res.RestartRequired) + } + // Fixing it back to the running cipher needs nothing + res = c4.ReloadConfigString(pkiConfig("aes", "10.1.0.2/24")) + require.NoError(t, res.Err) + assert.False(t, res.Failed) + assert.False(t, res.RestartRequired) + // A cert that keeps our networks applies without a restart c3 := config.NewC(test.NewLogger()) require.NoError(t, c3.LoadString(pkiConfig("aes", "10.1.0.2/24"))) @@ -178,3 +196,24 @@ func TestPKIReloadNewVersionWithOtherNetworks(t *testing.T) { assert.Equal(t, []netip.Prefix{netip.MustParsePrefix(tc.network)}, p.getCertState().myVpnNetworks, "accepted either way, as before") } } + +// FIPS 140-only mode refuses chachapoly at startup, so reloading to it fails rather than asks for a restart +func TestPKIReloadChachapolyInFIPSOnlyFails(t *testing.T) { + fipsOnly = func() bool { return true } + t.Cleanup(func() { fipsOnly = fips140.Enforced }) + ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_P256, time.Time{}, time.Time{}, nil, nil, nil) + _, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_P256, ca, caKey, "host", time.Time{}, time.Time{}, []netip.Prefix{netip.MustParsePrefix("10.1.0.2/24")}, nil, nil) + pkiConfig := func(cipher string) string { + return fmt.Sprintf("cipher: %s\npki:\n ca: %q\n cert: %q\n key: %q\n", cipher, caPEM, certPEM, keyPEM) + } + + c := config.NewC(test.NewLogger()) + require.NoError(t, c.LoadString(pkiConfig("aes"))) + _, err := NewPKIFromConfig(test.NewLogger(), c) + require.NoError(t, err) + + res := c.ReloadConfigString(pkiConfig("chachapoly")) + require.NoError(t, res.Err) + assert.True(t, res.Failed) + assert.False(t, res.RestartRequired) +}