Compare commits

..
Author SHA1 Message Date
JackDoanandClaude Fable 5 bc04261d0b linux: let the kernel resolve the tun.dev %d template atomically
Resolving the template in userspace (LinkList then TUNSETIFF) races with
concurrent instances: both can pick nebula0, and with multiqueue the loser
silently attaches as a second queue to the winner's device instead of
failing. TUNSETIFF natively substitutes a single %d via dev_alloc_name,
atomically and always allocating a fresh device, so keep the fail-fast
validation but pass the template through and read the resolved name back.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Fg7oD3uZ4jZjjQ4gRsPAwB
2026-07-09 18:37:30 -05:00
JackDoan 57e1a9b6af linux: allow %d anywhere in the tun.dev template 2026-07-08 13:43:43 -05:00
78 changed files with 1009 additions and 4903 deletions

No files matched your search

+4 -14
View File
@@ -25,9 +25,9 @@ inputs:
required: false required: false
default: "code-signer" default: "code-signer"
key-prefix: key-prefix:
description: "S3 key prefix to write under; defaults to code-signing/<owner>/<repo> of the calling repo" description: "S3 key prefix the caller is authorized to write under"
required: false required: false
default: "" default: "code-signing/slackhq/nebula"
runs: runs:
using: composite using: composite
@@ -43,15 +43,8 @@ runs:
with: with:
role-to-assume: ${{ inputs.role }} role-to-assume: ${{ inputs.role }}
aws-region: ${{ inputs.region }} aws-region: ${{ inputs.region }}
# An STS secret key with special characters does not survive the # Default is 12 retries to ride out IAM trust-policy propagation; once
# pwsh -> make -> MSYS sh -> aws.exe chain, and SigV4 then signs with a # the role is stable we want a real misconfiguration to fail fast.
# key that no longer matches, so the first S3 upload fails with
# SignatureDoesNotMatch. Retries the assume until it comes back clean.
# Same fix as DefinedNet/dnclient#867.
special-characters-workaround: true
# Overridden by the workaround above and kept for whenever that goes:
# the default 12 rides out IAM trust-policy propagation, and once the
# role is stable a real misconfiguration should fail fast.
retry-max-attempts: 5 retry-max-attempts: 5
- name: Sign .exe files - name: Sign .exe files
@@ -64,9 +57,6 @@ runs:
KEY_PREFIX: ${{ inputs.key-prefix }} KEY_PREFIX: ${{ inputs.key-prefix }}
run: | run: |
set -eu set -eu
# Default the prefix to this repo so the S3 key attributes the sign correctly.
# nebula-nightly runs this same action but writes under its own repo's prefix.
KEY_PREFIX="${KEY_PREFIX:-code-signing/$GITHUB_REPOSITORY}"
RUN="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" RUN="${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
find "$SIGN_PATH" -name '*.exe' -print | while read -r path find "$SIGN_PATH" -name '*.exe' -print | while read -r path
+17 -88
View File
@@ -12,9 +12,9 @@ jobs:
steps: steps:
- uses: actions/checkout@v7 - uses: actions/checkout@v7
- uses: actions/setup-go@v7 - uses: actions/setup-go@v6
with: with:
go-version: '1.26' go-version: '1.25'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -38,9 +38,9 @@ jobs:
steps: steps:
- uses: actions/checkout@v7 - uses: actions/checkout@v7
- uses: actions/setup-go@v7 - uses: actions/setup-go@v6
with: with:
go-version: '1.26' go-version: '1.25'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -73,81 +73,27 @@ jobs:
build-darwin: build-darwin:
name: Build Universal Darwin name: Build Universal Darwin
env: env:
HAS_SIGNING_CREDS: ${{ secrets.APPLE_SIGNING_ROLE_ARN != '' }} HAS_SIGNING_CREDS: ${{ secrets.AC_USERNAME != '' }}
runs-on: macos-latest runs-on: macos-latest
permissions:
id-token: write
contents: read
steps: steps:
- uses: actions/checkout@v7 - uses: actions/checkout@v7
- uses: actions/setup-go@v7 - uses: actions/setup-go@v6
with: with:
go-version: '1.26' go-version: '1.25'
check-latest: true check-latest: true
# GitHub holds ARNs, not credentials, and ARNs outlive a rotation
- name: Configure AWS credentials
if: env.HAS_SIGNING_CREDS == 'true'
uses: aws-actions/configure-aws-credentials@v6
with:
role-to-assume: ${{ secrets.APPLE_SIGNING_ROLE_ARN }}
aws-region: us-east-2
# parse-json-secrets unpacks into SIGNING_* and ASC_*, masked on the way in
- name: Fetch signing credentials
if: env.HAS_SIGNING_CREDS == 'true'
uses: aws-actions/aws-secretsmanager-get-secrets@v3
with:
parse-json-secrets: true
secret-ids: |
SIGNING,${{ secrets.APPLE_SIGNING_DEVELOPER_ID_ARN }}
ASC,${{ secrets.APPLE_NOTARY_KEY_ARN }}
- name: Import certificates - name: Import certificates
if: env.HAS_SIGNING_CREDS == 'true' if: env.HAS_SIGNING_CREDS == 'true'
uses: Apple-Actions/import-codesign-certs@v7 uses: Apple-Actions/import-codesign-certs@v7
with: with:
p12-file-base64: ${{ env.SIGNING_P12_BASE64 }} p12-file-base64: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_P12_BASE64 }}
p12-password: ${{ env.SIGNING_PASSWORD }} p12-password: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_PASSWORD }}
# The action imports but does not check the chain validates, which is how a p12
# missing its intermediate reaches a failing codesign
- name: Check the identity is usable
if: env.HAS_SIGNING_CREDS == 'true'
run: |
: "${SIGNING_IDENTITY_SHA1:?empty, so the secret has no identity_sha1}"
identities=$(security find-identity -v -p codesigning signing_temp.keychain)
case "$identities" in
*"$SIGNING_IDENTITY_SHA1"*) ;;
*) printf '%s\n' "$identities" >&2; exit 1 ;;
esac
# notarytool wants the key as a file
- name: Write the App Store Connect key
if: env.HAS_SIGNING_CREDS == 'true'
run: |
mkdir -p ~/private_keys
chmod 700 ~/private_keys
key_path="$HOME/private_keys/AuthKey_${ASC_KEY_ID}.p8"
(umask 077; printf '%s\n' "$ASC_PRIVATE_KEY" > "$key_path")
echo "ASC_P8=$key_path" >> "$GITHUB_ENV"
- name: Drop the credentials from the environment
if: env.HAS_SIGNING_CREDS == 'true'
run: |
# The action's own inventory, so a new field in a secret is covered
python3 -c '
import json, os
raw = os.environ.get("SECRETS_LIST_CLEAN_UP")
if raw is None and os.environ.get("SIGNING_P12_BASE64"):
raise SystemExit("SECRETS_LIST_CLEAN_UP is gone, fetched secrets are not being scrubbed")
keep = {"SIGNING_IDENTITY_SHA1", "ASC_KEY_ID", "ASC_ISSUER_ID"}
names = [n for n in json.loads(raw or "[]") if n not in keep]
print("\n".join(f"{n}=" for n in dict.fromkeys(names)))
' >> "$GITHUB_ENV"
- name: Build, sign, and notarize - name: Build, sign, and notarize
env:
AC_USERNAME: ${{ secrets.AC_USERNAME }}
AC_PASSWORD: ${{ secrets.AC_PASSWORD }}
run: | run: |
rm -rf release rm -rf release
mkdir release mkdir release
@@ -156,34 +102,17 @@ jobs:
lipo -create -output ./release/nebula ./build/darwin-amd64/nebula ./build/darwin-arm64/nebula lipo -create -output ./release/nebula ./build/darwin-amd64/nebula ./build/darwin-arm64/nebula
lipo -create -output ./release/nebula-cert ./build/darwin-amd64/nebula-cert ./build/darwin-arm64/nebula-cert lipo -create -output ./release/nebula-cert ./build/darwin-amd64/nebula-cert ./build/darwin-arm64/nebula-cert
# Unset in a fork, which has no credentials to sign with if [ -n "$AC_USERNAME" ]; then
if [ -n "$SIGNING_IDENTITY_SHA1" ]; then codesign -s "10BC1FDDEB6CE753550156C0669109FAC49E4D1E" -f -v --timestamp --options=runtime -i "net.defined.nebula" ./release/nebula
codesign -s "$SIGNING_IDENTITY_SHA1" -f -v --timestamp --options=runtime -i "net.defined.nebula" ./release/nebula codesign -s "10BC1FDDEB6CE753550156C0669109FAC49E4D1E" -f -v --timestamp --options=runtime -i "net.defined.nebula-cert" ./release/nebula-cert
codesign -s "$SIGNING_IDENTITY_SHA1" -f -v --timestamp --options=runtime -i "net.defined.nebula-cert" ./release/nebula-cert
fi fi
zip -j release/nebula-darwin.zip release/nebula-cert release/nebula zip -j release/nebula-darwin.zip release/nebula-cert release/nebula
if [ -n "$ASC_P8" ]; then if [ -n "$AC_USERNAME" ]; then
xcrun notarytool submit ./release/nebula-darwin.zip --key "$ASC_P8" --key-id "$ASC_KEY_ID" --issuer "$ASC_ISSUER_ID" --wait xcrun notarytool submit ./release/nebula-darwin.zip --team-id "576H3XS7FP" --apple-id "$AC_USERNAME" --password "$AC_PASSWORD" --wait
fi fi
- name: Drop the signing key
if: always() && env.HAS_SIGNING_CREDS == 'true'
run: |
# Locked, not deleted: import-codesign-certs deletes it in its own post
# step and fails the job if it is already gone. Locked is unusable.
security lock-keychain signing_temp.keychain || true
rm -f "$ASC_P8"
# Nothing later in this job needs AWS
python3 -c '
import json, os
names = json.loads(os.environ.get("SECRETS_LIST_CLEAN_UP") or "[]")
names += ["ASC_P8", "SIGNING_IDENTITY_SHA1", "ASC_KEY_ID", "ASC_ISSUER_ID",
"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
print("\n".join(f"{n}=" for n in dict.fromkeys(names)))
' >> "$GITHUB_ENV"
- name: Upload artifacts - name: Upload artifacts
uses: actions/upload-artifact@v7 uses: actions/upload-artifact@v7
with: with:
+6 -6
View File
@@ -32,9 +32,9 @@ jobs:
- uses: actions/checkout@v7 - uses: actions/checkout@v7
- uses: actions/setup-go@v7 - uses: actions/setup-go@v6
with: with:
go-version: '1.26' go-version: '1.25'
check-latest: true check-latest: true
- name: add hashicorp source - name: add hashicorp source
@@ -64,9 +64,9 @@ jobs:
- uses: actions/checkout@v7 - uses: actions/checkout@v7
- uses: actions/setup-go@v7 - uses: actions/setup-go@v6
with: with:
go-version: '1.26' go-version: '1.25'
check-latest: true check-latest: true
- name: add hashicorp source - name: add hashicorp source
@@ -90,9 +90,9 @@ jobs:
- uses: actions/checkout@v7 - uses: actions/checkout@v7
- uses: actions/setup-go@v7 - uses: actions/setup-go@v6
with: with:
go-version: '1.26' go-version: '1.25'
check-latest: true check-latest: true
# WSL2 + Ubuntu so the smoke can run a real linux peer with its own # WSL2 + Ubuntu so the smoke can run a real linux peer with its own
+2 -23
View File
@@ -20,9 +20,9 @@ jobs:
- uses: actions/checkout@v7 - uses: actions/checkout@v7
- uses: actions/setup-go@v7 - uses: actions/setup-go@v6
with: with:
go-version: '1.26' go-version: '1.25'
check-latest: true check-latest: true
- name: build - name: build
@@ -61,24 +61,3 @@ jobs:
run: NAME="smoke-p256" ./smoke.sh run: NAME="smoke-p256" ./smoke.sh
timeout-minutes: 10 timeout-minutes: 10
smoke-self:
name: Run self traffic smoke test on macOS
runs-on: macos-latest
steps:
- uses: actions/checkout@v7
- uses: actions/setup-go@v7
with:
go-version: '1.26'
check-latest: true
- name: build
run: make bin
- name: run smoke-self
working-directory: ./.github/workflows/smoke
run: ./smoke-self.sh
timeout-minutes: 10
-130
View File
@@ -1,130 +0,0 @@
#!/bin/bash
# A host must be able to reach its own overlay address. Where the kernel sends
# that traffic through the tun rather than over loopback, nebula sees it and
# hands it straight back (immediatelyForwardToSelf), and whether the kernel
# accepts what comes back is only answerable against a real kernel. Runs one
# nebula on this machine as root and aims every probe at its own address.
set -e -x
set -o pipefail
V4=192.0.2.1
V6=2001:db8::1
case "$(uname -s)" in
Darwin) TUN_DEV=utun ;;
*) TUN_DEV=tun0 ;;
esac
ROOT="$(cd ../../.. && pwd)"
rm -rf build/self
mkdir -p build/self
cd build/self
cleanup() {
echo
echo " *** cleanup"
echo
set +e
if [ -n "$NEBULA_PID" ]
then
sudo kill "$NEBULA_PID"
fi
{ kill $(jobs -p); wait; } 2>/dev/null
sed 's/^/ [self] /' nebula.log
}
trap cleanup EXIT
# perl is on every platform this runs on; timeout(1) is not.
alarm() {
perl -e 'alarm shift; exec @ARGV' "$@"
}
RESULTS=""
FAILED=""
probe() {
local name="$1"
shift
if "$@"
then
RESULTS="$RESULTS $name=ok"
else
RESULTS="$RESULTS $name=FAIL"
FAILED="$FAILED $name"
fi
}
# Send one datagram, then wait for the listener to have written it out.
udp_probe() {
echo self | alarm 5 nc -u -w1 "$1" 3000 || true
set +x
for _ in $(seq 1 20)
do
if grep -q self "$2"
then
set -x
return 0
fi
sleep 0.25
done
set -x
return 1
}
"$ROOT/nebula-cert" ca -name "Smoke Test"
"$ROOT/nebula-cert" sign -name self -networks "$V4/24,$V6/64"
HOST=self AM_LIGHTHOUSE=true TUN_DEV="$TUN_DEV" ../../genconfig.sh >self.yml
"$ROOT/nebula" -config self.yml -test
sudo -v
sudo "$ROOT/nebula" -config self.yml >nebula.log 2>&1 &
NEBULA_PID=$!
for _ in $(seq 1 40)
do
ifconfig | grep "inet6 $V6 " >/dev/null && break
sleep 0.25
done
ifconfig | grep "inet $V4 "
ifconfig | grep "inet6 $V6 "
nc -l "$V4" 2000 >/dev/null &
nc -l "$V6" 2000 >/dev/null &
nc -u -l "$V4" 3000 >udp4.txt &
nc -u -l "$V6" 3000 >udp6.txt &
sleep 1
set +x
echo
echo " *** Testing self traffic from $V4"
echo
set -x
probe icmp4 alarm 5 ping -c1 "$V4"
probe tcp4 alarm 5 nc -z "$V4" 2000
probe udp4 udp_probe "$V4" udp4.txt
set +x
echo
echo " *** Testing self traffic from $V6"
echo
set -x
probe icmp6 alarm 5 ping6 -c1 "$V6"
probe tcp6 alarm 5 nc -z "$V6" 2000
probe udp6 udp_probe "$V6" udp6.txt
set +x
echo
echo " *** self traffic:$RESULTS"
echo
if [ -n "$FAILED" ]
then
echo "self traffic failed:$FAILED" >&2
exit 1
fi
+5 -48
View File
@@ -51,19 +51,15 @@ wsl -d $Distro -- bash -c "rm -rf $WslDir && mkdir -p $WslDir" | Out-Null
$DevName = 'nebula-smoke' $DevName = 'nebula-smoke'
$Ip1 = '192.168.241.1' $Ip1 = '192.168.241.1'
$Ip2 = '192.168.241.2' $Ip2 = '192.168.241.2'
# Dual stack on purpose: a v4-only overlay never exercises the v6 side of tun.mtu.
$Ip6_1 = 'fd42:4242:241::1'
$Ip6_2 = 'fd42:4242:241::2'
$Mtu = 1300
$Port = 4242 $Port = 4242
& $NebulaCert ca -name 'smoke-ca' -out-crt "$WorkDir\ca.crt" -out-key "$WorkDir\ca.key" & $NebulaCert ca -name 'smoke-ca' -out-crt "$WorkDir\ca.crt" -out-key "$WorkDir\ca.key"
if ($LASTEXITCODE -ne 0) { throw "nebula-cert ca failed (exit $LASTEXITCODE)" } if ($LASTEXITCODE -ne 0) { throw "nebula-cert ca failed (exit $LASTEXITCODE)" }
& $NebulaCert sign -name 'lighthouse' -networks "$Ip1/24,$Ip6_1/64" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\lighthouse.crt" -out-key "$WorkDir\lighthouse.key" & $NebulaCert sign -name 'lighthouse' -networks "$Ip1/24" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\lighthouse.crt" -out-key "$WorkDir\lighthouse.key"
if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign lighthouse failed (exit $LASTEXITCODE)" } if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign lighthouse failed (exit $LASTEXITCODE)" }
& $NebulaCert sign -name 'peer' -networks "$Ip2/24,$Ip6_2/64" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\peer.crt" -out-key "$WorkDir\peer.key" & $NebulaCert sign -name 'peer' -networks "$Ip2/24" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\peer.crt" -out-key "$WorkDir\peer.key"
if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign peer failed (exit $LASTEXITCODE)" } if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign peer failed (exit $LASTEXITCODE)" }
# Windows lighthouse config. # Windows lighthouse config.
@@ -86,7 +82,7 @@ tun:
drop_local_broadcast: false drop_local_broadcast: false
drop_multicast: false drop_multicast: false
tx_queue: 500 tx_queue: 500
mtu: $Mtu mtu: 1300
network_category: private network_category: private
logging: logging:
level: info level: info
@@ -130,7 +126,7 @@ tun:
drop_local_broadcast: false drop_local_broadcast: false
drop_multicast: false drop_multicast: false
tx_queue: 500 tx_queue: 500
mtu: $Mtu mtu: 1300
logging: logging:
level: info level: info
format: text format: text
@@ -173,7 +169,7 @@ Write-Host '=== WSL diagnostic ==='
wsl --version 2>&1 | Out-Host wsl --version 2>&1 | Out-Host
wsl --list --verbose 2>&1 | Out-Host wsl --list --verbose 2>&1 | Out-Host
wsl -d $Distro -u root -- uname -a | Out-Host wsl -d $Distro -u root -- uname -a | Out-Host
wsl -d $Distro -u root -- bash -c "modprobe tun 2>&1 || true; mkdir -p /dev/net; [ -c /dev/net/tun ] || mknod /dev/net/tun c 10 200; chmod 600 /dev/net/tun; { echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6; echo 0 > /proc/sys/net/ipv6/conf/default/disable_ipv6; } 2>/dev/null || true; ls -l /dev/net/tun" wsl -d $Distro -u root -- bash -c "modprobe tun 2>&1 || true; mkdir -p /dev/net; [ -c /dev/net/tun ] || mknod /dev/net/tun c 10 200; chmod 600 /dev/net/tun; ls -l /dev/net/tun"
if ($LASTEXITCODE -ne 0) { throw "failed to prepare /dev/net/tun in WSL (TUN support missing?)" } if ($LASTEXITCODE -ne 0) { throw "failed to prepare /dev/net/tun in WSL (TUN support missing?)" }
# Deliberately no New-NetFirewallRule calls here -- nebula's windows_bypass_wdf # Deliberately no New-NetFirewallRule calls here -- nebula's windows_bypass_wdf
@@ -218,16 +214,6 @@ try {
} }
Write-Host "OK: $DevName NetworkCategory=Private" Write-Host "OK: $DevName NetworkCategory=Private"
# v6 silently kept the adapter default of 65535 while v4 was correct.
foreach ($family in @('IPv4', 'IPv6')) {
Wait-Until -TimeoutSec 30 -What "$DevName $family NlMtu=$Mtu" -Predicate {
if ($lhProc.HasExited) { throw "lighthouse exited (code $($lhProc.ExitCode)) before $family mtu was set" }
$rows = @(Get-NetIPInterface -InterfaceAlias $DevName -AddressFamily $family -ErrorAction SilentlyContinue)
$rows.Count -gt 0 -and -not ($rows | Where-Object { $_.NlMtu -ne $Mtu })
}
Write-Host "OK: $DevName $family NlMtu=$Mtu"
}
Wait-Until -TimeoutSec 30 -What "WSL nebula1 with $Ip2" -Predicate { Wait-Until -TimeoutSec 30 -What "WSL nebula1 with $Ip2" -Predicate {
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before tun was ready" } if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before tun was ready" }
$r = wsl -d $Distro -u root -- bash -c "ip -o addr show nebula1 2>/dev/null | grep -q 'inet $Ip2' && echo yes" $r = wsl -d $Distro -u root -- bash -c "ip -o addr show nebula1 2>/dev/null | grep -q 'inet $Ip2' && echo yes"
@@ -235,13 +221,6 @@ try {
} }
Write-Host "OK: WSL nebula1 has $Ip2" Write-Host "OK: WSL nebula1 has $Ip2"
Wait-Until -TimeoutSec 30 -What "WSL nebula1 with $Ip6_2" -Predicate {
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the v6 address was up" }
$r = wsl -d $Distro -u root -- bash -c "ip -o addr show nebula1 2>/dev/null | grep -q 'inet6 $Ip6_2' && echo yes"
("$r").Trim() -eq 'yes'
}
Write-Host "OK: WSL nebula1 has $Ip6_2"
Wait-Until -TimeoutSec 30 -What "ping from WSL peer to windows lighthouse ($Ip1)" -Predicate { Wait-Until -TimeoutSec 30 -What "ping from WSL peer to windows lighthouse ($Ip1)" -Predicate {
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before ping succeeded" } if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before ping succeeded" }
$r = wsl -d $Distro -u root -- bash -c "ping -c1 -W1 $Ip1 >/dev/null 2>&1 && echo OK" $r = wsl -d $Distro -u root -- bash -c "ping -c1 -W1 $Ip1 >/dev/null 2>&1 && echo OK"
@@ -255,28 +234,6 @@ try {
} }
Write-Host "OK: windows lighthouse -> WSL peer" Write-Host "OK: windows lighthouse -> WSL peer"
# Otherwise the v6 networks only prove the interface exists, not that it forwards.
Wait-Until -TimeoutSec 30 -What "v6 ping from WSL peer to windows lighthouse ($Ip6_1)" -Predicate {
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the v6 ping succeeded" }
$r = wsl -d $Distro -u root -- bash -c "ping -6 -c1 -W1 $Ip6_1 >/dev/null 2>&1 && echo OK"
("$r").Trim() -eq 'OK'
}
Write-Host "OK: WSL peer -> windows lighthouse over v6"
# The NlMtu check above proves the value was written, not that windows honors it. A payload this size only
# crosses the tunnel as fragments cut at NlMtu on the way out of the adapter. Left at the adapter default it
# reaches nebula whole, overflows the udp.MTU (9001) write buffer and is dropped, so the ping never answers.
# Anything smaller would ride out as one oversized datagram and survive on IP fragmentation of the underlay.
$BigPayload = 9000
foreach ($target in @(@{ Family = 'v4'; Ip = $Ip2 }, @{ Family = 'v6'; Ip = $Ip6_2 })) {
Wait-Until -TimeoutSec 30 -What "$($target.Family) ping with a $BigPayload byte payload from windows lighthouse to WSL peer ($($target.Ip))" -Predicate {
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the large $($target.Family) ping succeeded" }
$null = & ping.exe -n 1 -w 1000 -l $BigPayload $target.Ip
$LASTEXITCODE -eq 0
}
Write-Host "OK: windows lighthouse -> WSL peer, $BigPayload byte $($target.Family) payload"
}
Write-Host '' Write-Host ''
Write-Host 'All smoke checks passed.' Write-Host 'All smoke checks passed.'
} }
+7 -7
View File
@@ -20,9 +20,9 @@ jobs:
- uses: actions/checkout@v7 - uses: actions/checkout@v7
- uses: actions/setup-go@v7 - uses: actions/setup-go@v6
with: with:
go-version: '1.26' go-version: '1.25'
check-latest: true check-latest: true
- name: Install goimports - name: Install goimports
@@ -42,7 +42,7 @@ jobs:
- name: golangci-lint - name: golangci-lint
uses: golangci/golangci-lint-action@v9 uses: golangci/golangci-lint-action@v9
with: with:
version: v2.12 version: v2.5
test: test:
name: Test ${{ matrix.name }} name: Test ${{ matrix.name }}
@@ -80,9 +80,9 @@ jobs:
- uses: actions/checkout@v7 - uses: actions/checkout@v7
- uses: actions/setup-go@v7 - uses: actions/setup-go@v6
with: with:
go-version: '1.26' go-version: '1.25'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -125,9 +125,9 @@ jobs:
- uses: actions/checkout@v7 - uses: actions/checkout@v7
- uses: actions/setup-go@v7 - uses: actions/setup-go@v6
with: with:
go-version: '1.26' go-version: '1.25'
check-latest: true check-latest: true
- name: Build ${{ matrix.name }} - name: Build ${{ matrix.name }}
+1 -128
View File
@@ -7,130 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased] ## [Unreleased]
## [1.11.2] - 2026-09-21
See the [v1.11.2](https://github.com/slackhq/nebula/milestone/31?closed=1) milestone for a complete list of changes.
### Fixed
- Windows tracks NLMTU per address family and Nebula only ever set it for `AF_INET`, so the IPv6 MTU on every
Windows adapter sat at the adapter default of 65535 regardless of `tun.mtu` and the stack could hand Nebula
packets far larger than configured. An IPv6 enabled overlay with `tun.mtu` under 1280 now refuses to start,
matching the other platforms. (#1871)
- Android 11+ denies `bind()` on netlink route sockets inside the app sandbox, so local address discovery
failed and an Android node advertised no underlay addresses, leaving peers on the same LAN to reach it only
at the address a lighthouse observed. (#1881)
- Recompute the transport checksum on packets a host sends to its own overlay address. On macOS the kernel
leaves the checksum unfinished for hardware offload, which does not survive the trip through userspace, so
TCP and UDP to a host's own IPv6 overlay address were dropped as corrupt. (#1862)
## [1.11.1] - 2026-08-21
See the [v1.11.1](https://github.com/slackhq/nebula/milestone/30?closed=1) milestone for a complete list of changes.
### Changed
- IPv6 packets whose next header is a protocol Nebula does not parse (SCTP, GRE, IP-in-IP, etc.) are now
classified as that protocol with no ports, closing a firewall bypass where a crafted payload could steer
the classifier into reading one as TCP/UDP and matching a TCP/UDP rule. These packets are now matched as
their true protocol, so only a `proto: any` rule allows them. If you carry one of these protocols over the
overlay, confirm a `proto: any` rule covers it before upgrading, it may have been passing only through this
bypass. (#1840)
- Drop the dependency on `github.com/cyberdelia/go-metrics-graphite`, which has been unmaintained for over ten
years, by inlining the small amount of code Nebula used. (#1832)
### Fixed
- The ICMPv6 type was read from the wrong byte when classifying IPv6 packets, so the echo identifier used
for conntrack was never picked up. (#1840)
- Enforce outbound message counter limits so a tunnel is rehandshaked before the counter can wrap, preventing
nonce reuse. This is unreachable in practice, but is enforced as a defense-in-depth measure. (#1841)
- Prevent `nebula-cert ca` from running out of memory on 32bit systems when generating encrypted private keys. (#1834)
- Tolerate `ErrDumpInterrupted` when listing tun addresses on Linux, so a transient interrupted netlink dump
no longer aborts startup. (#1835)
## [1.11.0] - 2026-07-23
See the [v1.11.0](https://github.com/slackhq/nebula/milestone/25?closed=1) milestone for a complete list of changes.
### Breaking
- Logging has switched from logrus to Go's structured `slog`. Log output changes: levels are upper case
(`level=INFO`), trace prints as `level=DEBUG-4`, timestamps are always RFC3339Nano and `logging.timestamp_format`
is ignored, and some messages were reworded. Review any log parsing before upgrading. This is also an API break
for embedders, as constructors now take a `*slog.Logger`. (#1672, #1734, #1621)
- `firewall.inbound_action` and `firewall.outbound_action` (used to set reject vs. drop policy) were each being
applied to the opposite direction, that is now corrected. This only affects how blocked packets are answered, not
which packets the firewall allows or denies. If you set either of these you are getting the behavior of the other
one today and likely want to swap them before upgrading. (#1798)
- On Windows, Nebula now installs WFP PERMIT filters for the nebula adapter and the listener port by default. WFP
sits below Windows Defender Firewall, so any WDF inbound rules you rely on for either will no longer apply. Set
`tun.windows_bypass_wdf` and `listen.windows_bypass_wdf` to false to leave WDF in charge. (#1710)
- On Windows, the nebula device is now set to the `private` network category instead of whatever Windows decided,
which is usually `Public`. This makes the host firewall less restrictive on the overlay. Set
`tun.network_category` to `unset` to keep the old behavior. (#1710)
- Reject packets for non-TCP now use ICMP code 13, communication administratively prohibited, instead of code 3,
port unreachable. Anything keying off the old code needs updating. (#1766, #1768)
- The SSH debug server's profiling commands are now confined to `sshd.sandbox_dir`, which defaults to
`$TMP/nebula-debug`. Relative paths resolve inside it and absolute paths outside it are rejected, so anything
scripting `start-cpu-profile`, `save-heap-profile`, or `save-mutex-profile` with a path elsewhere needs the
directory set. The directory is not created for you. (#1622)
### Added
- Sign the Windows release binaries. (#1718)
- Generate IPv6 reject packets, matching the existing IPv4 behavior. (#1766, #1767, #1768)
- Accept `-` in `nebula-cert` to read from stdin or write to stdout. (#1714)
- Search for both `config.yml` and `config.yaml` in service and command line modes. (#1717)
- Add version labels to the Docker/OCI images. (#1772)
- Rebind the listener and re-query lighthouses on macOS when the underlay network changes, so devices moving
between wifi and wired or between networks recover without waiting for dead tunnel detection. Controlled by
`listen.rebind_on_network_change` (default `true`, not reloadable). (#1816)
### Changed
- Reload the firewall when the unsafe networks in the certificate change. (#1719)
- Reconfigure, start, and stop the stats listener on a config reload instead of requiring a restart. (#1670)
- Update a static host's addresses when they change on reload. (#1713)
- Don't require a port on ICMP firewall rules. (#1609)
- Connection track ICMP traffic. (#1602)
- Return `NODATA` instead of `NXDOMAIN` from the DNS server for a name that exists but has no record of the
requested type, so clients that query `AAAA` first (busybox/Alpine) fall through to `A`. (#1668)
- Record the local host's details in the DNS server. (#1716)
- Install Windows unsafe routes as link routes. (#1709)
- Reduce relay handshake log spam, and only log a handshake send error at error level when the remote list
changes. (#1733, #1765, #1810)
- Start, stop, and reload subsystems (DNS, stats, conntrack, ssh, punchy) cleanly without leaking goroutines. (#1640, #1654, #1661, #1667, #1669, #1708, #1806, #1815)
- `Control` is now safe to stop and wait on from any lifecycle state, and a new `Control.Wait` blocks until nebula
has fully stopped and returns the first fatal reader error. Failed starts release the udp sockets and tun fd
instead of leaking them. (#1794)
- Trigger an immediate lighthouse update when reconnecting to or adding a lighthouse instead of waiting for the next update tick. (#1645)
- Bring the Darwin and OpenBSD tun implementations in line with the other BSDs. (#1703)
- Update to build against go v1.26. (#1818)
- Various dependency updates. (#1586, #1587, #1604, #1617, #1618, #1627, #1628, #1629, #1652, #1664, #1665, #1697, #1721, #1732, #1742, #1743, #1750, #1763, #1771, #1782, #1800, #1807)
### Fixed
- Fix a data race on a host's remote address that could send packets to the wrong address during a roam. (#1773)
- Fix tunnels that could permanently escape connection manager monitoring. (#1752)
- Fix a crash when reloading the SSH server's trusted keys. (#1787)
- Fix hostmap corruption when a host has multiple overlay addresses. Each address now gets its own list instead of
a single shared chain, which also fixes two latent bugs on the add and makePrimary paths. (#1788, #1790)
- Apply `remote_allow_list` IPv4 rules to 4-in-6 mapped addresses. (#1786)
- Don't panic in the DNS server on a short or empty query name. (#1635)
- Advance the replay window on relayed packets so a relay drops replayed frames instead of re-forwarding them. (#1751)
- Fix a race in relay state handling. (#1753)
- Lock replay window updates so concurrent readers can't corrupt it. (#1802)
- Reject malformed handshakes more reliably, including invalid ed25519 key lengths. (#1601, #1756)
- Properly handle `closetunnel` packets. (#1638)
- Fix an IPv6 extension-header length overflow that could make the firewall parse the wrong protocol and ports. (#1789)
- Fix relay re-establishment when a handshake arrives over a relay entry that a one-sided teardown left
`Disestablished`, which silently dropped every send until dead tunnel detection forced a re-handshake. (#1805)
- Don't build new relay state on a tunnel that was just discarded. (#1796)
- Don't delete the wrong pending hostinfo in the handshake manager. (#1811)
- Don't call the packet reader after a UDP error on Darwin. (#1755)
- Open the FreeBSD tun device non blocking. (#1666)
## [1.10.3] - 2026-02-06 ## [1.10.3] - 2026-02-06
### Security ### Security
@@ -912,10 +788,7 @@ created.)
- Initial public release. - Initial public release.
[Unreleased]: https://github.com/slackhq/nebula/compare/v1.11.2...HEAD [Unreleased]: https://github.com/slackhq/nebula/compare/v1.10.3...HEAD
[1.11.2]: https://github.com/slackhq/nebula/releases/tag/v1.11.2
[1.11.1]: https://github.com/slackhq/nebula/releases/tag/v1.11.1
[1.11.0]: https://github.com/slackhq/nebula/releases/tag/v1.11.0
[1.10.3]: https://github.com/slackhq/nebula/releases/tag/v1.10.3 [1.10.3]: https://github.com/slackhq/nebula/releases/tag/v1.10.3
[1.10.2]: https://github.com/slackhq/nebula/releases/tag/v1.10.2 [1.10.2]: https://github.com/slackhq/nebula/releases/tag/v1.10.2
[1.10.1]: https://github.com/slackhq/nebula/releases/tag/v1.10.1 [1.10.1]: https://github.com/slackhq/nebula/releases/tag/v1.10.1
+1 -7
View File
@@ -222,14 +222,11 @@ test-cov-html:
go test -coverprofile=coverage.out go test -coverprofile=coverage.out
go tool cover -html=coverage.out go tool cover -html=coverage.out
# The package builds only compile. The final line links an android binary so a linker-only failure
# cannot pass CI.
build-test-mobile: build-test-mobile:
GOARCH=amd64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=amd64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=arm64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=amd64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=amd64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=arm64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=android go build -o /dev/null ${NEBULA_CMD_PATH}
bench: bench:
go test -bench=. go test -bench=.
@@ -278,13 +275,10 @@ smoke-docker-race: BUILD_ARGS = -race
smoke-docker-race: CGO_ENABLED = 1 smoke-docker-race: CGO_ENABLED = 1
smoke-docker-race: smoke-docker smoke-docker-race: smoke-docker
smoke-self: bin
cd .github/workflows/smoke/ && ./smoke-self.sh
smoke-vagrant/%: bin-docker build/%/nebula smoke-vagrant/%: bin-docker build/%/nebula
cd .github/workflows/smoke/ && ./build.sh $* cd .github/workflows/smoke/ && ./build.sh $*
cd .github/workflows/smoke/ && ./smoke-vagrant.sh $* cd .github/workflows/smoke/ && ./smoke-vagrant.sh $*
.FORCE: .FORCE:
.PHONY: all all-linux all-freebsd all-openbsd all-netbsd all-darwin all-windows all-cross-linux all-cross-linux-arm all-cross-linux-mips all-cross-linux-other all-cross-darwin all-cross-windows bench bench-cpu bench-cpu-long bin build-test-mobile e2e e2ev e2evv e2evvv e2evvvv proto release service smoke-docker smoke-docker-race smoke-self test test-cov-html smoke-vagrant/% .PHONY: all all-linux all-freebsd all-openbsd all-netbsd all-darwin all-windows all-cross-linux all-cross-linux-arm all-cross-linux-mips all-cross-linux-other all-cross-darwin all-cross-windows bench bench-cpu bench-cpu-long bin build-test-mobile e2e e2ev e2evv e2evvv e2evvvv proto release service smoke-docker smoke-docker-race test test-cov-html smoke-vagrant/%
.DEFAULT_GOAL := bin .DEFAULT_GOAL := bin
+2 -17
View File
@@ -8,7 +8,6 @@ import (
"fmt" "fmt"
"io" "io"
"math" "math"
"math/bits"
"net/netip" "net/netip"
"os" "os"
"strings" "strings"
@@ -45,20 +44,6 @@ type caFlags struct {
} }
func newCaFlags() *caFlags { func newCaFlags() *caFlags {
// prevent running out of memory on 32-bit systems by defaulting to
// RFC9106's recommendation for memory-constrained environments
var (
defaultArgonMemory uint
defaultArgonIterations uint
)
if bits.UintSize == 32 {
defaultArgonMemory = 64 * 1024
defaultArgonIterations = 3
} else {
defaultArgonMemory = 2 * 1024 * 1024
defaultArgonIterations = 1
}
cf := caFlags{set: flag.NewFlagSet("ca", flag.ContinueOnError)} cf := caFlags{set: flag.NewFlagSet("ca", flag.ContinueOnError)}
cf.set.Usage = func() {} cf.set.Usage = func() {}
cf.name = cf.set.String("name", "", "Required: name of the certificate authority") cf.name = cf.set.String("name", "", "Required: name of the certificate authority")
@@ -70,9 +55,9 @@ func newCaFlags() *caFlags {
cf.groups = cf.set.String("groups", "", "Optional: comma separated list of groups. This will limit which groups subordinate certs can use") cf.groups = cf.set.String("groups", "", "Optional: comma separated list of groups. This will limit which groups subordinate certs can use")
cf.networks = cf.set.String("networks", "", "Optional: comma separated list of ip address and network in CIDR notation. This will limit which ip addresses and networks subordinate certs can use in networks") cf.networks = cf.set.String("networks", "", "Optional: comma separated list of ip address and network in CIDR notation. This will limit which ip addresses and networks subordinate certs can use in networks")
cf.unsafeNetworks = cf.set.String("unsafe-networks", "", "Optional: comma separated list of ip address and network in CIDR notation. This will limit which ip addresses and networks subordinate certs can use in unsafe networks") cf.unsafeNetworks = cf.set.String("unsafe-networks", "", "Optional: comma separated list of ip address and network in CIDR notation. This will limit which ip addresses and networks subordinate certs can use in unsafe networks")
cf.argonMemory = cf.set.Uint("argon-memory", defaultArgonMemory, "Optional: Argon2 memory parameter (in KiB) used for encrypted private key passphrase") cf.argonMemory = cf.set.Uint("argon-memory", 2*1024*1024, "Optional: Argon2 memory parameter (in KiB) used for encrypted private key passphrase")
cf.argonParallelism = cf.set.Uint("argon-parallelism", 4, "Optional: Argon2 parallelism parameter used for encrypted private key passphrase") cf.argonParallelism = cf.set.Uint("argon-parallelism", 4, "Optional: Argon2 parallelism parameter used for encrypted private key passphrase")
cf.argonIterations = cf.set.Uint("argon-iterations", defaultArgonIterations, "Optional: Argon2 iterations parameter used for encrypted private key passphrase") cf.argonIterations = cf.set.Uint("argon-iterations", 1, "Optional: Argon2 iterations parameter used for encrypted private key passphrase")
cf.encryption = cf.set.Bool("encrypt", false, "Optional: prompt for passphrase and write out-key in an encrypted format") cf.encryption = cf.set.Bool("encrypt", false, "Optional: prompt for passphrase and write out-key in an encrypted format")
cf.curve = cf.set.String("curve", "25519", "EdDSA/ECDSA Curve (25519, P256)") cf.curve = cf.set.String("curve", "25519", "EdDSA/ECDSA Curve (25519, P256)")
cf.p11url = p11Flag(cf.set) cf.p11url = p11Flag(cf.set)
+5 -25
View File
@@ -7,9 +7,7 @@ import (
"bytes" "bytes"
"encoding/pem" "encoding/pem"
"errors" "errors"
"math/bits"
"os" "os"
"strconv"
"strings" "strings"
"testing" "testing"
"time" "time"
@@ -24,18 +22,6 @@ func Test_caSummary(t *testing.T) {
} }
func Test_caHelp(t *testing.T) { func Test_caHelp(t *testing.T) {
var (
defaultArgonMemory string
defaultArgonIterations string
)
if bits.UintSize == 32 {
defaultArgonMemory = strconv.Itoa(64 * 1024)
defaultArgonIterations = strconv.Itoa(3)
} else {
defaultArgonMemory = strconv.Itoa(2 * 1024 * 1024)
defaultArgonIterations = strconv.Itoa(1)
}
ob := &bytes.Buffer{} ob := &bytes.Buffer{}
caHelp(ob) caHelp(ob)
assert.Equal( assert.Equal(
@@ -43,9 +29,9 @@ func Test_caHelp(t *testing.T) {
"Usage of "+os.Args[0]+" ca <flags>: create a self signed certificate authority\n"+ "Usage of "+os.Args[0]+" ca <flags>: create a self signed certificate authority\n"+
" Pass \"-\" to any path flag to read from stdin or write to stdout.\n"+ " Pass \"-\" to any path flag to read from stdin or write to stdout.\n"+
" -argon-iterations uint\n"+ " -argon-iterations uint\n"+
" \tOptional: Argon2 iterations parameter used for encrypted private key passphrase (default "+defaultArgonIterations+")\n"+ " \tOptional: Argon2 iterations parameter used for encrypted private key passphrase (default 1)\n"+
" -argon-memory uint\n"+ " -argon-memory uint\n"+
" \tOptional: Argon2 memory parameter (in KiB) used for encrypted private key passphrase (default "+defaultArgonMemory+")\n"+ " \tOptional: Argon2 memory parameter (in KiB) used for encrypted private key passphrase (default 2097152)\n"+
" -argon-parallelism uint\n"+ " -argon-parallelism uint\n"+
" \tOptional: Argon2 parallelism parameter used for encrypted private key passphrase (default 4)\n"+ " \tOptional: Argon2 parallelism parameter used for encrypted private key passphrase (default 4)\n"+
" -curve string\n"+ " -curve string\n"+
@@ -202,16 +188,10 @@ func Test_ca(t *testing.T) {
k, _ := pem.Decode(rb) k, _ := pem.Decode(rb)
ned, err := cert.UnmarshalNebulaEncryptedData(k.Bytes) ned, err := cert.UnmarshalNebulaEncryptedData(k.Bytes)
require.NoError(t, err) require.NoError(t, err)
// we won't know salt in advance, so just check start of string
if bits.UintSize == 32 { assert.Equal(t, uint32(2*1024*1024), ned.EncryptionMetadata.Argon2Parameters.Memory)
assert.Equal(t, uint32(64*1024), ned.EncryptionMetadata.Argon2Parameters.Memory)
assert.Equal(t, uint32(3), ned.EncryptionMetadata.Argon2Parameters.Iterations)
} else {
assert.Equal(t, uint32(2*1024*1024), ned.EncryptionMetadata.Argon2Parameters.Memory)
assert.Equal(t, uint32(1), ned.EncryptionMetadata.Argon2Parameters.Iterations)
}
assert.Equal(t, uint8(4), ned.EncryptionMetadata.Argon2Parameters.Parallelism) assert.Equal(t, uint8(4), ned.EncryptionMetadata.Argon2Parameters.Parallelism)
assert.Equal(t, uint32(1), ned.EncryptionMetadata.Argon2Parameters.Iterations)
// verify the key is valid and decrypt-able // verify the key is valid and decrypt-able
var curve cert.Curve var curve cert.Curve
+4 -8
View File
@@ -53,12 +53,7 @@ func main() {
l := logging.NewLogger(os.Stdout) l := logging.NewLogger(os.Stdout)
if *serviceFlag != "" { if *serviceFlag != "" {
if *configTest { if err := doService(configPath, configTest, Build, serviceFlag); err != nil {
fmt.Println("-test is not supported with -service, run the config test without -service")
os.Exit(1)
}
if err := doService(configPath, Build, serviceFlag); err != nil {
l.Error("Service command failed", "error", err) l.Error("Service command failed", "error", err)
os.Exit(1) os.Exit(1)
} }
@@ -98,14 +93,15 @@ func main() {
} }
if !*configTest { if !*configTest {
if err := ctrl.Start(); err != nil { wait, err := ctrl.Start()
if err != nil {
util.LogWithContextIfNeeded("Error while running", err, l) util.LogWithContextIfNeeded("Error while running", err, l)
os.Exit(1) os.Exit(1)
} }
go ctrl.ShutdownBlock() go ctrl.ShutdownBlock()
if err := ctrl.Wait(); err != nil { if err := wait(); err != nil {
l.Error("Nebula stopped due to fatal error", "error", err) l.Error("Nebula stopped due to fatal error", "error", err)
os.Exit(2) os.Exit(2)
} }
+6 -25
View File
@@ -3,7 +3,6 @@ package main
import ( import (
"fmt" "fmt"
"log" "log"
"os"
"github.com/kardianos/service" "github.com/kardianos/service"
"github.com/slackhq/nebula" "github.com/slackhq/nebula"
@@ -15,6 +14,7 @@ var logger service.Logger
type program struct { type program struct {
configPath *string configPath *string
configTest *bool
build string build string
control *nebula.Control control *nebula.Control
} }
@@ -40,41 +40,22 @@ func (p *program) Start(s service.Service) error {
} }
}) })
p.control, err = nebula.Main(c, false, Build, l, nil) p.control, err = nebula.Main(c, *p.configTest, Build, l, nil)
if err != nil { if err != nil {
return err return err
} }
if err := p.control.Start(); err != nil { p.control.Start()
return err
}
// Nebula can stop itself on a fatal packet reader error, make sure to log it if it happens.
go func() {
if err := p.control.Wait(); err != nil {
logger.Error(fmt.Sprintf("Nebula stopped due to fatal error: %v", err))
os.Exit(2)
}
}()
return nil return nil
} }
func (p *program) Stop(s service.Service) error { func (p *program) Stop(s service.Service) error {
logger.Info("Nebula service stopping.") logger.Info("Nebula service stopping.")
if p.control == nil {
return nil
}
p.control.Stop() p.control.Stop()
// block until nebula has fully drained before reporting stopped.
// error logging is handled by Start.
_ = p.control.Wait()
return nil return nil
} }
func doService(configPath *string, build string, serviceFlag *string) error { func doService(configPath *string, configTest *bool, build string, serviceFlag *string) error {
if *configPath == "" { if *configPath == "" {
p, err := config.DefaultPath() p, err := config.DefaultPath()
if err != nil { if err != nil {
@@ -92,6 +73,7 @@ func doService(configPath *string, build string, serviceFlag *string) error {
prg := &program{ prg := &program{
configPath: configPath, configPath: configPath,
configTest: configTest,
build: build, build: build,
} }
@@ -123,9 +105,8 @@ func doService(configPath *string, build string, serviceFlag *string) error {
switch *serviceFlag { switch *serviceFlag {
case "run": case "run":
if err := s.Run(); err != nil { if err := s.Run(); err != nil {
// Route any errors to the system logger and report the failure // Route any errors to the system logger
logger.Error(err) logger.Error(err)
return err
} }
default: default:
if err := service.Control(s, *serviceFlag); err != nil { if err := service.Control(s, *serviceFlag); err != nil {
-96
View File
@@ -1,96 +0,0 @@
//go:build linux && !android && !e2e_testing
package main
import (
"fmt"
"net/netip"
"os"
"path/filepath"
"runtime"
"testing"
"time"
"github.com/slackhq/nebula"
"github.com/slackhq/nebula/cert"
cert_test "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/require"
)
// TestControlStopClosesOnTimer reproduces the dnclient lifecycle: nebula runs as
// a library, and on a config update dnclient calls Stop() in-process to tear the
// old instance down before starting a new one. This boots a real nebula (real
// blocking UDP sockets, tun disabled), lets it run, then Stop()s it on a timer
// and asserts it actually closes. If the reader goroutines parked in recvmmsg
// don't wake on Close(), Wait() blocks forever and this fails with a goroutine
// dump instead of relying on a process signal to unstick them.
func TestControlStopClosesOnTimer(t *testing.T) {
l := test.NewLogger()
dir := t.TempDir()
before := time.Now().Add(-time.Hour)
after := time.Now().Add(time.Hour)
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, before, after, nil, nil, nil)
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
_, _, keyPEM, certPEM := cert_test.NewTestCert(cert.Version2, cert.Curve_CURVE25519, ca, caKey, "close-on-timer", before, after, networks, nil, nil)
caPath := filepath.Join(dir, "ca.pem")
certPath := filepath.Join(dir, "cert.pem")
keyPath := filepath.Join(dir, "key.pem")
require.NoError(t, os.WriteFile(caPath, caPEM, 0o600))
require.NoError(t, os.WriteFile(certPath, certPEM, 0o600))
require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600))
// tun disabled so no device/root is needed; routines: 2 so we exercise the
// multi-socket (SO_REUSEPORT) teardown, which is where dnclient runs.
configBody := fmt.Sprintf(`
pki:
ca: %s
cert: %s
key: %s
listen:
host: 127.0.0.1
port: 0
tun:
disabled: true
firewall:
outbound:
- port: any
proto: any
host: any
inbound:
- port: any
proto: any
host: any
routines: 2
`, caPath, certPath, keyPath)
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(configBody), 0o600))
c := config.NewC(l)
require.NoError(t, c.Load(dir))
ctrl, err := nebula.Main(c, false, "close-on-timer", l, nil)
require.NoError(t, err)
require.NoError(t, ctrl.Start())
// Run like a live nebula, then close on a timer, exactly as dnclient does.
<-time.NewTimer(5 * time.Second).C
stopped := make(chan struct{})
go func() {
ctrl.Stop() // closes the udp sockets (shutdown(2)) and the tun
ctrl.Wait() // blocks until every reader goroutine has returned
close(stopped)
}()
select {
case <-stopped:
t.Log("nebula closed cleanly on timer")
case <-time.After(10 * time.Second):
buf := make([]byte, 1<<20)
n := runtime.Stack(buf, true)
t.Fatalf("nebula did NOT close within 10s of Stop(): a blocking reader never woke\n%s", buf[:n])
}
}
+3 -2
View File
@@ -84,7 +84,8 @@ func main() {
} }
if !*configTest { if !*configTest {
if err := ctrl.Start(); err != nil { wait, err := ctrl.Start()
if err != nil {
util.LogWithContextIfNeeded("Error while running", err, l) util.LogWithContextIfNeeded("Error while running", err, l)
os.Exit(1) os.Exit(1)
} }
@@ -92,7 +93,7 @@ func main() {
go ctrl.ShutdownBlock() go ctrl.ShutdownBlock()
notifyReady(l) notifyReady(l)
if err := ctrl.Wait(); err != nil { if err := wait(); err != nil {
l.Error("Nebula stopped due to fatal error", "error", err) l.Error("Nebula stopped due to fatal error", "error", err)
os.Exit(2) os.Exit(2)
} }
-20
View File
@@ -323,12 +323,6 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
return closeTunnel, hostinfo, nil return closeTunnel, hostinfo, nil
} }
if hostinfo.ConnectionState != nil && hostinfo.ConnectionState.messageCounter.Load() >= RejectAfterMessages {
// Send path can't encrypt a CloseTunnel notify, so just delete locally; the peer recovers via recv_error.
hostinfo.logger(cm.l).Error("Dropping tunnel, message counter is exhausted")
return deleteTunnel, hostinfo, nil
}
primary := cm.hostMap.Hosts[hostinfo.vpnAddrs[0]] primary := cm.hostMap.Hosts[hostinfo.vpnAddrs[0]]
mainHostInfo := true mainHostInfo := true
if primary != nil && primary != hostinfo { if primary != nil && primary != hostinfo {
@@ -454,11 +448,6 @@ func (cm *connectionManager) shouldSwapPrimary(current *HostInfo) bool {
return false return false
} }
if current.ConnectionState.messageCounter.Load() >= RehandshakeAfterMessages {
// This tunnel is being rolled for counter exhaustion, never swap back onto its spent key.
return false
}
crt := cm.intf.pki.getCertState().getCertificate(current.ConnectionState.myCert.Version()) crt := cm.intf.pki.getCertState().getCertificate(current.ConnectionState.myCert.Version())
if crt == nil { if crt == nil {
//my cert was reloaded away. We should definitely swap from this tunnel //my cert was reloaded away. We should definitely swap from this tunnel
@@ -555,15 +544,6 @@ func (cm *connectionManager) tryRehandshake(hostinfo *HostInfo) {
"reason", "current cert version < pki.initiatingVersion", "reason", "current cert version < pki.initiatingVersion",
) )
cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
return
}
if hostinfo.ConnectionState.messageCounter.Load() >= RehandshakeAfterMessages {
cm.l.Info("Re-handshaking with remote",
"vpnAddrs", hostinfo.vpnAddrs,
"reason", "message counter rehandshake threshold reached",
)
cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil) cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
return return
} }
-74
View File
@@ -25,7 +25,6 @@ func newTestLighthouse() *LightHouse {
lighthouses := []netip.Addr{} lighthouses := []netip.Addr{}
staticList := map[netip.Addr]struct{}{} staticList := map[netip.Addr]struct{}{}
lh.localAddrsFn = func(*LocalAllowList) []netip.Addr { return nil }
lh.lighthouses.Store(&lighthouses) lh.lighthouses.Store(&lighthouses)
lh.staticList.Store(&staticList) lh.staticList.Store(&staticList)
@@ -199,79 +198,6 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0]) assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
} }
func Test_NewConnectionManager_CounterLimits(t *testing.T) {
l := test.NewLogger()
localrange := netip.MustParsePrefix("10.1.1.1/24")
vpnIp := netip.MustParseAddr("172.1.1.2")
preferredRanges := []netip.Prefix{localrange}
// Very incomplete mock objects
hostMap := newHostMap(l)
hostMap.preferredRanges.Store(&preferredRanges)
cs := &CertState{
initiatingVersion: cert.Version1,
privateKey: []byte{},
v1Cert: &dummyCert{version: cert.Version1},
v1Credential: nil,
}
lh := newTestLighthouse()
ifce := &Interface{
hostMap: hostMap,
inside: &overlaytest.NoopTun{},
outside: &udp.NoopConn{},
firewall: &Firewall{},
lightHouse: lh,
pki: &PKI{},
myVpnAddrs: []netip.Addr{netip.MustParseAddr("172.1.1.1")}, // sorts below vpnIp so shouldSwapPrimary can proceed
handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
l: l,
}
ifce.pki.cs.Store(cs)
conf := config.NewC(test.NewLogger())
punchy := NewPunchyFromConfig(test.NewLogger(), conf, nil)
nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy)
nc.intf = ifce
hostinfo := &HostInfo{
vpnAddrs: []netip.Addr{vpnIp},
localIndexId: 1099,
remoteIndexId: 9901,
}
hostinfo.ConnectionState = &ConnectionState{
myCert: &dummyCert{version: cert.Version1},
}
nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
// Below the rehandshake threshold, no handshake is started
hostinfo.ConnectionState.messageCounter.Store(RehandshakeAfterMessages - 1)
nc.tryRehandshake(hostinfo)
assert.Nil(t, ifce.handshakeManager.QueryVpnAddr(vpnIp))
// A tunnel on its current cert would normally swap to primary
assert.True(t, nc.shouldSwapPrimary(hostinfo))
// At the rehandshake threshold, a new handshake is started
hostinfo.ConnectionState.messageCounter.Store(RehandshakeAfterMessages)
nc.tryRehandshake(hostinfo)
assert.NotNil(t, ifce.handshakeManager.QueryVpnAddr(vpnIp))
// An exhausted tunnel being rolled must never swap back to primary onto its spent key
assert.False(t, nc.shouldSwapPrimary(hostinfo))
// Still below the reject limit, the tunnel stays up
nc.In(hostinfo)
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
assert.Equal(t, tryRehandshake, decision)
// At the reject limit, the tunnel is deleted locally without a doomed CloseTunnel notify
hostinfo.ConnectionState.messageCounter.Store(RejectAfterMessages)
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
assert.Equal(t, deleteTunnel, decision)
}
func Test_NewConnectionManager_DisconnectInactive(t *testing.T) { func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
l := test.NewLogger() l := test.NewLogger()
localrange := netip.MustParsePrefix("10.1.1.1/24") localrange := netip.MustParsePrefix("10.1.1.1/24")
+3 -82
View File
@@ -2,29 +2,15 @@ package nebula
import ( import (
"encoding/json" "encoding/json"
"fmt"
"log/slog"
"sync" "sync"
"sync/atomic" "sync/atomic"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/handshake" "github.com/slackhq/nebula/handshake"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/noiseutil" "github.com/slackhq/nebula/noiseutil"
) )
const ( const ReplayWindow = 1024
ReplayWindow = 1024
// RehandshakeAfterMessages rolls keys inside the AES-GCM data-volume margin (~2^-36 advantage at 64KB frames).
RehandshakeAfterMessages = uint64(1) << 34
// RejectAfterMessages is the nonce ceiling enforced by noiseutil; a tunnel here is deleted locally, not notified.
RejectAfterMessages = noiseutil.RejectAfterMessages
)
// RehandshakeAfterMessages must stay below RejectAfterMessages so tunnels roll before the hard send stop.
const _ = RejectAfterMessages - RehandshakeAfterMessages
type ConnectionState struct { type ConnectionState struct {
eKey noiseutil.CipherState eKey noiseutil.CipherState
@@ -34,7 +20,6 @@ type ConnectionState struct {
initiator bool initiator bool
messageCounter atomic.Uint64 messageCounter atomic.Uint64
window *Bits window *Bits
decryptLock sync.Mutex
writeLock sync.Mutex writeLock sync.Mutex
} }
@@ -42,12 +27,7 @@ type ConnectionState struct {
// completed handshake.Result. It seeds messageCounter and the replay window so // completed handshake.Result. It seeds messageCounter and the replay window so
// that the post-handshake message indices already used on the wire don't count // that the post-handshake message indices already used on the wire don't count
// as missed traffic in the data plane. // as missed traffic in the data plane.
func newConnectionStateFromResult(r *handshake.Result) (*ConnectionState, error) { func newConnectionStateFromResult(r *handshake.Result) *ConnectionState {
// Refuse a MessageIndex too big for the replay window: it can only be a bug, and would spin the seed loop below.
if r.MessageIndex >= ReplayWindow {
return nil, fmt.Errorf("handshake message index %d exceeds replay window", r.MessageIndex)
}
ci := &ConnectionState{ ci := &ConnectionState{
myCert: r.MyCert, myCert: r.MyCert,
initiator: r.Initiator, initiator: r.Initiator,
@@ -60,7 +40,7 @@ func newConnectionStateFromResult(r *handshake.Result) (*ConnectionState, error)
for i := uint64(1); i <= r.MessageIndex; i++ { for i := uint64(1); i <= r.MessageIndex; i++ {
ci.window.Update(nil, i) ci.window.Update(nil, i)
} }
return ci, nil return ci
} }
func (cs *ConnectionState) MarshalJSON() ([]byte, error) { func (cs *ConnectionState) MarshalJSON() ([]byte, error) {
@@ -71,65 +51,6 @@ func (cs *ConnectionState) MarshalJSON() ([]byte, error) {
}) })
} }
// NextMessageCounter reserves the next 1-based counter; RejectAfterMessages is the first we refuse, pinned to not wrap.
func (cs *ConnectionState) NextMessageCounter() (uint64, bool) {
c := cs.messageCounter.Add(1)
if c >= RejectAfterMessages {
cs.messageCounter.Store(RejectAfterMessages)
return c, false
}
return c, true
}
func (cs *ConnectionState) Curve() cert.Curve { func (cs *ConnectionState) Curve() cert.Curve {
return cs.myCert.Curve() return cs.myCert.Curve()
} }
func (cs *ConnectionState) Decrypt(l *slog.Logger, messageCounter uint64, out []byte, packet []byte, nb []byte) ([]byte, error) {
var err error
cs.decryptLock.Lock()
result := cs.window.Check(l, messageCounter)
cs.decryptLock.Unlock()
if !result {
return nil, ErrAlreadySeen
}
out, err = cs.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], messageCounter, nb)
if err != nil {
return nil, err
}
cs.decryptLock.Lock()
result = cs.window.Update(l, messageCounter)
cs.decryptLock.Unlock()
if !result {
return nil, ErrAlreadySeen
}
return out, nil
}
// VerifyRelay verifies AEAD protected (but not encrypted) relay frames. packet must be length-checked by the caller.
func (cs *ConnectionState) VerifyRelay(l *slog.Logger, messageCounter uint64, packet []byte, nb []byte) error {
cs.decryptLock.Lock()
result := cs.window.Check(l, messageCounter)
cs.decryptLock.Unlock()
if !result {
return ErrAlreadySeen
}
signedPayload := packet[:len(packet)-cs.dKey.Overhead()]
signatureValue := packet[len(packet)-cs.dKey.Overhead():]
_, err := cs.dKey.DecryptDanger(nil, signedPayload, signatureValue, messageCounter, nb)
if err != nil {
return err
}
cs.decryptLock.Lock()
result = cs.window.Update(l, messageCounter)
cs.decryptLock.Unlock()
if !result {
return ErrAlreadySeen
}
return nil
}
+2 -53
View File
@@ -6,12 +6,10 @@ import (
"time" "time"
"github.com/flynn/noise" "github.com/flynn/noise"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
ct "github.com/slackhq/nebula/cert_test" ct "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/handshake" "github.com/slackhq/nebula/handshake"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
@@ -81,51 +79,11 @@ func runTestHandshake(t *testing.T) (initR, respR *handshake.Result) {
return initR, respR return initR, respR
} }
func TestConnectionState_NextMessageCounter(t *testing.T) {
cs := &ConnectionState{}
cs.messageCounter.Store(RejectAfterMessages - 2)
c, ok := cs.NextMessageCounter()
assert.True(t, ok)
assert.Equal(t, RejectAfterMessages-1, c)
// Hitting the limit refuses and pins the counter there
c, ok = cs.NextMessageCounter()
assert.False(t, ok)
assert.Equal(t, RejectAfterMessages, c)
assert.Equal(t, RejectAfterMessages, cs.messageCounter.Load())
// Continued send attempts stay refused and the counter never wraps
for i := 0; i < 10; i++ {
_, ok = cs.NextMessageCounter()
assert.False(t, ok)
}
assert.Equal(t, RejectAfterMessages, cs.messageCounter.Load())
}
// TestSendNoMetricsDropsExhausted drives the send path to the exhausted drop; metric and out flag prove it.
func TestSendNoMetricsDropsExhausted(t *testing.T) {
initR, _ := runTestHandshake(t)
ci, err := newConnectionStateFromResult(initR)
require.NoError(t, err)
ci.messageCounter.Store(RejectAfterMessages - 1)
f := &Interface{l: test.NewLogger(), messageMetrics: &MessageMetrics{txExhausted: metrics.NewCounter()}}
hostinfo := &HostInfo{vpnAddrs: []netip.Addr{netip.MustParseAddr("10.0.0.1")}, ConnectionState: ci}
f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, []byte{}, make([]byte, 12), make([]byte, mtu), 0)
// The crossing send is refused: it records an exhaustion drop and never reaches connectionManager.Out.
assert.Equal(t, int64(1), f.messageMetrics.txExhausted.Count())
assert.False(t, hostinfo.out.Load())
}
func TestNewConnectionStateFromResult(t *testing.T) { func TestNewConnectionStateFromResult(t *testing.T) {
initR, respR := runTestHandshake(t) initR, respR := runTestHandshake(t)
t.Run("initiator", func(t *testing.T) { t.Run("initiator", func(t *testing.T) {
ci, err := newConnectionStateFromResult(initR) ci := newConnectionStateFromResult(initR)
require.NoError(t, err)
assert.True(t, ci.initiator) assert.True(t, ci.initiator)
assert.Equal(t, initR.MyCert, ci.myCert) assert.Equal(t, initR.MyCert, ci.myCert)
assert.Equal(t, initR.RemoteCert, ci.peerCert) assert.Equal(t, initR.RemoteCert, ci.peerCert)
@@ -144,17 +102,8 @@ func TestNewConnectionStateFromResult(t *testing.T) {
assert.True(t, ci.window.Check(nil, 3), "counter 3 must not be pre-seeded") assert.True(t, ci.window.Check(nil, 3), "counter 3 must not be pre-seeded")
}) })
t.Run("message index too large is refused", func(t *testing.T) {
bad := *initR
bad.MessageIndex = ReplayWindow
ci, err := newConnectionStateFromResult(&bad)
require.Error(t, err)
assert.Nil(t, ci)
})
t.Run("responder", func(t *testing.T) { t.Run("responder", func(t *testing.T) {
ci, err := newConnectionStateFromResult(respR) ci := newConnectionStateFromResult(respR)
require.NoError(t, err)
assert.False(t, ci.initiator) assert.False(t, ci.initiator)
assert.Equal(t, respR.MyCert, ci.myCert) assert.Equal(t, respR.MyCert, ci.myCert)
assert.Equal(t, respR.RemoteCert, ci.peerCert) assert.Equal(t, respR.RemoteCert, ci.peerCert)
+24 -58
View File
@@ -53,7 +53,6 @@ type Control struct {
statsStart func() statsStart func()
dnsStart func() dnsStart func()
lighthouseStart func() lighthouseStart func()
networkChangeStart func(rebind func())
connectionManagerStart func(context.Context) connectionManagerStart func(context.Context)
} }
@@ -70,29 +69,29 @@ type ControlHostInfo struct {
} }
// Start actually runs nebula, this is a nonblocking call. // Start actually runs nebula, this is a nonblocking call.
// Use Wait to block until nebula has fully stopped and to learn whether a fatal reader error caused the shutdown. // The returned function blocks until nebula has fully stopped and returns the
func (c *Control) Start() error { // first fatal reader error (if any). A nil error means nebula shut down
// gracefully; a non-nil error means a reader hit an unexpected failure that
// triggered the shutdown.
func (c *Control) Start() (func() error, error) {
c.stateLock.Lock() c.stateLock.Lock()
defer c.stateLock.Unlock() defer c.stateLock.Unlock()
switch c.state { switch c.state {
case StateReady: case StateReady:
//yay! //yay!
case StateStopped, StateStopping: case StateStopped, StateStopping:
return ErrAlreadyStopped return nil, ErrAlreadyStopped
case StateStarted: case StateStarted:
return ErrAlreadyStarted return nil, ErrAlreadyStarted
default: default:
return ErrUnknownState return nil, ErrUnknownState
} }
// Activate the interface // Activate the interface
err := c.f.activate() err := c.f.activate()
if err != nil { if err != nil {
// Cancel before Close so a caller returning from Wait always observes a dead Context
c.cancel()
_ = c.f.Close()
c.state = StateStopped c.state = StateStopped
return err return nil, err
} }
// Call all the delayed funcs that waited patiently for the interface to be created. // Call all the delayed funcs that waited patiently for the interface to be created.
@@ -105,9 +104,6 @@ func (c *Control) Start() error {
if c.dnsStart != nil { if c.dnsStart != nil {
go c.dnsStart() go c.dnsStart()
} }
if c.networkChangeStart != nil {
go c.networkChangeStart(c.RebindUDPServer)
}
if c.connectionManagerStart != nil { if c.connectionManagerStart != nil {
go c.connectionManagerStart(c.ctx) go c.connectionManagerStart(c.ctx)
} }
@@ -118,9 +114,13 @@ func (c *Control) Start() error {
c.f.triggerShutdown = c.Stop c.f.triggerShutdown = c.Stop
// Start reading packets. // Start reading packets.
c.f.run() out, err := c.f.run()
if err != nil {
c.state = StateStopped
return nil, err
}
c.state = StateStarted c.state = StateStarted
return nil return out, nil
} }
func (c *Control) State() RunState { func (c *Control) State() RunState {
@@ -133,26 +133,10 @@ func (c *Control) Context() context.Context {
return c.ctx return c.ctx
} }
// Stop tears nebula down, closing all tunnels and releasing everything it holds. // Stop is a non-blocking call that signals nebula to close all tunnels and shut down
// Use Wait to block until the shutdown has completed.
// A Control that has been stopped cannot be started again, Start will return ErrAlreadyStopped.
func (c *Control) Stop() { func (c *Control) Stop() {
c.stateLock.Lock() c.stateLock.Lock()
switch c.state { if c.state != StateStarted {
case StateStarted:
// Fall through to the full teardown below
case StateReady:
// Never started
c.cancel()
c.state = StateStopped
if err := c.f.Close(); err != nil {
c.l.Error("Close interface failed", "error", err)
}
c.stateLock.Unlock()
return
default:
c.stateLock.Unlock() c.stateLock.Unlock()
// We are stopping or stopped already // We are stopping or stopped already
return return
@@ -161,26 +145,19 @@ func (c *Control) Stop() {
c.state = StateStopping c.state = StateStopping
c.stateLock.Unlock() c.stateLock.Unlock()
// Closing tunnels can be slow with a large hostmap, don't hold the lock for it // Stop the handshakeManager (and other services), to prevent new tunnels from
// being created while we're shutting them all down.
c.cancel() c.cancel()
c.CloseAllTunnels(false)
c.stateLock.Lock() c.CloseAllTunnels(false)
c.state = StateStopped
if err := c.f.Close(); err != nil { if err := c.f.Close(); err != nil {
c.l.Error("Close interface failed", "error", err) c.l.Error("Close interface failed", "error", err)
} }
c.stateLock.Lock()
c.state = StateStopped
c.stateLock.Unlock() c.stateLock.Unlock()
} }
// Wait blocks until nebula has fully stopped, either via Stop or an internal fatal error,
// and returns the first fatal packet reader error if there was one.
// It is safe to call from multiple goroutines and at any point in the lifecycle,
// but a Wait on a Control that is never started and never stopped will block forever.
func (c *Control) Wait() error {
return c.f.wait()
}
// ShutdownBlock will listen for and block on term and interrupt signals, calling Control.Stop() once signalled // ShutdownBlock will listen for and block on term and interrupt signals, calling Control.Stop() once signalled
func (c *Control) ShutdownBlock() { func (c *Control) ShutdownBlock() {
sigChan := make(chan os.Signal, 1) sigChan := make(chan os.Signal, 1)
@@ -193,20 +170,9 @@ func (c *Control) ShutdownBlock() {
c.Stop() c.Stop()
} }
// RebindUDPServer asks the UDP listener to rebind it's listener. Mainly used on mobile clients when interfaces change. // RebindUDPServer asks the UDP listener to rebind it's listener. Mainly used on mobile clients when interfaces change
func (c *Control) RebindUDPServer() { func (c *Control) RebindUDPServer() {
c.stateLock.Lock() _ = c.f.outside.Rebind()
defer c.stateLock.Unlock()
if c.state != StateStarted {
return
}
// A failure here means we are likely still pinned to the interface we came up on, so the rest of this is
// unlikely to help. Say so instead of silently carrying on as if we rebound.
if err := c.f.outside.Rebind(); err != nil {
c.l.Error("Failed to rebind udp socket", "error", err)
}
// Trigger a lighthouse update, useful for mobile clients that should have an update interval of 0 // Trigger a lighthouse update, useful for mobile clients that should have an update interval of 0
c.f.lightHouse.SendUpdate() c.f.lightHouse.SendUpdate()
-292
View File
@@ -1,292 +0,0 @@
package nebula
import (
"context"
"errors"
"io"
"net/netip"
"sync"
"testing"
"time"
"github.com/gaissmai/bart"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type fakeDevice struct {
closeOnce sync.Once
closedCh chan struct{}
closed bool
}
func newFakeDevice() *fakeDevice {
return &fakeDevice{closedCh: make(chan struct{})}
}
// Read blocks until Close like a real tun with no traffic, then reports EOF
// the same way a closed device does
func (d *fakeDevice) Read(p []byte) (int, error) {
<-d.closedCh
return 0, io.EOF
}
func (d *fakeDevice) Write(p []byte) (int, error) { return len(p), nil }
func (d *fakeDevice) Close() error {
d.closeOnce.Do(func() {
d.closed = true
close(d.closedCh)
})
return nil
}
func (d *fakeDevice) Activate() error { return nil }
func (d *fakeDevice) Networks() []netip.Prefix { return nil }
func (d *fakeDevice) Name() string { return "fake" }
func (d *fakeDevice) RoutesFor(netip.Addr) routing.Gateways { return nil }
func (d *fakeDevice) SupportsMultiqueue() bool { return false }
func (d *fakeDevice) NewMultiQueueReader() (io.ReadWriteCloser, error) {
return nil, errors.New("unsupported")
}
// newReadyControl hand-builds the minimum Control that Main would have
// produced right before Start, including the construction token NewInterface
// takes so waiters block until Close releases the resources
func newReadyControl(t *testing.T) (*Control, *fakeDevice, *fakeConn) {
l := test.NewLogger()
dev := newFakeDevice()
conn := &fakeConn{}
ctx, cancel := context.WithCancel(context.Background())
myVpnNet := netip.MustParsePrefix("10.128.0.1/16")
nt := new(bart.Lite)
nt.Insert(myVpnNet)
cs := &CertState{
myVpnNetworks: []netip.Prefix{myVpnNet},
myVpnNetworksTable: nt,
}
lh, err := NewLightHouseFromConfig(ctx, l, config.NewC(l), cs, nil, nil)
require.NoError(t, err)
f := &Interface{
ctx: ctx,
inside: dev,
outside: conn,
writers: []udp.Conn{conn},
readers: make([]io.ReadWriteCloser, 1),
routines: 1,
hostMap: newHostMap(l),
lightHouse: lh,
l: l,
}
f.wg.Add(1)
return &Control{
state: StateReady,
f: f,
l: l,
ctx: ctx,
cancel: cancel,
}, dev, conn
}
func TestControl_StopBeforeStart(t *testing.T) {
c, dev, conn := newReadyControl(t)
// A Stop on a never started control must release everything Main acquired
c.Stop()
assert.Equal(t, StateStopped, c.State())
assert.True(t, dev.closed, "the tun device should have been closed")
assert.True(t, conn.closed, "the udp socket should have been closed")
require.ErrorIs(t, c.ctx.Err(), context.Canceled, "the service context should have been cancelled")
// Wait must return promptly now that the resources are released
require.NoError(t, c.Wait())
// A stopped control can never be started
require.ErrorIs(t, c.Start(), ErrAlreadyStopped)
// A second Stop is a harmless no-op
c.Stop()
assert.Equal(t, StateStopped, c.State())
require.NoError(t, c.Wait())
}
func TestControl_WaitBlocksUntilStop(t *testing.T) {
c, _, _ := newReadyControl(t)
done := make(chan error, 1)
go func() { done <- c.Wait() }()
select {
case <-done:
t.Fatal("Wait returned before Stop")
case <-time.After(50 * time.Millisecond):
}
c.Stop()
select {
case err := <-done:
require.NoError(t, err)
case <-time.After(time.Second):
t.Fatal("Wait did not return after Stop")
}
}
type fakeConn struct {
closed bool
rebinds int
}
func (c *fakeConn) Rebind() error { c.rebinds++; return nil }
func (c *fakeConn) LocalAddr() (netip.AddrPort, error) { return netip.AddrPort{}, nil }
func (c *fakeConn) ListenOut(_ udp.EncReader) error { return nil }
func (c *fakeConn) WriteTo(_ []byte, _ netip.AddrPort) error { return nil }
func (c *fakeConn) ReloadConfig(_ *config.C) {}
func (c *fakeConn) SupportsMultipleReaders() bool { return true }
func (c *fakeConn) Close() error { c.closed = true; return nil }
type multiqueueDevice struct {
*fakeDevice
}
func (d *multiqueueDevice) SupportsMultiqueue() bool { return true }
func TestControl_StartMultiqueueFailureReleases(t *testing.T) {
dev := &multiqueueDevice{fakeDevice: newFakeDevice()}
conn := &fakeConn{}
ctx, cancel := context.WithCancel(context.Background())
f := &Interface{
ctx: ctx,
inside: dev,
outside: conn,
writers: []udp.Conn{conn},
readers: make([]io.ReadWriteCloser, 2),
routines: 2,
l: test.NewLogger(),
}
f.wg.Add(1)
c := &Control{
state: StateReady,
f: f,
l: test.NewLogger(),
ctx: ctx,
cancel: cancel,
}
// The second reader fails to open, everything must be released
require.Error(t, c.Start())
assert.Equal(t, StateStopped, c.State())
assert.True(t, dev.closed, "the tun device should have been closed")
assert.True(t, conn.closed, "the udp socket should have been closed")
require.ErrorIs(t, c.ctx.Err(), context.Canceled)
// And Wait must not hang on the construction token
require.NoError(t, c.Wait())
}
func TestInterface_CloseIsIdempotent(t *testing.T) {
dev := newFakeDevice()
f := &Interface{
inside: dev,
l: test.NewLogger(),
}
f.wg.Add(1)
require.NoError(t, f.Close())
assert.True(t, dev.closed)
// A second Close must not double release the wg token or the device
require.NoError(t, f.Close())
require.NoError(t, f.wait())
}
func TestControl_FatalErrorReportsThroughWait(t *testing.T) {
c, dev, conn := newReadyControl(t)
// Mirror what Start wires up, without needing real packet readers
c.f.triggerShutdown = c.Stop
c.state = StateStarted
boom := errors.New("boom")
c.f.onFatal(boom)
require.ErrorIs(t, c.Wait(), boom)
assert.Equal(t, StateStopped, c.State())
assert.True(t, dev.closed)
assert.True(t, conn.closed)
// A second fatal error must not fire the shutdown again or replace the first
c.f.onFatal(errors.New("later"))
require.ErrorIs(t, c.Wait(), boom)
// Wait stays factual, a Stop after the death does not mask the error
c.Stop()
require.ErrorIs(t, c.Wait(), boom)
}
func TestControl_ConcurrentStopAndStart(t *testing.T) {
c, _, _ := newReadyControl(t)
var wg sync.WaitGroup
for i := 0; i < 2; i++ {
wg.Go(func() { c.Stop() })
}
wg.Go(func() { _ = c.Start() })
wg.Go(func() {
_ = c.Wait()
// A returned Wait must always observe the final state, no matter how
// the race resolved
assert.Equal(t, StateStopped, c.State())
})
wg.Wait()
// However the race resolves, the control must end fully stopped with no
// panic and Wait must observe the final state
require.NoError(t, c.Wait())
assert.Equal(t, StateStopped, c.State())
require.ErrorIs(t, c.Start(), ErrAlreadyStopped)
}
func TestControl_StartStopLifecycle(t *testing.T) {
c, dev, conn := newReadyControl(t)
require.NoError(t, c.Start())
assert.Equal(t, StateStarted, c.State())
require.ErrorIs(t, c.Start(), ErrAlreadyStarted)
// Stop must unpark the reader blocked in the device and release everything
c.Stop()
assert.Equal(t, StateStopped, c.State())
assert.True(t, dev.closed, "the tun device should have been closed")
assert.True(t, conn.closed, "the udp socket should have been closed")
require.ErrorIs(t, c.ctx.Err(), context.Canceled)
// The reader drained off a closed device, that is not a fatal error
require.NoError(t, c.Wait())
require.ErrorIs(t, c.Start(), ErrAlreadyStopped)
}
func TestControl_RebindIsGatedByState(t *testing.T) {
c, _, conn := newReadyControl(t)
// A rebind before Start reaches nothing, the interface is not up
c.RebindUDPServer()
assert.Equal(t, 0, conn.rebinds, "rebind before start must be a no-op")
require.NoError(t, c.Start())
c.RebindUDPServer()
assert.Equal(t, 1, conn.rebinds, "rebind while started must reach the conn")
// A rebind racing a completed stop must not touch the closed conn
c.Stop()
require.NoError(t, c.Wait())
c.RebindUDPServer()
assert.Equal(t, 1, conn.rebinds, "rebind after stop must be a no-op")
}
+1 -21
View File
@@ -108,19 +108,7 @@ func (c *Control) GetVpnAddrs() []netip.Addr {
} }
func (c *Control) GetUDPAddr() netip.AddrPort { func (c *Control) GetUDPAddr() netip.AddrPort {
return c.f.outside.(*udp.TesterConn).GetAddr() return c.f.outside.(*udp.TesterConn).Addr
}
// SetUDPAddr moves this node to a new underlay address, standing in for a laptop waking up on a different
// network. Register the new address with the router as well or nothing will route back.
func (c *Control) SetUDPAddr(addr netip.AddrPort) {
c.f.outside.(*udp.TesterConn).SetAddr(addr)
}
// SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of
// whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker.
func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {
c.f.lightHouse.localAddrsFn = fn
} }
func (c *Control) KillPendingTunnel(vpnIp netip.Addr) bool { func (c *Control) KillPendingTunnel(vpnIp netip.Addr) bool {
@@ -137,14 +125,6 @@ func (c *Control) GetHostmap() *HostMap {
return c.f.hostMap return c.f.hostMap
} }
// GetHostmapIndexCount returns the number of entries in the main hostmap Indexes table, holding
// the hostmap read lock so tests can poll it while connection manager churns tunnels.
func (c *Control) GetHostmapIndexCount() int {
c.f.hostMap.RLock()
defer c.f.hostMap.RUnlock()
return len(c.f.hostMap.Indexes)
}
func (c *Control) GetF() *Interface { func (c *Control) GetF() *Interface {
return c.f return c.f
} }
+7 -16
View File
@@ -97,7 +97,8 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
newAddr := getDnsServerAddr(c) newAddr := getDnsServerAddr(c)
d.serverMu.Lock() d.serverMu.Lock()
running := d.server != nil running := d.server
runningStarted := d.started
sameAddr := d.addr == newAddr sameAddr := d.addr == newAddr
d.addr = newAddr d.addr = newAddr
d.enabled.Store(enabled) d.enabled.Store(enabled)
@@ -111,7 +112,7 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
} }
if !enabled { if !enabled {
if running { if running != nil {
d.Stop() d.Stop()
} }
// Drop any records that accumulated while enabled; a later re-enable // Drop any records that accumulated while enabled; a later re-enable
@@ -120,12 +121,12 @@ func (d *dnsServer) reload(c *config.C, initial bool) error {
return nil return nil
} }
if !running { if running == nil {
// Was disabled (or never started); bring it up now. // Was disabled (or never started); bring it up now.
go d.Start() go d.Start()
} else if !sameAddr { } else if !sameAddr {
// Stop clears the slot before shutting down, otherwise the Start below can find the dying server and refuse d.shutdownServer(running, runningStarted, "reload")
d.Stop() // Old Start goroutine has now exited; bring up a fresh listener on the new address.
go d.Start() go d.Start()
} }
@@ -161,9 +162,7 @@ func (d *dnsServer) Start() {
started := make(chan struct{}) started := make(chan struct{})
d.serverMu.Lock() d.serverMu.Lock()
// Re-check enabled under the lock, a disable that raced our check above snapshots the slot under it too. if d.ctx.Err() != nil {
// Two reloads in quick succession can both spawn a Start, the loser would orphan the live listener past Stop
if d.ctx.Err() != nil || d.server != nil || !d.enabled.Load() {
d.serverMu.Unlock() d.serverMu.Unlock()
return return
} }
@@ -201,14 +200,6 @@ func (d *dnsServer) Start() {
close(started) close(started)
} }
// Release our slot, unless a reload already replaced us, so a dead listener can't block a future Start
d.serverMu.Lock()
if d.server == server {
d.server = nil
d.started = nil
}
d.serverMu.Unlock()
if err != nil { if err != nil {
d.l.Warn("Failed to run the DNS responder", "error", err) d.l.Warn("Failed to run the DNS responder", "error", err)
} }
+4 -206
View File
@@ -194,51 +194,14 @@ func TestDnsServer_reload_initial_serveDnsWithoutLighthouse(t *testing.T) {
} }
func TestDnsServer_reload_sameAddr_noOp(t *testing.T) { func TestDnsServer_reload_sameAddr_noOp(t *testing.T) {
port := freeUDPPort(t)
ds, c := newTestDnsServer(t) ds, c := newTestDnsServer(t)
setDnsConfig(c, "127.0.0.1", port, true, true) setDnsConfig(c, "127.0.0.1", "0", true, true)
require.NoError(t, ds.reload(c, true)) require.NoError(t, ds.reload(c, true))
// No server running yet, no addr change. Reload should not spawn anything.
go ds.Start()
waitForBind(t, ds)
ds.serverMu.Lock()
before := ds.server
ds.serverMu.Unlock()
require.NotNil(t, before)
// Same address, so the running listener must be left alone rather than rebuilt under live queries
require.NoError(t, ds.reload(c, false)) require.NoError(t, ds.reload(c, false))
assert.True(t, ds.enabled.Load()) assert.True(t, ds.enabled.Load())
assert.Nil(t, ds.server)
ds.serverMu.Lock()
after := ds.server
ds.serverMu.Unlock()
assert.Same(t, before, after, "a same-address reload must not restart the listener")
ds.Stop()
}
// The branch the old sameAddr test was accidentally hitting: enabled with nothing running means reload starts it.
func TestDnsServer_reload_whenNotRunning_starts(t *testing.T) {
port := freeUDPPort(t)
ds, c := newTestDnsServer(t)
setDnsConfig(c, "127.0.0.1", port, true, true)
// initial only records config, it never starts anything
require.NoError(t, ds.reload(c, true))
ds.serverMu.Lock()
assert.Nil(t, ds.server, "the initial reload must not start a listener")
ds.serverMu.Unlock()
require.NoError(t, ds.reload(c, false))
waitForBind(t, ds)
ds.serverMu.Lock()
assert.NotNil(t, ds.server, "a reload with nothing running should bring DNS up")
ds.serverMu.Unlock()
ds.Stop()
} }
func TestDnsServer_StartStop_lifecycle(t *testing.T) { func TestDnsServer_StartStop_lifecycle(t *testing.T) {
@@ -464,168 +427,3 @@ func waitFor(t *testing.T, cond func() bool) {
} }
t.Fatal("timed out waiting for condition") t.Fatal("timed out waiting for condition")
} }
// Two reloads in quick succession, or a HUP before Control.Start, can race two Starts at the same listener.
func TestDnsServer_Start_isIdempotent(t *testing.T) {
port := freeUDPPort(t)
ds, c := newTestDnsServer(t)
setDnsConfig(c, "127.0.0.1", port, true, true)
require.NoError(t, ds.reload(c, true))
go ds.Start()
waitForBind(t, ds)
ds.serverMu.Lock()
first := ds.server
ds.serverMu.Unlock()
require.NotNil(t, first)
// If the second Start replaces the tracked server, Stop kills the wrong one and the port leaks
done := make(chan struct{})
go func() {
ds.Start()
close(done)
}()
select {
case <-done:
case <-time.After(time.Second * 5):
t.Fatal("second Start never returned")
}
ds.serverMu.Lock()
second := ds.server
ds.serverMu.Unlock()
assert.Same(t, first, second, "a second Start must not replace the running server")
// The real proof, after Stop the port must actually be free
ds.Stop()
waitFor(t, func() bool {
pc, err := net.ListenPacket("udp", "127.0.0.1:"+port)
if err != nil {
return false
}
_ = pc.Close()
return true
})
}
// An address change must actually end up listening on the new port. Start's guard refuses when a server is already
// installed, so reload has to clear the slot before shutting the old one down.
func TestDnsServer_reload_addrChange_restarts(t *testing.T) {
first := freeUDPPort(t)
second := freeUDPPort(t)
ds, c := newTestDnsServer(t)
setDnsConfig(c, "127.0.0.1", first, true, true)
require.NoError(t, ds.reload(c, true))
go ds.Start()
waitForBind(t, ds)
// Cycle a few times, the failure this guards against depends on which goroutine wins serverMu
for i := range 8 {
want := second
if i%2 == 1 {
want = first
}
setDnsConfig(c, "127.0.0.1", want, true, true)
require.NoError(t, ds.reload(c, false))
waitForBind(t, ds)
ds.serverMu.Lock()
srv := ds.server
ds.serverMu.Unlock()
require.NotNil(t, srv, "reload left DNS down instead of restarting it")
require.Equal(t, "127.0.0.1:"+want, srv.Addr, "reload should be serving the new address")
}
// Land back on second so the port assertions below are meaningful
setDnsConfig(c, "127.0.0.1", second, true, true)
require.NoError(t, ds.reload(c, false))
waitForBind(t, ds)
// The old port must be released and the new one actually held
waitFor(t, func() bool {
pc, err := net.ListenPacket("udp", "127.0.0.1:"+first)
if err != nil {
return false
}
_ = pc.Close()
return true
})
_, err := net.ListenPacket("udp", "127.0.0.1:"+second)
require.Error(t, err, "the new address should be bound by the DNS responder")
ds.Stop()
}
// A listener that dies on its own must release the slot, or a later same-addr reload sees it as running and no-ops.
func TestDnsServer_Start_bindFailure_releasesSlot(t *testing.T) {
port := freeUDPPort(t)
blocker, err := net.ListenPacket("udp", "127.0.0.1:"+port)
require.NoError(t, err)
ds, c := newTestDnsServer(t)
setDnsConfig(c, "127.0.0.1", port, true, true)
require.NoError(t, ds.reload(c, true))
ds.Start() // returns once the bind fails
ds.serverMu.Lock()
assert.Nil(t, ds.server, "a listener that failed to bind must not stay parked in the slot")
ds.serverMu.Unlock()
// With the slot released, a reload can retry once the port frees up
require.NoError(t, blocker.Close())
require.NoError(t, ds.reload(c, false))
waitForBind(t, ds)
ds.serverMu.Lock()
assert.NotNil(t, ds.server, "a same-addr reload should retry after a failed bind")
ds.serverMu.Unlock()
ds.Stop()
}
// A disable that lands while Start is between its unlocked check and the guard must not leave a listener behind.
func TestDnsServer_Start_refusesWhenDisabledUnderLock(t *testing.T) {
port := freeUDPPort(t)
ds, c := newTestDnsServer(t)
setDnsConfig(c, "127.0.0.1", port, true, true)
require.NoError(t, ds.reload(c, true))
require.True(t, ds.enabled.Load())
// Holding serverMu parks Start on the lock, the only way to land the disable in that window on purpose
ds.serverMu.Lock()
done := make(chan struct{})
go func() {
ds.Start()
close(done)
}()
select {
case <-done:
ds.serverMu.Unlock()
t.Fatal("Start returned early, the test never exercised the window")
case <-time.After(time.Millisecond * 100):
}
// The disable reload's critical section. It sees nothing running, so it never calls Stop.
ds.enabled.Store(false)
ds.serverMu.Unlock()
select {
case <-done:
case <-time.After(time.Second * 5):
t.Fatal("Start never returned")
}
ds.serverMu.Lock()
assert.Nil(t, ds.server, "Start must not install a listener a disable already cancelled")
ds.serverMu.Unlock()
pc, err := net.ListenPacket("udp", "127.0.0.1:"+port)
require.NoError(t, err, "an orphaned listener is still holding the port")
_ = pc.Close()
}
+30 -98
View File
@@ -405,7 +405,7 @@ func TestStage1Race(t *testing.T) {
r.Log("Spin until connection manager tears down a tunnel") r.Log("Spin until connection manager tears down a tunnel")
for myControl.GetHostmapIndexCount()+theirControl.GetHostmapIndexCount() > 2 { for len(myControl.GetHostmap().Indexes)+len(theirControl.GetHostmap().Indexes) > 2 {
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
t.Log("Connection manager hasn't ticked yet") t.Log("Connection manager hasn't ticked yet")
time.Sleep(time.Second) time.Sleep(time.Second)
@@ -453,11 +453,9 @@ func TestUncleanShutdownRaceLoser(t *testing.T) {
r.Log("Nuke my hostmap") r.Log("Nuke my hostmap")
myHostmap := myControl.GetHostmap() myHostmap := myControl.GetHostmap()
myHostmap.Lock()
myHostmap.Hosts = map[netip.Addr]*nebula.HostInfo{} myHostmap.Hosts = map[netip.Addr]*nebula.HostInfo{}
myHostmap.Indexes = map[uint32]*nebula.HostInfo{} myHostmap.Indexes = map[uint32]*nebula.HostInfo{}
myHostmap.RemoteIndexes = map[uint32]*nebula.HostInfo{} myHostmap.RemoteIndexes = map[uint32]*nebula.HostInfo{}
myHostmap.Unlock()
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me again"))) myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me again")))
p = r.RouteForAllUntilTxTun(theirControl) p = r.RouteForAllUntilTxTun(theirControl)
@@ -467,10 +465,10 @@ func TestUncleanShutdownRaceLoser(t *testing.T) {
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
r.Log("Wait for the dead index to go away") r.Log("Wait for the dead index to go away")
start := theirControl.GetHostmapIndexCount() start := len(theirControl.GetHostmap().Indexes)
for { for {
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
if theirControl.GetHostmapIndexCount() < start { if len(theirControl.GetHostmap().Indexes) < start {
break break
} }
time.Sleep(time.Second) time.Sleep(time.Second)
@@ -506,11 +504,9 @@ func TestUncleanShutdownRaceWinner(t *testing.T) {
r.Log("Nuke my hostmap") r.Log("Nuke my hostmap")
theirHostmap := theirControl.GetHostmap() theirHostmap := theirControl.GetHostmap()
theirHostmap.Lock()
theirHostmap.Hosts = map[netip.Addr]*nebula.HostInfo{} theirHostmap.Hosts = map[netip.Addr]*nebula.HostInfo{}
theirHostmap.Indexes = map[uint32]*nebula.HostInfo{} theirHostmap.Indexes = map[uint32]*nebula.HostInfo{}
theirHostmap.RemoteIndexes = map[uint32]*nebula.HostInfo{} theirHostmap.RemoteIndexes = map[uint32]*nebula.HostInfo{}
theirHostmap.Unlock()
theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them again"))) theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them again")))
p = r.RouteForAllUntilTxTun(myControl) p = r.RouteForAllUntilTxTun(myControl)
@@ -521,10 +517,10 @@ func TestUncleanShutdownRaceWinner(t *testing.T) {
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
r.Log("Wait for the dead index to go away") r.Log("Wait for the dead index to go away")
start := myControl.GetHostmapIndexCount() start := len(myControl.GetHostmap().Indexes)
for { for {
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
if myControl.GetHostmapIndexCount() < start { if len(myControl.GetHostmap().Indexes) < start {
break break
} }
time.Sleep(time.Second) time.Sleep(time.Second)
@@ -632,10 +628,10 @@ func TestReestablishRelays(t *testing.T) {
r.Log("Close the tunnel") r.Log("Close the tunnel")
relayControl.CloseTunnel(theirVpnIpNet[0].Addr(), true) relayControl.CloseTunnel(theirVpnIpNet[0].Addr(), true)
start := myControl.GetHostmapIndexCount() start := len(myControl.GetHostmap().Indexes)
curIndexes := myControl.GetHostmapIndexCount() curIndexes := len(myControl.GetHostmap().Indexes)
for curIndexes >= start { for curIndexes >= start {
curIndexes = myControl.GetHostmapIndexCount() curIndexes = len(myControl.GetHostmap().Indexes)
r.Logf("Wait for the dead index to go away:start=%v indexes, current=%v indexes", start, curIndexes) r.Logf("Wait for the dead index to go away:start=%v indexes, current=%v indexes", start, curIndexes)
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me should fail"))) myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me should fail")))
@@ -725,70 +721,6 @@ func TestReestablishRelays(t *testing.T) {
} }
func TestRelayHandshakeOverDisestablishedEntry(t *testing.T) {
t.Parallel()
// If them tears down the tunnel while me keeps Established relay state, me's next
// handshake flows through the relay with no fresh CreateRelayRequest and lands on
// them's Disestablished terminal relay entry. them must re-establish that entry, or
// its first transmit deletes its only relay and the tunnel is born transmit-dead:
// them can receive but every send is silently dropped.
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me ", "10.128.0.1/24", m{"relay": m{"use_relays": true}})
relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay ", "10.128.0.128/24", m{"relay": m{"am_relay": true}})
theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them ", "10.128.0.2/24", m{"relay": m{"use_relays": true}})
// Teach my how to get to the relay and that their can be reached via the relay
myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
// Build a router so we don't have to reason who gets which packet
r := router.NewR(t, myControl, relayControl, theirControl)
defer r.RenderFlow()
// Start the servers
myControl.Start()
relayControl.Start()
theirControl.Start()
t.Log("Trigger a handshake from me to them via the relay")
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
p := r.RouteForAllUntilTxTun(theirControl)
assertUdpPacket(t, []byte("Hi from me"), p, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
oldIdx := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false).LocalIndex
t.Log("Close the tunnel on them only, marking their relay entry Disestablished")
theirControl.CloseTunnel(myVpnIpNet[0].Addr(), true)
t.Log("Re-handshake from me, riding the still-Established relay state")
myControl.ReHandshake(theirVpnIpNet[0].Addr())
for {
h := myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false)
if h != nil && h.LocalIndex != oldIdx && h.RemoteIndex != 0 {
break
}
r.RouteForAllExitFunc(func(*udp.Packet, *nebula.Control) router.ExitType {
return router.RouteAndExit
})
}
hAtThem := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
require.NotNil(t, hAtThem, "them should have completed the relayed handshake")
require.Equal(t, []netip.Addr{relayVpnIpNet[0].Addr()}, hAtThem.CurrentRelaysToMe, "them should know a relay for the new tunnel")
t.Log("Send from them to me; their only relay entry must survive the transmit")
theirControl.InjectTunPacket(BuildTunUDPPacket(myVpnIpNet[0].Addr(), 80, theirVpnIpNet[0].Addr(), 80, []byte("Hi from them")))
require.Never(t, func() bool {
h := theirControl.GetHostInfoByVpnAddr(myVpnIpNet[0].Addr(), false)
return h == nil || len(h.CurrentRelaysToMe) == 0
}, time.Second, 10*time.Millisecond, "them deleted its only relay entry; the tunnel is permanently transmit-dead")
p = r.RouteForAllUntilTxTun(myControl)
assertUdpPacket(t, []byte("Hi from them"), p, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), 80, 80)
r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
}
func TestStage1RaceRelays(t *testing.T) { func TestStage1RaceRelays(t *testing.T) {
t.Parallel() t.Parallel()
//NOTE: this is a race between me and relay resulting in a full tunnel from me to them via relay //NOTE: this is a race between me and relay resulting in a full tunnel from me to them via relay
@@ -887,18 +819,18 @@ func TestStage1RaceRelays2(t *testing.T) {
t.Log("Wait until we remove extra tunnels") t.Log("Wait until we remove extra tunnels")
t.Logf("Waiting for hostinfos to be removed... myControl=%d theirControl=%d relayControl=%d", t.Logf("Waiting for hostinfos to be removed... myControl=%d theirControl=%d relayControl=%d",
myControl.GetHostmapIndexCount(), len(myControl.GetHostmap().Indexes),
theirControl.GetHostmapIndexCount(), len(theirControl.GetHostmap().Indexes),
relayControl.GetHostmapIndexCount(), len(relayControl.GetHostmap().Indexes),
) )
hostInfos := myControl.GetHostmapIndexCount() + theirControl.GetHostmapIndexCount() + relayControl.GetHostmapIndexCount() hostInfos := len(myControl.GetHostmap().Indexes) + len(theirControl.GetHostmap().Indexes) + len(relayControl.GetHostmap().Indexes)
retries := 60 retries := 60
for hostInfos > 6 && retries > 0 { for hostInfos > 6 && retries > 0 {
hostInfos = myControl.GetHostmapIndexCount() + theirControl.GetHostmapIndexCount() + relayControl.GetHostmapIndexCount() hostInfos = len(myControl.GetHostmap().Indexes) + len(theirControl.GetHostmap().Indexes) + len(relayControl.GetHostmap().Indexes)
t.Logf("Waiting for hostinfos to be removed... myControl=%d theirControl=%d relayControl=%d", t.Logf("Waiting for hostinfos to be removed... myControl=%d theirControl=%d relayControl=%d",
myControl.GetHostmapIndexCount(), len(myControl.GetHostmap().Indexes),
theirControl.GetHostmapIndexCount(), len(theirControl.GetHostmap().Indexes),
relayControl.GetHostmapIndexCount(), len(relayControl.GetHostmap().Indexes),
) )
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
t.Log("Connection manager hasn't ticked yet") t.Log("Connection manager hasn't ticked yet")
@@ -992,24 +924,24 @@ func TestRehandshakingRelays(t *testing.T) {
assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
r.RenderHostmaps("working hostmaps", myControl, relayControl, theirControl) r.RenderHostmaps("working hostmaps", myControl, relayControl, theirControl)
// We should have two hostinfos on all sides // We should have two hostinfos on all sides
for myControl.GetHostmapIndexCount() != 2 { for len(myControl.GetHostmap().Indexes) != 2 {
t.Logf("Waiting for myControl hostinfos (%v != 2) to get cleaned up from lack of use...", myControl.GetHostmapIndexCount()) t.Logf("Waiting for myControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(myControl.GetHostmap().Indexes))
r.Log("Assert the relay tunnel still works") r.Log("Assert the relay tunnel still works")
assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
r.Log("yupitdoes") r.Log("yupitdoes")
time.Sleep(time.Second) time.Sleep(time.Second)
} }
t.Logf("myControl hostinfos got cleaned up!") t.Logf("myControl hostinfos got cleaned up!")
for theirControl.GetHostmapIndexCount() != 2 { for len(theirControl.GetHostmap().Indexes) != 2 {
t.Logf("Waiting for theirControl hostinfos (%v != 2) to get cleaned up from lack of use...", theirControl.GetHostmapIndexCount()) t.Logf("Waiting for theirControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(theirControl.GetHostmap().Indexes))
r.Log("Assert the relay tunnel still works") r.Log("Assert the relay tunnel still works")
assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
r.Log("yupitdoes") r.Log("yupitdoes")
time.Sleep(time.Second) time.Sleep(time.Second)
} }
t.Logf("theirControl hostinfos got cleaned up!") t.Logf("theirControl hostinfos got cleaned up!")
for relayControl.GetHostmapIndexCount() != 2 { for len(relayControl.GetHostmap().Indexes) != 2 {
t.Logf("Waiting for relayControl hostinfos (%v != 2) to get cleaned up from lack of use...", relayControl.GetHostmapIndexCount()) t.Logf("Waiting for relayControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(relayControl.GetHostmap().Indexes))
r.Log("Assert the relay tunnel still works") r.Log("Assert the relay tunnel still works")
assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
r.Log("yupitdoes") r.Log("yupitdoes")
@@ -1097,24 +1029,24 @@ func TestRehandshakingRelaysPrimary(t *testing.T) {
assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
r.RenderHostmaps("working hostmaps", myControl, relayControl, theirControl) r.RenderHostmaps("working hostmaps", myControl, relayControl, theirControl)
// We should have two hostinfos on all sides // We should have two hostinfos on all sides
for myControl.GetHostmapIndexCount() != 2 { for len(myControl.GetHostmap().Indexes) != 2 {
t.Logf("Waiting for myControl hostinfos (%v != 2) to get cleaned up from lack of use...", myControl.GetHostmapIndexCount()) t.Logf("Waiting for myControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(myControl.GetHostmap().Indexes))
r.Log("Assert the relay tunnel still works") r.Log("Assert the relay tunnel still works")
assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
r.Log("yupitdoes") r.Log("yupitdoes")
time.Sleep(time.Second) time.Sleep(time.Second)
} }
t.Logf("myControl hostinfos got cleaned up!") t.Logf("myControl hostinfos got cleaned up!")
for theirControl.GetHostmapIndexCount() != 2 { for len(theirControl.GetHostmap().Indexes) != 2 {
t.Logf("Waiting for theirControl hostinfos (%v != 2) to get cleaned up from lack of use...", theirControl.GetHostmapIndexCount()) t.Logf("Waiting for theirControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(theirControl.GetHostmap().Indexes))
r.Log("Assert the relay tunnel still works") r.Log("Assert the relay tunnel still works")
assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
r.Log("yupitdoes") r.Log("yupitdoes")
time.Sleep(time.Second) time.Sleep(time.Second)
} }
t.Logf("theirControl hostinfos got cleaned up!") t.Logf("theirControl hostinfos got cleaned up!")
for relayControl.GetHostmapIndexCount() != 2 { for len(relayControl.GetHostmap().Indexes) != 2 {
t.Logf("Waiting for relayControl hostinfos (%v != 2) to get cleaned up from lack of use...", relayControl.GetHostmapIndexCount()) t.Logf("Waiting for relayControl hostinfos (%v != 2) to get cleaned up from lack of use...", len(relayControl.GetHostmap().Indexes))
r.Log("Assert the relay tunnel still works") r.Log("Assert the relay tunnel still works")
assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r) assertTunnel(t, theirVpnIpNet[0].Addr(), myVpnIpNet[0].Addr(), theirControl, myControl, r)
r.Log("yupitdoes") r.Log("yupitdoes")
@@ -1191,7 +1123,7 @@ func TestRehandshaking(t *testing.T) {
theirConfig.ReloadConfigString(string(rc)) theirConfig.ReloadConfigString(string(rc))
r.Log("Spin until there is only 1 tunnel") r.Log("Spin until there is only 1 tunnel")
for myControl.GetHostmapIndexCount()+theirControl.GetHostmapIndexCount() > 2 { for len(myControl.GetHostmap().Indexes)+len(theirControl.GetHostmap().Indexes) > 2 {
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
t.Log("Connection manager hasn't ticked yet") t.Log("Connection manager hasn't ticked yet")
time.Sleep(time.Second) time.Sleep(time.Second)
@@ -1291,7 +1223,7 @@ func TestRehandshakingLoser(t *testing.T) {
myConfig.ReloadConfigString(string(rc)) myConfig.ReloadConfigString(string(rc))
r.Log("Spin until there is only 1 tunnel") r.Log("Spin until there is only 1 tunnel")
for myControl.GetHostmapIndexCount()+theirControl.GetHostmapIndexCount() > 2 { for len(myControl.GetHostmap().Indexes)+len(theirControl.GetHostmap().Indexes) > 2 {
assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r) assertTunnel(t, myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), myControl, theirControl, r)
t.Log("Connection manager hasn't ticked yet") t.Log("Connection manager hasn't ticked yet")
time.Sleep(time.Second) time.Sleep(time.Second)
-225
View File
@@ -1,225 +0,0 @@
//go:build e2e_testing
// +build e2e_testing
package e2e
import (
"net/netip"
"testing"
"time"
"github.com/slackhq/nebula"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/e2e/router"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// reportedAddrs is what the lighthouse would hand a peer asking where vpnAddr is.
func reportedAddrs(t *testing.T, lh *nebula.Control, vpnAddr netip.Addr) []netip.AddrPort {
t.Helper()
cm := lh.QueryLighthouse(vpnAddr)
if cm == nil {
return nil
}
var out []netip.AddrPort
for _, c := range *cm {
out = append(out, c.Reported...)
out = append(out, c.Learned...)
}
return out
}
// waitForLighthouseMsg routes until a lighthouse message lands on lh, or gives up. Reports whether one arrived.
func waitForLighthouseMsg(t *testing.T, r *router.R, lh *nebula.Control, wait time.Duration) bool {
t.Helper()
h := &header.H{}
return r.RouteForAllExitFuncOrTimeout(wait, func(p *udp.Packet, c *nebula.Control) router.ExitType {
if c != lh {
return router.KeepRouting
}
// Punches are a single byte and never parse, they are just not what we are after
if err := h.Parse(p.Data); err != nil {
return router.KeepRouting
}
if h.Type == header.LightHouse {
return router.RouteAndExit
}
return router.KeepRouting
})
}
// A laptop that changes networks has to tell the lighthouse promptly, otherwise the lighthouse keeps handing peers
// the old address and their punches land nowhere. On a long lighthouse interval the only thing that closes that
// window is the rebind, which on darwin the network change monitor drives. The e2e build compiles the monitor out,
// so we call RebindUDPServer directly, which is the same thing the monitor does.
func TestRebindSendsLighthouseUpdate(t *testing.T) {
t.Parallel()
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
"lighthouse": m{"am_lighthouse": true},
})
// 600s interval, so nothing scheduled can send an update during this test. A rebind is the only thing that can.
myControl, _, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{
"lighthouse": m{
"hosts": []any{lhVpnIpNet[0].Addr().String()},
"interval": 600,
},
"static_host_map": m{
lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
},
})
r := router.NewR(t, lhControl, myControl)
defer r.RenderFlow()
lhControl.Start()
myControl.Start()
// Let the startup registration finish, then clear everything it left behind
require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an initial registration")
r.RouteFor(time.Millisecond * 400)
// Nothing should be talking to the lighthouse on its own now
require.False(t, waitForLighthouseMsg(t, r, lhControl, time.Millisecond*200),
"nothing should reach the lighthouse before the rebind")
myControl.RebindUDPServer()
assert.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5),
"a rebind should push an update to the lighthouse rather than waiting out the interval")
lhControl.Stop()
myControl.Stop()
}
// The other half of a rebind: every live tunnel requeries the lighthouse on its next send. That query is what makes
// the lighthouse tell the peer to punch toward our new address, which is the part that actually revives a tunnel
// whose remote NAT state died while we were on a different network.
func TestRebindRequeriesPeersOnNextSend(t *testing.T) {
t.Parallel()
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
"lighthouse": m{"am_lighthouse": true},
})
lhCfg := m{
"lighthouse": m{
"hosts": []any{lhVpnIpNet[0].Addr().String()},
"interval": 600,
// Without this the peers advertise this machine's real addresses and then try to punch at them,
// which the router has no route for.
"local_allow_list": m{
"10.0.0.0/24": true,
"::/0": false,
},
},
"static_host_map": m{
lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
},
}
myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", lhCfg)
theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.3/24", lhCfg)
r := router.NewR(t, lhControl, myControl, theirControl)
defer r.RenderFlow()
lhControl.Start()
myControl.Start()
theirControl.Start()
r.RouteFor(time.Millisecond * 500)
// Point the peers at each other directly, this test is about the rebind and not about lighthouse discovery
myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("initial")))
r.RouteFor(time.Second)
require.NotNil(t, myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false), "expected a tunnel to them")
r.RouteFor(time.Millisecond * 300)
// Assert on what the peer sees rather than on lighthouse traffic. A query for them makes the lighthouse send
// them a punch notification, which is the whole point. Our own update to the lighthouse sends them nothing,
// so this cannot be satisfied by the update the rebind itself pushes.
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("quiet")))
require.False(t, waitForLighthouseMsg(t, r, theirControl, time.Millisecond*300),
"an ordinary send should not requery the lighthouse")
myControl.RebindUDPServer()
r.RouteFor(time.Millisecond * 300) // let the update the rebind itself sends pass by
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("after rebind")))
assert.True(t, waitForLighthouseMsg(t, r, theirControl, time.Second*5),
"the first send after a rebind should requery the lighthouse, which then tells the peer to punch at us")
lhControl.Stop()
myControl.Stop()
theirControl.Stop()
}
// The scenario this whole thing exists for: a laptop sleeps at the office and wakes up at home on a new address.
// Until it tells the lighthouse, the lighthouse keeps handing peers the office address, so their punches land
// nowhere and the tunnel stays dead. On a long interval the rebind is the only thing that closes that window.
func TestRebindAdvertisesNewAddressAfterMove(t *testing.T) {
t.Parallel()
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
"lighthouse": m{"am_lighthouse": true},
})
myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{
"lighthouse": m{
"hosts": []any{lhVpnIpNet[0].Addr().String()},
"interval": 600,
},
"static_host_map": m{
lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
},
})
// Advertise wherever we currently are rather than this machine's real NICs, read fresh each time so a move
// is picked up.
myControl.SetLocalAddrsFn(func(*nebula.LocalAllowList) []netip.Addr {
return []netip.Addr{myControl.GetUDPAddr().Addr()}
})
r := router.NewR(t, lhControl, myControl)
defer r.RenderFlow()
lhControl.Start()
myControl.Start()
require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an initial registration")
r.RouteFor(time.Millisecond * 400)
require.Contains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), myUdpAddr,
"the lighthouse should know the address we started on")
// Wake up somewhere else
newAddr := netip.MustParseAddrPort("10.0.0.99:4242")
myControl.SetUDPAddr(newAddr)
r.AddRoute(newAddr.Addr(), newAddr.Port(), myControl)
// Nothing has told the lighthouse, and with interval 600 nothing scheduled will
r.RouteFor(time.Millisecond * 400)
require.NotContains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), newAddr,
"the lighthouse should still be handing out the old address before the rebind")
myControl.RebindUDPServer()
require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an update after the rebind")
r.RouteFor(time.Millisecond * 400)
assert.Contains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), newAddr,
"after the rebind the lighthouse should hand peers our new address")
lhControl.Stop()
myControl.Stop()
}
+28 -130
View File
@@ -114,28 +114,6 @@ type packet struct {
packet *udp.Packet packet *udp.Packet
tun bool // a packet pulled off a tun device tun bool // a packet pulled off a tun device
rx bool // the packet was received by a udp device rx bool // the packet was received by a udp device
// h is the nebula header, parsed once when the packet is recorded. parseErr says why there isn't one, which
// the flow log reports rather than hiding. Punchy sends a single byte, so an unparseable packet is normal.
h header.H
parseErr error
}
// fromAddr and toAddr are the addresses this packet actually travelled between. Reading them off the control
// instead would misreport the whole history once a test moves a node. Tun packets are synthesized without
// addresses, so they fall back to the control.
func (p *packet) fromAddr() netip.AddrPort {
if p.tun || !p.packet.From.IsValid() {
return p.from.GetUDPAddr()
}
return p.packet.From
}
func (p *packet) toAddr() netip.AddrPort {
if p.tun || !p.packet.To.IsValid() {
return p.to.GetUDPAddr()
}
return p.packet.To
} }
func (p *packet) WasReceived() { func (p *packet) WasReceived() {
@@ -271,7 +249,7 @@ func (r *R) renderFlow() {
continue continue
} }
addr := e.packet.fromAddr() addr := e.packet.from.GetUDPAddr()
if _, ok := participants[addr]; ok { if _, ok := participants[addr]; ok {
continue continue
} }
@@ -290,6 +268,7 @@ func (r *R) renderFlow() {
} }
// Print packets // Print packets
h := &header.H{}
for _, e := range r.flow { for _, e := range r.flow {
if e.packet == nil { if e.packet == nil {
//fmt.Fprintf(f, " note over %s: %s\n", strings.Join(participantsVals, ", "), e.note) //fmt.Fprintf(f, " note over %s: %s\n", strings.Join(participantsVals, ", "), e.note)
@@ -301,22 +280,21 @@ func (r *R) renderFlow() {
fmt.Fprintln(f, r.formatUdpPacket(p)) fmt.Fprintln(f, r.formatUdpPacket(p))
} else { } else {
if err := h.Parse(p.packet.Data); err != nil {
panic(err)
}
line := "--x" line := "--x"
if p.rx { if p.rx {
line = "->>" line = "->>"
} }
detail := fmt.Sprintf("%s(%s), index %v, counter: %v", fmt.Fprintf(f,
p.h.TypeName(), p.h.SubTypeName(), p.h.RemoteIndex, p.h.MessageCounter) " %s%s%s: %s(%s), index %v, counter: %v\n",
if p.parseErr != nil { normalizeName(p.from.GetUDPAddr().String()),
detail = fmt.Sprintf("unparsed, %v (%d bytes)", p.parseErr, len(p.packet.Data))
}
fmt.Fprintf(f, " %s%s%s: %s\n",
normalizeName(p.fromAddr().String()),
line, line,
normalizeName(p.toAddr().String()), normalizeName(p.to.GetUDPAddr().String()),
detail, h.TypeName(), h.SubTypeName(), h.RemoteIndex, h.MessageCounter,
) )
} }
} }
@@ -430,34 +408,29 @@ func (r *R) unlockedInjectFlow(from, to *nebula.Control, p *udp.Packet, tun bool
r.renderHostmaps(fmt.Sprintf("Packet %v", len(r.flow))) r.renderHostmaps(fmt.Sprintf("Packet %v", len(r.flow)))
var h header.H if len(r.ignoreFlows) > 0 {
var parseErr error var h header.H
if !tun { err := h.Parse(p.Data)
parseErr = h.Parse(p.Data) if err != nil {
} panic(err)
// Decide before copying, the copy comes from a freelist and an ignored packet would never be released
for _, i := range r.ignoreFlows {
if tun {
if i.tun.HasValue && i.tun.IsTrue {
return nil
}
continue
} }
// A packet we could not parse has no type to match against, so no rule can ignore it for _, i := range r.ignoreFlows {
if parseErr == nil && i.messageType == h.Type && i.subType == h.Subtype { if !tun {
return nil if i.messageType == h.Type && i.subType == h.Subtype {
return nil
}
} else if i.tun.HasValue && i.tun.IsTrue {
return nil
}
} }
} }
fp := &packet{ fp := &packet{
from: from, from: from,
to: to, to: to,
packet: p.Copy(), packet: p.Copy(),
tun: tun, tun: tun,
h: h,
parseErr: parseErr,
} }
r.flow = append(r.flow, flowEntry{packet: fp}) r.flow = append(r.flow, flowEntry{packet: fp})
@@ -717,81 +690,6 @@ func (r *R) RouteUntilAfterMsgType(sender *nebula.Control, msgType header.Messag
}) })
} }
// RouteFor routes everything that shows up for the given duration and then returns. Use it to let a test settle
// deterministically rather than sleeping and hoping: a single FlushAll races a completing handshake, which queues
// more packets right behind it.
func (r *R) RouteFor(d time.Duration) {
r.RouteForAllExitFuncOrTimeout(d, func(*udp.Packet, *nebula.Control) ExitType {
return KeepRouting
})
}
// RouteForAllExitFuncOrTimeout is RouteForAllExitFunc with a deadline, reporting whether whatDo asked to exit
// before time ran out. The unbounded version blocks forever on a quiet network, so this is what a test needs to
// assert that something does NOT happen, or to route for a fixed settling period.
func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc) bool {
sc := make([]reflect.SelectCase, 0, len(r.controls)+1)
cm := make([]*nebula.Control, 0, len(r.controls))
for _, c := range r.controls {
sc = append(sc, reflect.SelectCase{
Dir: reflect.SelectRecv,
Chan: reflect.ValueOf(c.GetUDPTxChan()),
Send: reflect.Value{},
})
cm = append(cm, c)
}
timer := time.NewTimer(timeout)
defer timer.Stop()
sc = append(sc, reflect.SelectCase{
Dir: reflect.SelectRecv,
Chan: reflect.ValueOf(timer.C),
Send: reflect.Value{},
})
for {
x, rx, _ := reflect.Select(sc)
if x == len(cm) {
return false
}
r.Lock()
p := rx.Interface().(*udp.Packet)
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
if receiver == nil {
r.Unlock()
panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String())
}
e := whatDo(p, receiver)
switch e {
case ExitNow:
r.Unlock()
p.Release()
return true
case RouteAndExit:
fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
receiver.InjectUDPPacket(p)
fp.WasReceived()
r.Unlock()
p.Release()
return true
case KeepRouting:
fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
receiver.InjectUDPPacket(p)
fp.WasReceived()
default:
panic(fmt.Sprintf("Unknown exitFunc return: %v", e))
}
r.Unlock()
p.Release()
}
}
func (r *R) RouteForAllUntilAfterMsgTypeTo(receiver *nebula.Control, msgType header.MessageType, subType header.MessageSubType) { func (r *R) RouteForAllUntilAfterMsgTypeTo(receiver *nebula.Control, msgType header.MessageType, subType header.MessageSubType) {
h := &header.H{} h := &header.H{}
r.RouteForAllExitFunc(func(p *udp.Packet, r *nebula.Control) ExitType { r.RouteForAllExitFunc(func(p *udp.Packet, r *nebula.Control) ExitType {
+6 -6
View File
@@ -43,8 +43,8 @@ func TestDropInactiveTunnels(t *testing.T) {
r.Log("Go inactive and wait for the tunnels to get dropped") r.Log("Go inactive and wait for the tunnels to get dropped")
waitStart := time.Now() waitStart := time.Now()
for { for {
myIndexes := myControl.GetHostmapIndexCount() myIndexes := len(myControl.GetHostmap().Indexes)
theirIndexes := theirControl.GetHostmapIndexCount() theirIndexes := len(theirControl.GetHostmap().Indexes)
if myIndexes == 0 && theirIndexes == 0 { if myIndexes == 0 && theirIndexes == 0 {
break break
} }
@@ -493,8 +493,8 @@ func TestCloseTunnelAuthenticated(t *testing.T) {
waitStart := time.Now() waitStart := time.Now()
for { for {
myIndexes := myControl.GetHostmapIndexCount() myIndexes := len(myControl.GetHostmap().Indexes)
theirIndexes := theirControl.GetHostmapIndexCount() theirIndexes := len(theirControl.GetHostmap().Indexes)
if myIndexes == 0 && theirIndexes == 0 { if myIndexes == 0 && theirIndexes == 0 {
break break
} }
@@ -548,8 +548,8 @@ func TestCloseTunnelAuthenticated(t *testing.T) {
r.Log("Injected bogus close tunnel. Let's see!") r.Log("Injected bogus close tunnel. Let's see!")
waitStart = time.Now() waitStart = time.Now()
for { for {
myIndexes := myControl.GetHostmapIndexCount() myIndexes := len(myControl.GetHostmap().Indexes)
theirIndexes := theirControl.GetHostmapIndexCount() theirIndexes := len(theirControl.GetHostmap().Indexes)
if myIndexes == 0 { if myIndexes == 0 {
t.Fatal("myIndexes should not be 0") t.Fatal("myIndexes should not be 0")
} }
+4 -8
View File
@@ -146,14 +146,6 @@ listen:
# Default true; set to false to leave WDF in charge of inbound decisions on the listener port. Not reloadable. # Default true; set to false to leave WDF in charge of inbound decisions on the listener port. Not reloadable.
#windows_bypass_wdf: true #windows_bypass_wdf: true
# On macOS only
# macOS scopes the udp socket to the interface it was created on, so moving between networks (wifi to wired,
# office to home) leaves Nebula sending out an interface that no longer has a route. When true, Nebula watches
# the routing socket and rebinds the listener once the change settles.
# iOS does not use this, the host app drives the same rebind itself.
# Default true. Not reloadable.
#rebind_on_network_change: true
# By default, Nebula replies to packets it has no tunnel for with a "recv_error" packet. This packet helps speed up reconnection # By default, Nebula replies to packets it has no tunnel for with a "recv_error" packet. This packet helps speed up reconnection
# in the case that Nebula on either side did not shut down cleanly. This response can be abused as a way to discover if Nebula is running # in the case that Nebula on either side did not shut down cleanly. This response can be abused as a way to discover if Nebula is running
# on a host though. This option lets you configure if you want to send "recv_error" packets always, never, or only to private network remotes. # on a host though. This option lets you configure if you want to send "recv_error" packets always, never, or only to private network remotes.
@@ -250,6 +242,10 @@ tun:
# When tun is disabled, a lighthouse can be started without a local tun interface (and therefore without root) # When tun is disabled, a lighthouse can be started without a local tun interface (and therefore without root)
disabled: false disabled: false
# Name of the device. If not set, a default will be chosen by the OS. # Name of the device. If not set, a default will be chosen by the OS.
# For Linux: a single `%d` anywhere in the name is treated as a template and replaced with the
# lowest number that yields an unused device name (e.g. `nebula%d` becomes `nebula0`, then `nebula1`, and so on, `neb%dprod` becomes `neb0prod`).
# Only on Linux: `nebula%d` is the default if tun.dev is unset.
# The name, both before and after %d substitution, must be shorter than the kernel limit of 16 characters.
# For macOS: if set, must be in the form `utun[0-9]+`. # For macOS: if set, must be in the form `utun[0-9]+`.
# For NetBSD: Required to be set, must be in the form `tun[0-9]+` # For NetBSD: Required to be set, must be in the form `tun[0-9]+`
dev: nebula1 dev: nebula1
-9
View File
@@ -8,15 +8,6 @@ Before=sshd.service
Type=notify Type=notify
NotifyAccess=main NotifyAccess=main
SyslogIdentifier=nebula SyslogIdentifier=nebula
# Uncomment to run as an unprivileged user with only CAP_NET_ADMIN. Requires a
# nebula user that owns the config directory. Add CAP_NET_BIND_SERVICE to both
# lines if any listener (lighthouse DNS, listen.port, stats, sshd) binds <1024.
#User=nebula
#Group=nebula
#CapabilityBoundingSet=CAP_NET_ADMIN
#AmbientCapabilities=CAP_NET_ADMIN
ExecReload=/bin/kill -HUP $MAINPID ExecReload=/bin/kill -HUP $MAINPID
ExecStart=/usr/local/bin/nebula -config /etc/nebula/config.yml ExecStart=/usr/local/bin/nebula -config /etc/nebula/config.yml
Restart=always Restart=always
+8 -8
View File
@@ -44,8 +44,8 @@ type Firewall struct {
InRules *FirewallTable InRules *FirewallTable
OutRules *FirewallTable OutRules *FirewallTable
InboundSendReject bool InSendReject bool
OutboundSendReject bool OutSendReject bool
//TODO: we should have many more options for TCP, an option for ICMP, and mimic the kernel a bit better //TODO: we should have many more options for TCP, an option for ICMP, and mimic the kernel a bit better
// https://www.kernel.org/doc/Documentation/networking/nf_conntrack-sysctl.txt // https://www.kernel.org/doc/Documentation/networking/nf_conntrack-sysctl.txt
@@ -216,23 +216,23 @@ func NewFirewallFromConfig(l *slog.Logger, cs *CertState, c *config.C) (*Firewal
inboundAction := c.GetString("firewall.inbound_action", "drop") inboundAction := c.GetString("firewall.inbound_action", "drop")
switch inboundAction { switch inboundAction {
case "reject": case "reject":
fw.InboundSendReject = true fw.InSendReject = true
case "drop": case "drop":
fw.InboundSendReject = false fw.InSendReject = false
default: default:
l.Warn("invalid firewall.inbound_action, defaulting to `drop`", "action", inboundAction) l.Warn("invalid firewall.inbound_action, defaulting to `drop`", "action", inboundAction)
fw.InboundSendReject = false fw.InSendReject = false
} }
outboundAction := c.GetString("firewall.outbound_action", "drop") outboundAction := c.GetString("firewall.outbound_action", "drop")
switch outboundAction { switch outboundAction {
case "reject": case "reject":
fw.OutboundSendReject = true fw.OutSendReject = true
case "drop": case "drop":
fw.OutboundSendReject = false fw.OutSendReject = false
default: default:
l.Warn("invalid firewall.outbound_action, defaulting to `drop`", "action", outboundAction) l.Warn("invalid firewall.outbound_action, defaulting to `drop`", "action", outboundAction)
fw.OutboundSendReject = false fw.OutSendReject = false
} }
err := AddFirewallRulesFromConfig(l, false, c, fw) err := AddFirewallRulesFromConfig(l, false, c, fw)
+8 -7
View File
@@ -1,13 +1,13 @@
module github.com/slackhq/nebula module github.com/slackhq/nebula
go 1.26.0 go 1.25.0
require ( require (
dario.cat/mergo v1.0.2 dario.cat/mergo v1.0.2
filippo.io/bigmod v0.1.0 filippo.io/bigmod v0.1.0
github.com/DefinedNet/netlink v1.3.2-dn.1
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
github.com/armon/go-radix v1.0.0 github.com/armon/go-radix v1.0.0
github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432
github.com/flynn/noise v1.1.0 github.com/flynn/noise v1.1.0
github.com/gaissmai/bart v0.28.0 github.com/gaissmai/bart v0.28.0
github.com/gogo/protobuf v1.3.2 github.com/gogo/protobuf v1.3.2
@@ -21,14 +21,15 @@ require (
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
github.com/stretchr/testify v1.11.1 github.com/stretchr/testify v1.11.1
github.com/vishvananda/netlink v1.3.1
go.uber.org/goleak v1.3.0 go.uber.org/goleak v1.3.0
go.yaml.in/yaml/v3 v3.0.4 go.yaml.in/yaml/v3 v3.0.4
golang.org/x/crypto v0.54.0 golang.org/x/crypto v0.53.0
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
golang.org/x/net v0.57.0 golang.org/x/net v0.56.0
golang.org/x/sync v0.22.0 golang.org/x/sync v0.21.0
golang.org/x/sys v0.47.0 golang.org/x/sys v0.46.0
golang.org/x/term v0.45.0 golang.org/x/term v0.44.0
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
golang.zx2c4.com/wireguard/windows v1.0.1 golang.zx2c4.com/wireguard/windows v1.0.1
+16 -12
View File
@@ -3,8 +3,6 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
filippo.io/bigmod v0.1.0 h1:UNzDk7y9ADKST+axd9skUpBQeW7fG2KrTZyOE4uGQy8= filippo.io/bigmod v0.1.0 h1:UNzDk7y9ADKST+axd9skUpBQeW7fG2KrTZyOE4uGQy8=
filippo.io/bigmod v0.1.0/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI= filippo.io/bigmod v0.1.0/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI=
github.com/DefinedNet/netlink v1.3.2-dn.1 h1:Md2PsUE1Xuy1Zxs/Bm6356n1kt5bq2HspXlPJ6pEspg=
github.com/DefinedNet/netlink v1.3.2-dn.1/go.mod h1:o4YSn8Jw7zoDcYXDmV77TRd4eS5sSLuE+vu2zAN67C8=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -21,6 +19,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432 h1:M5QgkYacWj0Xs8MhpIK/5uwU02icXpEoSo9sM2aRCps=
github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432/go.mod h1:xwIwAxMvYnVrGJPe2FKx5prTrnAjGOD8zvDOnxnrrkM=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -145,6 +145,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -160,8 +162,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= golang.org/x/crypto v0.53.0 h1:QZ4Muo8THX6CizN2vPPd5fBGHyogrdK9fG4wLPFUsto=
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= golang.org/x/crypto v0.53.0/go.mod h1:DNLU434OwVakk9PzuwV8w62mAJpRJL3vsgcfp4Qnsio=
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY= golang.org/x/exp v0.0.0-20230725093048-515e97ebf090 h1:Di6/M8l0O2lCLc6VVRWhgCiApHV8MnQurBnFSHsQtNY=
golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/exp v0.0.0-20230725093048-515e97ebf090/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
@@ -180,8 +182,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -189,8 +191,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -204,11 +206,13 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw=
golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= golang.org/x/term v0.44.0 h1:0rLvDRCtNj0gZkyIXhCyOb2OAzEhLVqc4B+hrsBhrmc=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= golang.org/x/term v0.44.0/go.mod h1:7ze4MdzUzLXpSAoFP1H0bOI9aXDqveSvatT5vKcFh2Y=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-117
View File
@@ -1,117 +0,0 @@
package nebula
// This file is a trimmed, inlined copy of the graphite exporter from
// github.com/cyberdelia/go-metrics-graphite, retaining only the Config type and
// the Once entrypoint that Nebula uses. The upstream package has been
// unmaintained for 10+ years, so it was vendored here to drop the dependency.
// See https://github.com/slackhq/nebula/issues/1831.
//
// Copyright 2015 Timothée Peignier. All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice,
// this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
import (
"bufio"
"fmt"
"net"
"strconv"
"strings"
"time"
"github.com/rcrowley/go-metrics"
)
// graphiteConfigExport provides a container with configuration parameters for
// the Graphite exporter.
type graphiteConfigExport struct {
Addr *net.TCPAddr // Network address to connect to
Registry metrics.Registry // Registry to be exported
FlushInterval time.Duration // Flush interval
DurationUnit time.Duration // Time conversion unit for durations
Prefix string // Prefix to be prepended to metric names
Percentiles []float64 // Percentiles to export from timers and histograms
}
// graphiteOnce performs a single submission to Graphite, returning a non-nil
// error on failed connections.
func graphiteOnce(c graphiteConfigExport) error {
now := time.Now().Unix()
du := float64(c.DurationUnit)
flushSeconds := float64(c.FlushInterval) / float64(time.Second)
conn, err := net.DialTCP("tcp", nil, c.Addr)
if err != nil {
return err
}
defer conn.Close()
w := bufio.NewWriter(conn)
c.Registry.Each(func(name string, i any) {
switch metric := i.(type) {
case metrics.Counter:
count := metric.Count()
fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, count, now)
fmt.Fprintf(w, "%s.%s.count_ps %.2f %d\n", c.Prefix, name, float64(count)/flushSeconds, now)
case metrics.Gauge:
fmt.Fprintf(w, "%s.%s.value %d %d\n", c.Prefix, name, metric.Value(), now)
case metrics.GaugeFloat64:
fmt.Fprintf(w, "%s.%s.value %f %d\n", c.Prefix, name, metric.Value(), now)
case metrics.Histogram:
h := metric.Snapshot()
ps := h.Percentiles(c.Percentiles)
fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, h.Count(), now)
fmt.Fprintf(w, "%s.%s.min %d %d\n", c.Prefix, name, h.Min(), now)
fmt.Fprintf(w, "%s.%s.max %d %d\n", c.Prefix, name, h.Max(), now)
fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, h.Mean(), now)
fmt.Fprintf(w, "%s.%s.std-dev %.2f %d\n", c.Prefix, name, h.StdDev(), now)
for psIdx, psKey := range c.Percentiles {
key := strings.Replace(strconv.FormatFloat(psKey*100.0, 'f', -1, 64), ".", "", 1)
fmt.Fprintf(w, "%s.%s.%s-percentile %.2f %d\n", c.Prefix, name, key, ps[psIdx], now)
}
case metrics.Meter:
m := metric.Snapshot()
fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, m.Count(), now)
fmt.Fprintf(w, "%s.%s.one-minute %.2f %d\n", c.Prefix, name, m.Rate1(), now)
fmt.Fprintf(w, "%s.%s.five-minute %.2f %d\n", c.Prefix, name, m.Rate5(), now)
fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, m.Rate15(), now)
fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, m.RateMean(), now)
case metrics.Timer:
t := metric.Snapshot()
ps := t.Percentiles(c.Percentiles)
count := t.Count()
fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, count, now)
fmt.Fprintf(w, "%s.%s.count_ps %.2f %d\n", c.Prefix, name, float64(count)/flushSeconds, now)
fmt.Fprintf(w, "%s.%s.min %d %d\n", c.Prefix, name, t.Min()/int64(du), now)
fmt.Fprintf(w, "%s.%s.max %d %d\n", c.Prefix, name, t.Max()/int64(du), now)
fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, t.Mean()/du, now)
fmt.Fprintf(w, "%s.%s.std-dev %.2f %d\n", c.Prefix, name, t.StdDev()/du, now)
for psIdx, psKey := range c.Percentiles {
key := strings.Replace(strconv.FormatFloat(psKey*100.0, 'f', -1, 64), ".", "", 1)
fmt.Fprintf(w, "%s.%s.%s-percentile %.2f %d\n", c.Prefix, name, key, ps[psIdx]/du, now)
}
fmt.Fprintf(w, "%s.%s.one-minute %.2f %d\n", c.Prefix, name, t.Rate1(), now)
fmt.Fprintf(w, "%s.%s.five-minute %.2f %d\n", c.Prefix, name, t.Rate5(), now)
fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, t.Rate15(), now)
fmt.Fprintf(w, "%s.%s.mean-rate %.2f %d\n", c.Prefix, name, t.RateMean(), now)
}
w.Flush()
})
return nil
}
+10 -27
View File
@@ -295,13 +295,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
hm.messageMetrics.Tx(header.Handshake, hh.machine.Subtype(), 1) hm.messageMetrics.Tx(header.Handshake, hh.machine.Subtype(), 1)
err := hm.outside.WriteTo(stage0, addr) err := hm.outside.WriteTo(stage0, addr)
if err != nil { if err != nil {
// These repeat every attempt, so match the success log below and only shout when the remotes changed hostinfo.logger(hm.l).Error("Failed to send handshake message",
level := slog.LevelDebug
if remotesHaveChanged {
level = slog.LevelError
}
hostinfo.logger(hm.l).Log(context.Background(), level, "Failed to send handshake message",
"udpAddr", addr, "udpAddr", addr,
"initiatorIndex", hostinfo.localIndexId, "initiatorIndex", hostinfo.localIndexId,
"handshake", hsFields, "handshake", hsFields,
@@ -436,11 +430,14 @@ func (hm *HandshakeManager) CheckAndComplete(hostinfo *HostInfo, handshakePacket
// Check if we already have a tunnel with this vpn ip // Check if we already have a tunnel with this vpn ip
existingHostInfo, found := hm.mainHostMap.Hosts[hostinfo.vpnAddrs[0]] existingHostInfo, found := hm.mainHostMap.Hosts[hostinfo.vpnAddrs[0]]
if found && existingHostInfo != nil { if found && existingHostInfo != nil {
// Is it just a delayed handshake packet? Check every hostinfo we hold for this address. testHostInfo := existingHostInfo
for _, testHostInfo := range hm.mainHostMap.unlockedGetHostList(hostinfo.vpnAddrs[0]) { for testHostInfo != nil {
// Is it just a delayed handshake packet?
if bytes.Equal(hostinfo.HandshakePacket[handshakePacket], testHostInfo.HandshakePacket[handshakePacket]) { if bytes.Equal(hostinfo.HandshakePacket[handshakePacket], testHostInfo.HandshakePacket[handshakePacket]) {
return testHostInfo, ErrAlreadySeen return testHostInfo, ErrAlreadySeen
} }
testHostInfo = testHostInfo.next
} }
// Is this a newer handshake? // Is this a newer handshake?
@@ -535,9 +532,7 @@ func (hm *HandshakeManager) DeleteHostInfo(hostinfo *HostInfo) {
func (hm *HandshakeManager) unlockedDeleteHostInfo(hostinfo *HostInfo) { func (hm *HandshakeManager) unlockedDeleteHostInfo(hostinfo *HostInfo) {
for _, addr := range hostinfo.vpnAddrs { for _, addr := range hostinfo.vpnAddrs {
if cur, ok := hm.vpnIps[addr]; ok && cur.hostinfo == hostinfo { delete(hm.vpnIps, addr)
delete(hm.vpnIps, addr)
}
} }
if len(hm.vpnIps) == 0 { if len(hm.vpnIps) == 0 {
@@ -749,14 +744,8 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
return return
} }
connState, err := newConnectionStateFromResult(result)
if err != nil {
f.l.Error("Discarding handshake with an invalid message index", "error", err, "vpnAddrs", vpnAddrs)
return
}
hostinfo := &HostInfo{ hostinfo := &HostInfo{
ConnectionState: connState, ConnectionState: newConnectionStateFromResult(result),
localIndexId: result.LocalIndex, localIndexId: result.LocalIndex,
remoteIndexId: result.RemoteIndex, remoteIndexId: result.RemoteIndex,
vpnAddrs: vpnAddrs, vpnAddrs: vpnAddrs,
@@ -874,13 +863,7 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
} }
// Handshake complete; build the ConnectionState now that we have keys and a verified peer cert. // Handshake complete; build the ConnectionState now that we have keys and a verified peer cert.
cs, err := newConnectionStateFromResult(result) hostinfo.ConnectionState = newConnectionStateFromResult(result)
if err != nil {
f.l.Error("Discarding handshake with an invalid message index", "error", err, "vpnAddrs", hostinfo.vpnAddrs)
hm.DeleteHostInfo(hostinfo)
return
}
hostinfo.ConnectionState = cs
remoteCert := result.RemoteCert remoteCert := result.RemoteCert
if remoteCert == nil { if remoteCert == nil {
@@ -1097,7 +1080,7 @@ func (hm *HandshakeManager) sendHandshakeResponse(via ViaSender, msg []byte, hos
hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0]) hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
// We received a valid handshake on this relay, so make sure the relay // We received a valid handshake on this relay, so make sure the relay
// state reflects that, in case it had been marked Disestablished. // state reflects that, in case it had been marked Disestablished.
via.relayHI.relayState.UpdateRelayForByIdxState(via.relay.LocalIndex, Established) via.relayHI.relayState.UpdateRelayForByIdxState(via.remoteIdx, Established)
f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false) f.SendVia(via.relayHI, via.relay, msg, make([]byte, 12), make([]byte, mtu), false)
f.l.Info("Handshake message sent", append(logFields, "relay", via.relayHI.vpnAddrs[0])...) f.l.Info("Handshake message sent", append(logFields, "relay", via.relayHI.vpnAddrs[0])...)
} }
+155 -151
View File
@@ -6,6 +6,7 @@ import (
"errors" "errors"
"fmt" "fmt"
"log/slog" "log/slog"
"net"
"net/netip" "net/netip"
"slices" "slices"
"sync" "sync"
@@ -17,6 +18,7 @@ import (
"github.com/slackhq/nebula/cert" "github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/logging"
) )
const defaultPromoteEvery = 1000 // Count of packets sent before we try moving a tunnel to a preferred underlay ip address const defaultPromoteEvery = 1000 // Count of packets sent before we try moving a tunnel to a preferred underlay ip address
@@ -54,20 +56,11 @@ type Relay struct {
} }
type HostMap struct { type HostMap struct {
sync.RWMutex //Because we concurrently read and write to our maps sync.RWMutex //Because we concurrently read and write to our maps
Indexes map[uint32]*HostInfo Indexes map[uint32]*HostInfo
Relays map[uint32]*HostInfo // Maps a Relay IDX to a Relay HostInfo object Relays map[uint32]*HostInfo // Maps a Relay IDX to a Relay HostInfo object
RemoteIndexes map[uint32]*HostInfo RemoteIndexes map[uint32]*HostInfo
// Hosts maps a vpn address to its primary hostinfo, one entry per address we hold a tunnel
// for. moreHosts only has an entry while an address is held by 2 or more hostinfos and stores
// the full most-recent-first list; moreHosts[a][0] is always the same hostinfo as Hosts[a].
// Each address gets its own independent list, so a hostinfo owning multiple addresses can
// never corrupt another address's ordering the way the old shared next/prev chain could.
// Entries in moreHosts are only ever written by unlockedSetHostsForAddr; Hosts is written
// directly only in the single-hostinfo fast paths where moreHosts is known to have no entry,
// and unlockedDeleteHostInfo swaps either map for a fresh one when it fully drains.
Hosts map[netip.Addr]*HostInfo Hosts map[netip.Addr]*HostInfo
moreHosts map[netip.Addr][]*HostInfo
preferredRanges atomic.Pointer[[]netip.Prefix] preferredRanges atomic.Pointer[[]netip.Prefix]
l *slog.Logger l *slog.Logger
} }
@@ -273,6 +266,10 @@ type HostInfo struct {
lastRoam time.Time lastRoam time.Time
lastRoamRemote netip.AddrPort lastRoamRemote netip.AddrPort
// Used to track other hostinfos for this vpn ip since only 1 can be primary
// Synchronised via hostmap lock and not the hostinfo lock.
next, prev *HostInfo
//TODO: in, out, and others might benefit from being an atomic.Int32. We could collapse connectionManager pendingDeletion, relayUsed, and in/out into this 1 thing //TODO: in, out, and others might benefit from being an atomic.Int32. We could collapse connectionManager pendingDeletion, relayUsed, and in/out into this 1 thing
in, out, pendingDeletion atomic.Bool in, out, pendingDeletion atomic.Bool
@@ -285,6 +282,7 @@ type HostInfo struct {
type ViaSender struct { type ViaSender struct {
UdpAddr netip.AddrPort UdpAddr netip.AddrPort
relayHI *HostInfo // relayHI is the host info object of the relay relayHI *HostInfo // relayHI is the host info object of the relay
remoteIdx uint32 // remoteIdx is the index included in the header of the received packet
relay *Relay // relay contains the rest of the relay information, including the PeerIP of the host trying to communicate with us. relay *Relay // relay contains the rest of the relay information, including the PeerIP of the host trying to communicate with us.
IsRelayed bool // IsRelayed is true if the packet was sent through a relay IsRelayed bool // IsRelayed is true if the packet was sent through a relay
} }
@@ -336,7 +334,6 @@ func newHostMap(l *slog.Logger) *HostMap {
Relays: map[uint32]*HostInfo{}, Relays: map[uint32]*HostInfo{},
RemoteIndexes: map[uint32]*HostInfo{}, RemoteIndexes: map[uint32]*HostInfo{},
Hosts: map[netip.Addr]*HostInfo{}, Hosts: map[netip.Addr]*HostInfo{},
moreHosts: map[netip.Addr][]*HostInfo{},
l: l, l: l,
} }
} }
@@ -385,55 +382,13 @@ func (hm *HostMap) EmitStats() {
metrics.GetOrRegisterGauge("hostmap.main.relayIndexes", nil).Update(int64(relaysLen)) metrics.GetOrRegisterGauge("hostmap.main.relayIndexes", nil).Update(int64(relaysLen))
} }
// unlockedSetHostsForAddr stores the per-address hostinfo list (list[0] is the primary). An empty // DeleteHostInfo will fully unlink the hostinfo and return true if it was the final hostinfo for this vpn ip
// list removes the address. This is the one place Hosts and moreHosts are written together, keep
// it that way. Callers must hold the write lock.
func (hm *HostMap) unlockedSetHostsForAddr(addr netip.Addr, list []*HostInfo) {
if len(list) == 0 {
delete(hm.Hosts, addr)
delete(hm.moreHosts, addr)
return
}
hm.Hosts[addr] = list[0]
if len(list) > 1 {
hm.moreHosts[addr] = list
} else {
delete(hm.moreHosts, addr)
}
}
// unlockedGetHostList returns every hostinfo holding addr, primary first, or nil if we have no
// tunnel for addr. The common single-hostinfo case builds a fresh one element list, so keep this
// off the packet hot path; the primary is a direct Hosts read. Callers must hold the lock (read
// or write).
func (hm *HostMap) unlockedGetHostList(addr netip.Addr) []*HostInfo {
if list, ok := hm.moreHosts[addr]; ok {
return list
}
if h, ok := hm.Hosts[addr]; ok {
return []*HostInfo{h}
}
return nil
}
// removeHostInfo returns list with hi removed (order preserved), or list unchanged if hi is
// absent. It deletes in place: every mutator holds the hostmap write lock and no reader ever
// retains a slice across a mutation (readers iterate under RLock), so there is no snapshot to
// invalidate.
func removeHostInfo(list []*HostInfo, hi *HostInfo) []*HostInfo {
idx := slices.Index(list, hi)
if idx < 0 {
return list
}
return slices.Delete(list, idx, idx+1)
}
// DeleteHostInfo will fully unlink the hostinfo and return true if no other hostinfo still holds
// any of its vpn addrs, meaning we no longer have a tunnel to the peer
func (hm *HostMap) DeleteHostInfo(hostinfo *HostInfo) bool { func (hm *HostMap) DeleteHostInfo(hostinfo *HostInfo) bool {
// Delete the host itself, ensuring it's not modified anymore // Delete the host itself, ensuring it's not modified anymore
hm.Lock() hm.Lock()
final := hm.unlockedDeleteHostInfo(hostinfo) // If we have a previous or next hostinfo then we are not the last one for this vpn ip
final := (hostinfo.next == nil && hostinfo.prev == nil)
hm.unlockedDeleteHostInfo(hostinfo)
hm.Unlock() hm.Unlock()
return final return final
@@ -445,66 +400,71 @@ func (hm *HostMap) MakePrimary(hostinfo *HostInfo) {
hm.unlockedMakePrimary(hostinfo) hm.unlockedMakePrimary(hostinfo)
} }
// unlockedMakePrimary reports whether hostinfo is (now) the primary for each of its addresses, func (hm *HostMap) unlockedMakePrimary(hostinfo *HostInfo) {
// false only when it is no longer in the hostmap at all. // Get the current primary, if it exists
func (hm *HostMap) unlockedMakePrimary(hostinfo *HostInfo) bool { oldHostinfo := hm.Hosts[hostinfo.vpnAddrs[0]]
// A hostinfo that is no longer in the hostmap must not be re-inserted here. Callers can race
// tunnel teardown, deciding to promote under the read lock and only taking the write lock // Every address in the hostinfo gets elevated to primary
// after a delete fully unlinked the hostinfo (connection manager swapPrimary, AddRelay). Every for _, vpnAddr := range hostinfo.vpnAddrs {
// live hostinfo is registered in Indexes by unlockedAddHostInfo, so this is a membership test. //NOTE: It is possible that we leave a dangling hostinfo here but connection manager works on
if hm.Indexes[hostinfo.localIndexId] != hostinfo { // indexes so it should be fine.
return false hm.Hosts[vpnAddr] = hostinfo
} }
// Move hostinfo to the front (primary) of each of its address lists. The lists are // If we are already primary then we won't bother re-linking
// independent per address, so this can never leave a dangling entry the way promoting if oldHostinfo == hostinfo {
// against a single shared chain could. return
for _, addr := range hostinfo.vpnAddrs {
if hm.Hosts[addr] == hostinfo {
// Already primary for this address, the list is already in the right order
continue
}
list := removeHostInfo(hm.unlockedGetHostList(addr), hostinfo)
list = append([]*HostInfo{hostinfo}, list...)
hm.unlockedSetHostsForAddr(addr, list)
} }
return true
// Unlink this hostinfo
if hostinfo.prev != nil {
hostinfo.prev.next = hostinfo.next
}
if hostinfo.next != nil {
hostinfo.next.prev = hostinfo.prev
}
// If there wasn't a previous primary then clear out any links
if oldHostinfo == nil {
hostinfo.next = nil
hostinfo.prev = nil
return
}
// Relink the hostinfo as primary
hostinfo.next = oldHostinfo
oldHostinfo.prev = hostinfo
hostinfo.prev = nil
} }
// unlockedDeleteHostInfo removes hostinfo from every one of its address lists and from the index func (hm *HostMap) unlockedDeleteHostInfo(hostinfo *HostInfo) {
// maps. It returns true if this was the last hostinfo for all of its addresses (we no longer have isLastHostinfo := hostinfo.next == nil && hostinfo.prev == nil
// any tunnel to the peer), which the caller uses to decide whether to clear learned lighthouse
// state and disestablish relays.
func (hm *HostMap) unlockedDeleteHostInfo(hostinfo *HostInfo) bool {
// Remove this hostinfo from each of its address lists. The lists are independent, so a
// sibling is never promoted to an address it does not own and no other list is touched.
final := true
for _, addr := range hostinfo.vpnAddrs { for _, addr := range hostinfo.vpnAddrs {
if list, ok := hm.moreHosts[addr]; ok { if hm.Hosts[addr] != hostinfo {
list = removeHostInfo(list, hostinfo) continue
hm.unlockedSetHostsForAddr(addr, list) }
if len(list) > 0 { if hostinfo.next != nil {
final = false // Promote the next hostinfo in the shared chain to primary for this address
} hm.Hosts[addr] = hostinfo.next
} else if existing, ok := hm.Hosts[addr]; ok { } else {
if existing == hostinfo { delete(hm.Hosts, addr)
// Common case, the only hostinfo for this address. moreHosts has no entry to clean up.
delete(hm.Hosts, addr)
} else {
// We don't hold this address but another hostinfo does, we still have a tunnel to the peer
final = false
}
} }
} }
// Go maps never shrink their buckets, replace fully drained maps so a node that churned
// through a large peer count gives the memory back. Same idiom as the index maps below.
if len(hm.Hosts) == 0 { if len(hm.Hosts) == 0 {
hm.Hosts = map[netip.Addr]*HostInfo{} hm.Hosts = map[netip.Addr]*HostInfo{}
} }
if len(hm.moreHosts) == 0 {
hm.moreHosts = map[netip.Addr][]*HostInfo{} // Splice this hostinfo out of the shared chain exactly once
if hostinfo.prev != nil {
hostinfo.prev.next = hostinfo.next
} }
if hostinfo.next != nil {
hostinfo.next.prev = hostinfo.prev
}
hostinfo.next = nil
hostinfo.prev = nil
// The remote index uses index ids outside our control so lets make sure we are only removing // The remote index uses index ids outside our control so lets make sure we are only removing
// the remote index pointer here if it points to the hostinfo we are deleting // the remote index pointer here if it points to the hostinfo we are deleting
@@ -528,7 +488,7 @@ func (hm *HostMap) unlockedDeleteHostInfo(hostinfo *HostInfo) bool {
) )
} }
if final { if isLastHostinfo {
// I have lost connectivity to my peers. My relay tunnel is likely broken. Mark the next // I have lost connectivity to my peers. My relay tunnel is likely broken. Mark the next
// hops as 'Requested' so that new relay tunnels are created in the future. // hops as 'Requested' so that new relay tunnels are created in the future.
hm.unlockedDisestablishVpnAddrRelayFor(hostinfo) hm.unlockedDisestablishVpnAddrRelayFor(hostinfo)
@@ -537,8 +497,6 @@ func (hm *HostMap) unlockedDeleteHostInfo(hostinfo *HostInfo) bool {
for _, localRelayIdx := range hostinfo.relayState.CopyRelayForIdxs() { for _, localRelayIdx := range hostinfo.relayState.CopyRelayForIdxs() {
delete(hm.Relays, localRelayIdx) delete(hm.Relays, localRelayIdx)
} }
return final
} }
func (hm *HostMap) QueryIndex(index uint32) *HostInfo { func (hm *HostMap) QueryIndex(index uint32) *HostInfo {
@@ -582,30 +540,19 @@ func (hm *HostMap) QueryVpnAddrsRelayFor(targetIps []netip.Addr, relayHostIp net
hm.RLock() hm.RLock()
defer hm.RUnlock() defer hm.RUnlock()
// This runs per relayed packet, so check the primary with a single map probe and only consult
// moreHosts when the primary can't relay for us.
h, ok := hm.Hosts[relayHostIp] h, ok := hm.Hosts[relayHostIp]
if !ok { if !ok {
return nil, nil, errors.New("unable to find host") return nil, nil, errors.New("unable to find host")
} }
for _, targetIp := range targetIps { for h != nil {
r, ok := h.relayState.QueryRelayForByIp(targetIp) for _, targetIp := range targetIps {
if ok && r.State == Established { r, ok := h.relayState.QueryRelayForByIp(targetIp)
return h, r, nil if ok && r.State == Established {
} return h, r, nil
}
if list, ok := hm.moreHosts[relayHostIp]; ok {
// list[0] is the primary we already checked
for _, h := range list[1:] {
for _, targetIp := range targetIps {
r, ok := h.relayState.QueryRelayForByIp(targetIp)
if ok && r.State == Established {
return h, r, nil
}
} }
} }
h = h.next
} }
return nil, nil, errors.New("unable to find host with relay") return nil, nil, errors.New("unable to find host with relay")
@@ -613,14 +560,20 @@ func (hm *HostMap) QueryVpnAddrsRelayFor(targetIps []netip.Addr, relayHostIp net
func (hm *HostMap) unlockedDisestablishVpnAddrRelayFor(hi *HostInfo) { func (hm *HostMap) unlockedDisestablishVpnAddrRelayFor(hi *HostInfo) {
for _, relayHostIp := range hi.relayState.CopyRelayIps() { for _, relayHostIp := range hi.relayState.CopyRelayIps() {
for _, h := range hm.unlockedGetHostList(relayHostIp) { if h, ok := hm.Hosts[relayHostIp]; ok {
h.relayState.UpdateRelayForByIpState(hi.vpnAddrs[0], Disestablished) for h != nil {
h.relayState.UpdateRelayForByIpState(hi.vpnAddrs[0], Disestablished)
h = h.next
}
} }
} }
for _, rs := range hi.relayState.CopyAllRelayFor() { for _, rs := range hi.relayState.CopyAllRelayFor() {
if rs.Type == ForwardingType { if rs.Type == ForwardingType {
for _, h := range hm.unlockedGetHostList(rs.PeerAddr) { if h, ok := hm.Hosts[rs.PeerAddr]; ok {
h.relayState.UpdateRelayForByIpState(hi.vpnAddrs[0], Disestablished) for h != nil {
h.relayState.UpdateRelayForByIpState(hi.vpnAddrs[0], Disestablished)
h = h.next
}
} }
} }
} }
@@ -670,27 +623,22 @@ func (hm *HostMap) unlockedAddHostInfo(hostinfo *HostInfo, f *Interface) {
} }
func (hm *HostMap) unlockedInnerAddHostInfo(vpnAddr netip.Addr, hostinfo *HostInfo, f *Interface) { func (hm *HostMap) unlockedInnerAddHostInfo(vpnAddr netip.Addr, hostinfo *HostInfo, f *Interface) {
existing, ok := hm.Hosts[vpnAddr] existing := hm.Hosts[vpnAddr]
if !ok { hm.Hosts[vpnAddr] = hostinfo
// Common case, the first hostinfo for this address. moreHosts stays empty.
hm.Hosts[vpnAddr] = hostinfo if existing != nil && existing != hostinfo {
return hostinfo.next = existing
existing.prev = hostinfo
} }
// The new hostinfo becomes the primary for this address. Remove any stale copy of it first so i := 1
// we never hold a duplicate, then prepend. check := hostinfo
list, ok := hm.moreHosts[vpnAddr] for check != nil {
if !ok { if i > MaxHostInfosPerVpnIp {
list = []*HostInfo{existing} hm.unlockedDeleteHostInfo(check)
} }
list = removeHostInfo(list, hostinfo) check = check.next
list = append([]*HostInfo{hostinfo}, list...) i++
hm.unlockedSetHostsForAddr(vpnAddr, list)
// Enforce the per-address cap by fully retiring the oldest hostinfo once we exceed it.
// Deleting it removes it from all of its addresses and the index maps, matching prior behavior.
if len(list) > MaxHostInfosPerVpnIp {
hm.unlockedDeleteHostInfo(list[len(list)-1])
} }
} }
@@ -863,3 +811,59 @@ func (i *HostInfo) logger(l *slog.Logger) *slog.Logger {
return li return li
} }
// Utility functions
func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
//FIXME: This function is pretty garbage
var finalAddrs []netip.Addr
ifaces, _ := net.Interfaces()
for _, i := range ifaces {
allow := allowList.AllowName(i.Name)
if l.Enabled(context.Background(), logging.LevelTrace) {
l.Log(context.Background(), logging.LevelTrace, "localAllowList.AllowName",
"interfaceName", i.Name,
"allow", allow,
)
}
if !allow {
continue
}
addrs, _ := i.Addrs()
for _, rawAddr := range addrs {
var addr netip.Addr
switch v := rawAddr.(type) {
case *net.IPNet:
//continue
addr, _ = netip.AddrFromSlice(v.IP)
case *net.IPAddr:
addr, _ = netip.AddrFromSlice(v.IP)
}
if !addr.IsValid() {
if l.Enabled(context.Background(), slog.LevelDebug) {
l.Debug("addr was invalid", "localAddr", rawAddr)
}
continue
}
addr = addr.Unmap()
if addr.IsLoopback() == false && addr.IsLinkLocalUnicast() == false {
isAllowed := allowList.Allow(addr)
if l.Enabled(context.Background(), logging.LevelTrace) {
l.Log(context.Background(), logging.LevelTrace, "localAllowList.Allow",
"localAddr", addr,
"allowed", isAllowed,
)
}
if !isAllowed {
continue
}
finalAddrs = append(finalAddrs, addr)
}
}
}
return finalAddrs
}
+182 -238
View File
@@ -2,7 +2,6 @@ package nebula
import ( import (
"net/netip" "net/netip"
"slices"
"testing" "testing"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
@@ -11,84 +10,78 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
// chainIds returns the localIndexIds of the hostinfos holding addr, primary (index 0) first. It
// also validates the Hosts/moreHosts sync contract on every call so a mutation that broke it
// fails fast.
func chainIds(t *testing.T, hm *HostMap, addr netip.Addr) []uint32 {
t.Helper()
assertHostMapInvariants(t, hm)
list := hm.unlockedGetHostList(addr)
ids := make([]uint32, len(list))
for i, h := range list {
ids[i] = h.localIndexId
}
return ids
}
// assertHostMapInvariants checks the Hosts/moreHosts contract: moreHosts only holds addresses
// with 2 or more hostinfos, its first entry is always the primary in Hosts, lists never hold
// duplicates, every hostinfo in a list owns the address and is registered in Indexes, and every
// indexed hostinfo is reachable through each of its addresses.
func assertHostMapInvariants(t *testing.T, hm *HostMap) {
t.Helper()
for addr, list := range hm.moreHosts {
require.GreaterOrEqualf(t, len(list), 2, "moreHosts[%s] must hold at least 2 hostinfos", addr)
require.Samef(t, hm.Hosts[addr], list[0], "moreHosts[%s][0] must match the primary in Hosts", addr)
seen := map[*HostInfo]bool{}
for _, h := range list {
require.NotNilf(t, h, "moreHosts[%s] must never hold a nil hostinfo", addr)
require.Falsef(t, seen[h], "moreHosts[%s] holds hostinfo %d twice", addr, h.localIndexId)
seen[h] = true
require.Samef(t, hm.Indexes[h.localIndexId], h, "moreHosts[%s] member %d is not registered in Indexes", addr, h.localIndexId)
require.Truef(t, slices.Contains(h.vpnAddrs, addr), "moreHosts[%s] member %d does not own the address", addr, h.localIndexId)
}
}
for addr, h := range hm.Hosts {
require.NotNilf(t, h, "Hosts[%s] must never be nil", addr)
require.Samef(t, hm.Indexes[h.localIndexId], h, "Hosts[%s] primary %d is not registered in Indexes", addr, h.localIndexId)
require.Truef(t, slices.Contains(h.vpnAddrs, addr), "Hosts[%s] primary (index %d) does not own the address", addr, h.localIndexId)
}
for idx, h := range hm.Indexes {
require.Equalf(t, idx, h.localIndexId, "Indexes[%d] holds hostinfo with localIndexId %d", idx, h.localIndexId)
for _, va := range h.vpnAddrs {
require.Truef(t, slices.Contains(hm.unlockedGetHostList(va), h), "indexed hostinfo %d is missing from the list for %s", idx, va)
}
}
}
func TestHostMap_MakePrimary(t *testing.T) { func TestHostMap_MakePrimary(t *testing.T) {
l := test.NewLogger() l := test.NewLogger()
hm := newHostMap(l) hm := newHostMap(l)
f := &Interface{} f := &Interface{}
a := netip.MustParseAddr("0.0.0.1")
h1 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 1} h1 := &HostInfo{vpnAddrs: []netip.Addr{netip.MustParseAddr("0.0.0.1")}, localIndexId: 1}
h2 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 2} h2 := &HostInfo{vpnAddrs: []netip.Addr{netip.MustParseAddr("0.0.0.1")}, localIndexId: 2}
h3 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 3} h3 := &HostInfo{vpnAddrs: []netip.Addr{netip.MustParseAddr("0.0.0.1")}, localIndexId: 3}
h4 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 4} h4 := &HostInfo{vpnAddrs: []netip.Addr{netip.MustParseAddr("0.0.0.1")}, localIndexId: 4}
hm.unlockedAddHostInfo(h4, f) hm.unlockedAddHostInfo(h4, f)
hm.unlockedAddHostInfo(h3, f) hm.unlockedAddHostInfo(h3, f)
hm.unlockedAddHostInfo(h2, f) hm.unlockedAddHostInfo(h2, f)
hm.unlockedAddHostInfo(h1, f) hm.unlockedAddHostInfo(h1, f)
// Most-recently-added is primary: h1, h2, h3, h4 // Make sure we go h1 -> h2 -> h3 -> h4
assert.Equal(t, []uint32{1, 2, 3, 4}, chainIds(t, hm, a)) prim := hm.QueryVpnAddr(netip.MustParseAddr("0.0.0.1"))
assert.Equal(t, h1, hm.QueryVpnAddr(a)) assert.Equal(t, h1.localIndexId, prim.localIndexId)
assert.Equal(t, h2.localIndexId, prim.next.localIndexId)
assert.Nil(t, prim.prev)
assert.Equal(t, h1.localIndexId, h2.prev.localIndexId)
assert.Equal(t, h3.localIndexId, h2.next.localIndexId)
assert.Equal(t, h2.localIndexId, h3.prev.localIndexId)
assert.Equal(t, h4.localIndexId, h3.next.localIndexId)
assert.Equal(t, h3.localIndexId, h4.prev.localIndexId)
assert.Nil(t, h4.next)
// Swap the middle to primary: h3, h1, h2, h4 // Swap h3/middle to primary
hm.MakePrimary(h3) hm.MakePrimary(h3)
assert.Equal(t, []uint32{3, 1, 2, 4}, chainIds(t, hm, a))
assert.Equal(t, h3, hm.QueryVpnAddr(a))
// Swap the tail to primary: h4, h3, h1, h2 // Make sure we go h3 -> h1 -> h2 -> h4
hm.MakePrimary(h4) prim = hm.QueryVpnAddr(netip.MustParseAddr("0.0.0.1"))
assert.Equal(t, []uint32{4, 3, 1, 2}, chainIds(t, hm, a)) assert.Equal(t, h3.localIndexId, prim.localIndexId)
assert.Equal(t, h1.localIndexId, prim.next.localIndexId)
assert.Nil(t, prim.prev)
assert.Equal(t, h2.localIndexId, h1.next.localIndexId)
assert.Equal(t, h3.localIndexId, h1.prev.localIndexId)
assert.Equal(t, h4.localIndexId, h2.next.localIndexId)
assert.Equal(t, h1.localIndexId, h2.prev.localIndexId)
assert.Equal(t, h2.localIndexId, h4.prev.localIndexId)
assert.Nil(t, h4.next)
// Swapping the current primary again is a no-op // Swap h4/tail to primary
hm.MakePrimary(h4) hm.MakePrimary(h4)
assert.Equal(t, []uint32{4, 3, 1, 2}, chainIds(t, hm, a))
// Make sure we go h4 -> h3 -> h1 -> h2
prim = hm.QueryVpnAddr(netip.MustParseAddr("0.0.0.1"))
assert.Equal(t, h4.localIndexId, prim.localIndexId)
assert.Equal(t, h3.localIndexId, prim.next.localIndexId)
assert.Nil(t, prim.prev)
assert.Equal(t, h1.localIndexId, h3.next.localIndexId)
assert.Equal(t, h4.localIndexId, h3.prev.localIndexId)
assert.Equal(t, h2.localIndexId, h1.next.localIndexId)
assert.Equal(t, h3.localIndexId, h1.prev.localIndexId)
assert.Equal(t, h1.localIndexId, h2.prev.localIndexId)
assert.Nil(t, h2.next)
// Swap h4 again should be no-op
hm.MakePrimary(h4)
// Make sure we go h4 -> h3 -> h1 -> h2
prim = hm.QueryVpnAddr(netip.MustParseAddr("0.0.0.1"))
assert.Equal(t, h4.localIndexId, prim.localIndexId)
assert.Equal(t, h3.localIndexId, prim.next.localIndexId)
assert.Nil(t, prim.prev)
assert.Equal(t, h1.localIndexId, h3.next.localIndexId)
assert.Equal(t, h4.localIndexId, h3.prev.localIndexId)
assert.Equal(t, h2.localIndexId, h1.next.localIndexId)
assert.Equal(t, h3.localIndexId, h1.prev.localIndexId)
assert.Equal(t, h1.localIndexId, h2.prev.localIndexId)
assert.Nil(t, h2.next)
} }
func TestHostMap_DeleteHostInfo(t *testing.T) { func TestHostMap_DeleteHostInfo(t *testing.T) {
@@ -96,14 +89,13 @@ func TestHostMap_DeleteHostInfo(t *testing.T) {
hm := newHostMap(l) hm := newHostMap(l)
f := &Interface{} f := &Interface{}
a := netip.MustParseAddr("0.0.0.1")
h1 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 1} h1 := &HostInfo{vpnAddrs: []netip.Addr{netip.MustParseAddr("0.0.0.1")}, localIndexId: 1}
h2 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 2} h2 := &HostInfo{vpnAddrs: []netip.Addr{netip.MustParseAddr("0.0.0.1")}, localIndexId: 2}
h3 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 3} h3 := &HostInfo{vpnAddrs: []netip.Addr{netip.MustParseAddr("0.0.0.1")}, localIndexId: 3}
h4 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 4} h4 := &HostInfo{vpnAddrs: []netip.Addr{netip.MustParseAddr("0.0.0.1")}, localIndexId: 4}
h5 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 5} h5 := &HostInfo{vpnAddrs: []netip.Addr{netip.MustParseAddr("0.0.0.1")}, localIndexId: 5}
h6 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 6} h6 := &HostInfo{vpnAddrs: []netip.Addr{netip.MustParseAddr("0.0.0.1")}, localIndexId: 6}
hm.unlockedAddHostInfo(h6, f) hm.unlockedAddHostInfo(h6, f)
hm.unlockedAddHostInfo(h5, f) hm.unlockedAddHostInfo(h5, f)
@@ -112,110 +104,94 @@ func TestHostMap_DeleteHostInfo(t *testing.T) {
hm.unlockedAddHostInfo(h2, f) hm.unlockedAddHostInfo(h2, f)
hm.unlockedAddHostInfo(h1, f) hm.unlockedAddHostInfo(h1, f)
// h6 is evicted by the MaxHostInfosPerVpnIp cap; the rest are newest-first. // h6 should be deleted
assert.Nil(t, hm.QueryIndex(h6.localIndexId)) assert.Nil(t, h6.next)
assert.Equal(t, []uint32{1, 2, 3, 4, 5}, chainIds(t, hm, a)) assert.Nil(t, h6.prev)
h := hm.QueryIndex(h6.localIndexId)
assert.Nil(t, h)
// Delete primary; not final since siblings remain. // Make sure we go h1 -> h2 -> h3 -> h4 -> h5
assert.False(t, hm.DeleteHostInfo(h1)) prim := hm.QueryVpnAddr(netip.MustParseAddr("0.0.0.1"))
assert.Equal(t, []uint32{2, 3, 4, 5}, chainIds(t, hm, a)) assert.Equal(t, h1.localIndexId, prim.localIndexId)
assert.Equal(t, h2.localIndexId, prim.next.localIndexId)
assert.Nil(t, prim.prev)
assert.Equal(t, h1.localIndexId, h2.prev.localIndexId)
assert.Equal(t, h3.localIndexId, h2.next.localIndexId)
assert.Equal(t, h2.localIndexId, h3.prev.localIndexId)
assert.Equal(t, h4.localIndexId, h3.next.localIndexId)
assert.Equal(t, h3.localIndexId, h4.prev.localIndexId)
assert.Equal(t, h5.localIndexId, h4.next.localIndexId)
assert.Equal(t, h4.localIndexId, h5.prev.localIndexId)
assert.Nil(t, h5.next)
// Deleting the same hostinfo again must not report final while siblings remain and must not // Delete primary
// disturb the list. The old chain code got this wrong: the first delete nil'd next/prev, so a hm.DeleteHostInfo(h1)
// second delete looked final and wiped lighthouse state out from under the live sibling. assert.Nil(t, h1.prev)
assert.False(t, hm.DeleteHostInfo(h1)) assert.Nil(t, h1.next)
assert.Equal(t, []uint32{2, 3, 4, 5}, chainIds(t, hm, a))
// Delete a middle node. // Make sure we go h2 -> h3 -> h4 -> h5
assert.False(t, hm.DeleteHostInfo(h3)) prim = hm.QueryVpnAddr(netip.MustParseAddr("0.0.0.1"))
assert.Equal(t, []uint32{2, 4, 5}, chainIds(t, hm, a)) assert.Equal(t, h2.localIndexId, prim.localIndexId)
assert.Equal(t, h3.localIndexId, prim.next.localIndexId)
assert.Nil(t, prim.prev)
assert.Equal(t, h3.localIndexId, h2.next.localIndexId)
assert.Equal(t, h2.localIndexId, h3.prev.localIndexId)
assert.Equal(t, h4.localIndexId, h3.next.localIndexId)
assert.Equal(t, h3.localIndexId, h4.prev.localIndexId)
assert.Equal(t, h5.localIndexId, h4.next.localIndexId)
assert.Equal(t, h4.localIndexId, h5.prev.localIndexId)
assert.Nil(t, h5.next)
// Delete the tail. // Delete in the middle
assert.False(t, hm.DeleteHostInfo(h5)) hm.DeleteHostInfo(h3)
assert.Equal(t, []uint32{2, 4}, chainIds(t, hm, a)) assert.Nil(t, h3.prev)
assert.Nil(t, h3.next)
// Delete the head; h4 remains and becomes primary. // Make sure we go h2 -> h4 -> h5
assert.False(t, hm.DeleteHostInfo(h2)) prim = hm.QueryVpnAddr(netip.MustParseAddr("0.0.0.1"))
assert.Equal(t, []uint32{4}, chainIds(t, hm, a)) assert.Equal(t, h2.localIndexId, prim.localIndexId)
assert.Equal(t, h4, hm.QueryVpnAddr(a)) assert.Equal(t, h4.localIndexId, prim.next.localIndexId)
assert.Nil(t, prim.prev)
assert.Equal(t, h4.localIndexId, h2.next.localIndexId)
assert.Equal(t, h2.localIndexId, h4.prev.localIndexId)
assert.Equal(t, h5.localIndexId, h4.next.localIndexId)
assert.Equal(t, h4.localIndexId, h5.prev.localIndexId)
assert.Nil(t, h5.next)
// Delete the only remaining item; final is true and the address is gone. // Delete the tail
assert.True(t, hm.DeleteHostInfo(h4)) hm.DeleteHostInfo(h5)
assert.Empty(t, chainIds(t, hm, a)) assert.Nil(t, h5.prev)
assert.Nil(t, hm.QueryVpnAddr(a)) assert.Nil(t, h5.next)
// Deleting an already-gone hostinfo is still final; nothing holds the address anymore. // Make sure we go h2 -> h4
assert.True(t, hm.DeleteHostInfo(h4)) prim = hm.QueryVpnAddr(netip.MustParseAddr("0.0.0.1"))
assert.Empty(t, chainIds(t, hm, a)) assert.Equal(t, h2.localIndexId, prim.localIndexId)
} assert.Equal(t, h4.localIndexId, prim.next.localIndexId)
assert.Nil(t, prim.prev)
assert.Equal(t, h4.localIndexId, h2.next.localIndexId)
assert.Equal(t, h2.localIndexId, h4.prev.localIndexId)
assert.Nil(t, h4.next)
// TestHostMap_MakePrimary_DeletedHostInfo covers promoting a hostinfo that lost a race with // Delete the head
// tunnel teardown: swapPrimary and AddRelay decide to promote while holding a stale pointer and hm.DeleteHostInfo(h2)
// only take the write lock after a delete fully unlinked the hostinfo. MakePrimary must be a assert.Nil(t, h2.prev)
// no-op, not a resurrection that installs an unmanaged primary. assert.Nil(t, h2.next)
func TestHostMap_MakePrimary_DeletedHostInfo(t *testing.T) {
l := test.NewLogger()
hm := newHostMap(l)
f := &Interface{}
a := netip.MustParseAddr("0.0.0.1")
h1 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 1} // Make sure we only have h4
h2 := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 2} prim = hm.QueryVpnAddr(netip.MustParseAddr("0.0.0.1"))
hm.unlockedAddHostInfo(h1, f) assert.Equal(t, h4.localIndexId, prim.localIndexId)
hm.unlockedAddHostInfo(h2, f) assert.Nil(t, prim.prev)
assert.Nil(t, prim.next)
assert.Nil(t, h4.next)
// h1 is fully deleted while another goroutine still holds a pointer to it. // Delete the only item
assert.False(t, hm.DeleteHostInfo(h1)) hm.DeleteHostInfo(h4)
assert.Equal(t, []uint32{2}, chainIds(t, hm, a)) assert.Nil(t, h4.prev)
assert.Nil(t, h4.next)
// The stale promote must not bring it back. // Make sure we have nil
hm.MakePrimary(h1) prim = hm.QueryVpnAddr(netip.MustParseAddr("0.0.0.1"))
assert.Equal(t, []uint32{2}, chainIds(t, hm, a)) assert.Nil(t, prim)
assert.Equal(t, h2, hm.QueryVpnAddr(a))
assert.Nil(t, hm.QueryIndex(h1.localIndexId))
}
// TestHostMap_QueryVpnAddrsRelayFor_NonPrimary makes sure a relay established on an older
// hostinfo is still found after a newer tunnel without relay state takes primary for the same
// address. The lookup checks the primary first and falls back to the rest of the list.
func TestHostMap_QueryVpnAddrsRelayFor_NonPrimary(t *testing.T) {
l := test.NewLogger()
hm := newHostMap(l)
f := &Interface{}
relayAddr := netip.MustParseAddr("0.0.0.9")
target := netip.MustParseAddr("0.0.0.1")
older := &HostInfo{
vpnAddrs: []netip.Addr{relayAddr},
localIndexId: 1,
relayState: RelayState{
relayForByAddr: map[netip.Addr]*Relay{},
relayForByIdx: map[uint32]*Relay{},
},
}
older.relayState.InsertRelay(target, 100, &Relay{Type: ForwardingType, State: Established, LocalIndex: 100, PeerAddr: target})
hm.unlockedAddHostInfo(older, f)
// The relay is found on the primary.
h, r, err := hm.QueryVpnAddrsRelayFor([]netip.Addr{target}, relayAddr)
require.NoError(t, err)
assert.Equal(t, older, h)
assert.Equal(t, uint32(100), r.LocalIndex)
// A re-handshake with no relay state takes primary; the established relay on the older
// hostinfo must still be found through the fallback.
newer := &HostInfo{vpnAddrs: []netip.Addr{relayAddr}, localIndexId: 2}
hm.unlockedAddHostInfo(newer, f)
assert.Equal(t, []uint32{2, 1}, chainIds(t, hm, relayAddr))
h, r, err = hm.QueryVpnAddrsRelayFor([]netip.Addr{target}, relayAddr)
require.NoError(t, err)
assert.Equal(t, older, h)
assert.Equal(t, uint32(100), r.LocalIndex)
// No hostinfo at all is a plain miss.
_, _, err = hm.QueryVpnAddrsRelayFor([]netip.Addr{target}, netip.MustParseAddr("0.0.0.42"))
require.Error(t, err)
} }
// TestHostMap_DeleteHostInfo_MultipleVpnAddrs exercises the case where a hostinfo carries more than one // TestHostMap_DeleteHostInfo_MultipleVpnAddrs exercises the case where a hostinfo carries more than one
@@ -240,82 +216,32 @@ func TestHostMap_DeleteHostInfo_MultipleVpnAddrs(t *testing.T) {
hm.unlockedAddHostInfo(other, f) hm.unlockedAddHostInfo(other, f)
hm.unlockedAddHostInfo(head, f) hm.unlockedAddHostInfo(head, f)
// head is primary for both addresses, other is next in each address's list. // head is primary for both addresses, other is next in the shared chain
assert.Equal(t, head, hm.QueryVpnAddr(a)) assert.Equal(t, head.localIndexId, hm.QueryVpnAddr(a).localIndexId)
assert.Equal(t, head, hm.QueryVpnAddr(b)) assert.Equal(t, head.localIndexId, hm.QueryVpnAddr(b).localIndexId)
assert.Equal(t, []uint32{2, 1}, chainIds(t, hm, a)) assert.Equal(t, other.localIndexId, head.next.localIndexId)
assert.Equal(t, []uint32{2, 1}, chainIds(t, hm, b)) assert.Equal(t, head.localIndexId, other.prev.localIndexId)
// Delete the head. other is still live, so it must become primary for BOTH addresses. // Delete the head. other is still live, so it must become primary for BOTH addresses.
assert.False(t, hm.DeleteHostInfo(head)) hm.DeleteHostInfo(head)
assert.Equal(t, other, hm.QueryVpnAddr(a))
assert.Equal(t, other, hm.QueryVpnAddr(b))
assert.Equal(t, []uint32{1}, chainIds(t, hm, a))
assert.Equal(t, []uint32{1}, chainIds(t, hm, b))
// head is fully removed from the index map. // Pre-fix: QueryVpnAddr(b) came back nil here because the second address was deleted rather than
// promoted, leaving other unreachable at b.
require.NotNil(t, hm.QueryVpnAddr(a))
require.NotNil(t, hm.QueryVpnAddr(b))
assert.Equal(t, other.localIndexId, hm.QueryVpnAddr(a).localIndexId)
assert.Equal(t, other.localIndexId, hm.QueryVpnAddr(b).localIndexId)
// other is now the only hostinfo in the chain
assert.Nil(t, other.prev)
assert.Nil(t, other.next)
// head is fully detached
assert.Nil(t, head.prev)
assert.Nil(t, head.next)
assert.Nil(t, hm.QueryIndex(head.localIndexId)) assert.Nil(t, hm.QueryIndex(head.localIndexId))
} }
// TestHostMap_DeleteHostInfo_DivergentVpnAddrs covers chained hostinfos for the same peer whose
// vpnAddrs sets differ (a re-handshake cert added a second address). Deleting the superset node
// must not promote a sibling to an address it does not own.
func TestHostMap_DeleteHostInfo_DivergentVpnAddrs(t *testing.T) {
l := test.NewLogger()
hm := newHostMap(l)
f := &Interface{}
a := netip.MustParseAddr("0.0.0.1")
b := netip.MustParseAddr("0.0.0.2")
// sub owns only a; super (a newer handshake) owns a and b.
sub := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 1}
super := &HostInfo{vpnAddrs: []netip.Addr{a, b}, localIndexId: 2}
hm.unlockedAddHostInfo(sub, f)
hm.unlockedAddHostInfo(super, f)
assert.Equal(t, []uint32{2, 1}, chainIds(t, hm, a))
assert.Equal(t, []uint32{2}, chainIds(t, hm, b))
// Delete super: a promotes to sub (which owns it); b has no remaining owner and must be
// removed, not dangled at sub (which does not own b).
assert.False(t, hm.DeleteHostInfo(super))
assert.Equal(t, []uint32{1}, chainIds(t, hm, a))
assert.Empty(t, chainIds(t, hm, b))
assert.Equal(t, sub, hm.QueryVpnAddr(a))
assert.Nil(t, hm.QueryVpnAddr(b))
assert.Nil(t, hm.QueryIndex(super.localIndexId))
// Deleting sub cleans up fully.
assert.True(t, hm.DeleteHostInfo(sub))
assert.Nil(t, hm.QueryVpnAddr(a))
assertHostMapInvariants(t, hm)
}
// TestHostMap_AddDivergentOverlap covers a new hostinfo claiming addresses currently owned by two
// DIFFERENT hostinfos. The old single shared next/prev chain overwrote a pointer and orphaned one
// of them (in Indexes but unreachable via its address); independent per-address lists cannot.
func TestHostMap_AddDivergentOverlap(t *testing.T) {
l := test.NewLogger()
hm := newHostMap(l)
f := &Interface{}
a := netip.MustParseAddr("0.0.0.1")
b := netip.MustParseAddr("0.0.0.2")
hiA := &HostInfo{vpnAddrs: []netip.Addr{a}, localIndexId: 1}
hiP := &HostInfo{vpnAddrs: []netip.Addr{b}, localIndexId: 2}
hm.unlockedAddHostInfo(hiA, f)
hm.unlockedAddHostInfo(hiP, f)
hiB := &HostInfo{vpnAddrs: []netip.Addr{a, b}, localIndexId: 3}
hm.unlockedAddHostInfo(hiB, f)
assert.Equal(t, []uint32{3, 1}, chainIds(t, hm, a))
assert.Equal(t, []uint32{3, 2}, chainIds(t, hm, b))
// hiA is still reachable via its address (not orphaned) and still indexed.
assert.Contains(t, chainIds(t, hm, a), hiA.localIndexId)
assert.NotNil(t, hm.QueryIndex(hiA.localIndexId))
}
// TestHostMap_MaxHostInfosPerVpnIp_MultipleVpnAddrs verifies the MaxHostInfosPerVpnIp overflow prune // TestHostMap_MaxHostInfosPerVpnIp_MultipleVpnAddrs verifies the MaxHostInfosPerVpnIp overflow prune
// (unlockedInnerAddHostInfo calls unlockedDeleteHostInfo on the oldest node once the chain is too long) // (unlockedInnerAddHostInfo calls unlockedDeleteHostInfo on the oldest node once the chain is too long)
// still behaves when hostinfos carry more than one vpnAddr. The pruned node is always the tail, so it is // still behaves when hostinfos carry more than one vpnAddr. The pruned node is always the tail, so it is
@@ -341,14 +267,32 @@ func TestHostMap_MaxHostInfosPerVpnIp_MultipleVpnAddrs(t *testing.T) {
oldest := hostinfos[len(hostinfos)-1] oldest := hostinfos[len(hostinfos)-1]
// The oldest hostinfo was pruned from both lists and the index map. // The oldest hostinfo should have been pruned and fully detached
assert.Nil(t, oldest.next)
assert.Nil(t, oldest.prev)
assert.Nil(t, hm.QueryIndex(oldest.localIndexId)) assert.Nil(t, hm.QueryIndex(oldest.localIndexId))
// Both addresses hold exactly MaxHostInfosPerVpnIp survivors in the same order; oldest is absent. // Both addresses resolve to the same head, and that head is one of the survivors (not the pruned one)
require.Len(t, chainIds(t, hm, a), MaxHostInfosPerVpnIp) primA := hm.QueryVpnAddr(a)
assert.Equal(t, chainIds(t, hm, a), chainIds(t, hm, b), "both addresses must list the same survivors in the same order") primB := hm.QueryVpnAddr(b)
assert.NotContains(t, chainIds(t, hm, a), oldest.localIndexId) require.NotNil(t, primA)
assert.Equal(t, hm.QueryVpnAddr(a), hm.QueryVpnAddr(b)) require.NotNil(t, primB)
assert.Equal(t, primA.localIndexId, primB.localIndexId)
assert.NotEqual(t, oldest.localIndexId, primA.localIndexId)
// Walk the shared chain: exactly MaxHostInfosPerVpnIp survivors, no cycles, oldest absent
seen := map[uint32]struct{}{}
for h := primA; h != nil; h = h.next {
_, dup := seen[h.localIndexId]
require.False(t, dup, "cycle detected in hostinfo chain")
seen[h.localIndexId] = struct{}{}
if h.next != nil {
assert.Equal(t, h.localIndexId, h.next.prev.localIndexId, "prev pointer must mirror next")
}
}
assert.Len(t, seen, MaxHostInfosPerVpnIp)
_, prunedStillPresent := seen[oldest.localIndexId]
assert.False(t, prunedStillPresent)
} }
func TestHostMap_reload(t *testing.T) { func TestHostMap_reload(t *testing.T) {
+5 -28
View File
@@ -37,7 +37,6 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
// routes packets from the Nebula addr to the Nebula addr through the Nebula // routes packets from the Nebula addr to the Nebula addr through the Nebula
// TUN device. // TUN device.
if immediatelyForwardToSelf { if immediatelyForwardToSelf {
iputil.SetTransportChecksum(packet)
_, err := f.readers[q].Write(packet) _, err := f.readers[q].Write(packet)
if err != nil { if err != nil {
f.l.Error("Failed to forward to tun", "error", err) f.l.Error("Failed to forward to tun", "error", err)
@@ -88,7 +87,7 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
} }
func (f *Interface) rejectInside(packet []byte, out []byte, q int) { func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
if !f.firewall.OutboundSendReject { if !f.firewall.InSendReject {
return return
} }
@@ -104,7 +103,7 @@ func (f *Interface) rejectInside(packet []byte, out []byte, q int) {
} }
func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *HostInfo, nb, out []byte, q int) { func (f *Interface) rejectOutside(packet []byte, ci *ConnectionState, hostinfo *HostInfo, nb, out []byte, q int) {
if !f.firewall.InboundSendReject { if !f.firewall.OutSendReject {
return return
} }
@@ -276,14 +275,6 @@ func (f *Interface) sendTo(t header.MessageType, st header.MessageSubType, ci *C
f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out, 0) f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out, 0)
} }
// dropExhausted records an exhaustion drop and logs once, on the crossing send, for a spent tunnel.
func (f *Interface) dropExhausted(hostinfo *HostInfo, c uint64, msg string) {
f.messageMetrics.TxExhausted(1)
if c == RejectAfterMessages {
hostinfo.logger(f.l).Error(msg)
}
}
// SendVia sends a payload through a Relay tunnel. No authentication or encryption is done // SendVia sends a payload through a Relay tunnel. No authentication or encryption is done
// to the payload for the ultimate target host, making this a useful method for sending // to the payload for the ultimate target host, making this a useful method for sending
// handshake messages to peers through relay tunnels. // handshake messages to peers through relay tunnels.
@@ -303,14 +294,7 @@ func (f *Interface) SendVia(via *HostInfo,
// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check // NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
via.ConnectionState.writeLock.Lock() via.ConnectionState.writeLock.Lock()
} }
c, ok := via.ConnectionState.NextMessageCounter() c := via.ConnectionState.messageCounter.Add(1)
if !ok {
if noiseutil.EncryptLockNeeded {
via.ConnectionState.writeLock.Unlock()
}
f.dropExhausted(via, c, "Dropping outbound relay packets, tunnel message counter is exhausted")
return
}
out = header.Encode(out, header.Version, header.Message, header.MessageRelay, relay.RemoteIndex, c) out = header.Encode(out, header.Version, header.Message, header.MessageRelay, relay.RemoteIndex, c)
f.connectionManager.Out(via) f.connectionManager.Out(via)
@@ -377,14 +361,7 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check // NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
ci.writeLock.Lock() ci.writeLock.Lock()
} }
c, ok := ci.NextMessageCounter() c := ci.messageCounter.Add(1)
if !ok {
if noiseutil.EncryptLockNeeded {
ci.writeLock.Unlock()
}
f.dropExhausted(hostinfo, c, "Dropping outbound packets, tunnel message counter is exhausted")
return
}
//l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p) //l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p)
out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c) out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
@@ -431,7 +408,7 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
if err != nil { if err != nil {
hostinfo.logger(f.l).Error("Failed to write outgoing packet", hostinfo.logger(f.l).Error("Failed to write outgoing packet",
"error", err, "error", err,
"udpAddr", hr, "udpAddr", remote,
) )
} }
} else { } else {
-263
View File
@@ -1,263 +0,0 @@
package nebula
import (
"encoding/binary"
"io"
"net/netip"
"testing"
"github.com/gaissmai/bart"
"github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const (
ipv4HeaderLen = 20
ipv6HeaderLen = 40
)
// capturingTun is a tio.Queue that records what is written to it. A queue that
// discards writes is indistinguishable from a packet that was never forwarded.
type capturingTun struct {
writes [][]byte
}
func (c *capturingTun) Read([]byte) (int, error) { return 0, io.EOF }
func (c *capturingTun) Close() error { return nil }
func (c *capturingTun) Write(b []byte) (int, error) {
c.writes = append(c.writes, append([]byte(nil), b...))
return len(b), nil
}
func newSelfForwardInterface(myAddrs ...netip.Addr) (*Interface, *capturingTun) {
vpnAddrs := &bart.Lite{}
for _, a := range myAddrs {
vpnAddrs.Insert(netip.PrefixFrom(a, a.BitLen()))
}
tun := &capturingTun{}
return &Interface{
l: test.NewLogger(),
myVpnAddrsTable: vpnAddrs,
myBroadcastAddrsTable: &bart.Lite{},
readers: []io.ReadWriteCloser{tun},
}, tun
}
func consumeInside(f *Interface, packet []byte) {
f.consumeInsidePacket(packet, &firewall.Packet{}, make([]byte, 12), make([]byte, mtu), 0, nil)
}
// l4Proto describes one upper-layer header for these tests: its IP next-header
// value, where its checksum field sits within the header, and how to build a
// minimal instance of it.
type l4Proto struct {
name string
nextHdr uint8
cksumAt int
build func() []byte
}
var (
tcpSyn = l4Proto{"tcp", firewall.ProtoTCP, 16, func() []byte {
h := make([]byte, 20)
binary.BigEndian.PutUint16(h[0:2], 49152)
binary.BigEndian.PutUint16(h[2:4], 443)
binary.BigEndian.PutUint32(h[4:8], 0x11223344) // sequence
h[12] = 5 << 4 // data offset, no options
h[13] = 0x02 // SYN
binary.BigEndian.PutUint16(h[14:16], 65535) // window
return h
}}
udpDatagram = l4Proto{"udp", firewall.ProtoUDP, 6, func() []byte {
h := make([]byte, 8+4)
binary.BigEndian.PutUint16(h[0:2], 49152)
binary.BigEndian.PutUint16(h[2:4], 53)
binary.BigEndian.PutUint16(h[4:6], uint16(len(h)))
copy(h[8:], "ping")
return h
}}
icmpEcho = l4Proto{"icmp", firewall.ProtoICMP, 2, func() []byte { return echoRequest(8) }}
icmpv6Echo = l4Proto{"icmpv6", firewall.ProtoICMPv6, 2, func() []byte { return echoRequest(128) }}
)
// echoRequest builds an echo request body. The type differs between ICMP and
// ICMPv6, the rest of the header does not.
func echoRequest(typ uint8) []byte {
h := make([]byte, 8)
h[0] = typ
binary.BigEndian.PutUint16(h[4:6], 0xbeef) // identifier
binary.BigEndian.PutUint16(h[6:8], 1) // sequence
return h
}
func buildIPv6(src, dst netip.Addr, p l4Proto) []byte {
l4 := p.build()
pkt := make([]byte, ipv6HeaderLen+len(l4))
pkt[0] = 0x60
binary.BigEndian.PutUint16(pkt[4:6], uint16(len(l4)))
pkt[6] = p.nextHdr
pkt[7] = 64
copy(pkt[8:24], src.AsSlice())
copy(pkt[24:40], dst.AsSlice())
copy(pkt[ipv6HeaderLen:], l4)
if l4 := pkt[ipv6HeaderLen:]; p.nextHdr == firewall.ProtoTCP || p.nextHdr == firewall.ProtoUDP {
sum := ipv6PseudoheaderSum(src, dst, uint32(p.nextHdr), uint32(len(l4)))
binary.BigEndian.PutUint16(l4[p.cksumAt:], ^fold(sumBytes(l4, sum)))
}
return pkt
}
func buildIPv4(src, dst netip.Addr, p l4Proto) []byte {
l4 := p.build()
pkt := make([]byte, ipv4HeaderLen+len(l4))
pkt[0] = 0x45
binary.BigEndian.PutUint16(pkt[2:4], uint16(len(pkt)))
pkt[8] = 64
pkt[9] = p.nextHdr
copy(pkt[12:16], src.AsSlice())
copy(pkt[16:20], dst.AsSlice())
copy(pkt[ipv4HeaderLen:], l4)
if l4 := pkt[ipv4HeaderLen:]; p.nextHdr == firewall.ProtoTCP || p.nextHdr == firewall.ProtoUDP {
sum := sumBytes(pkt[12:20], uint32(p.nextHdr)+uint32(len(l4)))
binary.BigEndian.PutUint16(l4[p.cksumAt:], ^fold(sumBytes(l4, sum)))
}
return pkt
}
// ipv6PseudoheaderSum is the RFC 2460 section 8.1 pseudo-header sum: source,
// destination, a 32 bit upper-layer packet length and a 32 bit zero-padded next
// header. Kept local to the test so these assertions do not check nebula's
// checksum code against itself.
func ipv6PseudoheaderSum(src, dst netip.Addr, nextHeader, length uint32) uint32 {
var csum uint32
s, d := src.AsSlice(), dst.AsSlice()
for i := 0; i < 16; i += 2 {
csum += uint32(s[i])<<8 | uint32(s[i+1])
csum += uint32(d[i])<<8 | uint32(d[i+1])
}
return csum + length + nextHeader
}
func sumBytes(b []byte, csum uint32) uint32 {
for i := 0; i+1 < len(b); i += 2 {
csum += uint32(b[i])<<8 | uint32(b[i+1])
}
if len(b)%2 == 1 {
csum += uint32(b[len(b)-1]) << 8
}
return csum
}
func fold(csum uint32) uint16 {
for csum > 0xffff {
csum = (csum >> 16) + (csum & 0xffff)
}
return uint16(csum)
}
// l4ChecksumValid6 verifies an IPv6 upper-layer checksum the way a receiver
// does: the pseudo-header plus the whole upper-layer segment, checksum field
// included, folds to 0xffff. The next header field is the upper-layer protocol
// only while there are no extension headers, which is all this file builds.
func l4ChecksumValid6(pkt []byte) bool {
src, _ := netip.AddrFromSlice(pkt[8:24])
dst, _ := netip.AddrFromSlice(pkt[24:40])
l4 := pkt[ipv6HeaderLen:]
return fold(sumBytes(l4, ipv6PseudoheaderSum(src, dst, uint32(pkt[6]), uint32(len(l4))))) == 0xffff
}
// l4ChecksumValid4 is the IPv4 counterpart: the RFC 793/768 pseudo-header is
// source, destination, a zero byte, the protocol and the upper-layer length.
func l4ChecksumValid4(pkt []byte) bool {
ihl := int(pkt[0]&0x0f) << 2
l4 := pkt[ihl:]
return fold(sumBytes(l4, sumBytes(pkt[12:20], uint32(pkt[9])+uint32(len(l4))))) == 0xffff
}
// TestConsumeInsidePacketSelfTraffic covers the self-addressed branch of
// consumeInsidePacket, taken where immediatelyForwardToSelf is set (see
// inside_bsd.go): the packet goes straight back to the tun, ahead of the
// firewall and the handshake.
func TestConsumeInsidePacketSelfTraffic(t *testing.T) {
v4 := netip.MustParseAddr("100.100.1.42")
v6 := netip.MustParseAddr("fd00::42")
tests := []struct {
name string
addr netip.Addr
pkt []byte
}{
{"ipv4/tcp", v4, buildIPv4(v4, v4, tcpSyn)},
{"ipv4/udp", v4, buildIPv4(v4, v4, udpDatagram)},
{"ipv4/icmp", v4, buildIPv4(v4, v4, icmpEcho)},
{"ipv6/tcp", v6, buildIPv6(v6, v6, tcpSyn)},
{"ipv6/udp", v6, buildIPv6(v6, v6, udpDatagram)},
{"ipv6/icmpv6", v6, buildIPv6(v6, v6, icmpv6Echo)},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
f, tun := newSelfForwardInterface(tt.addr)
// consumeInsidePacket writes through the slice it is handed, so a
// packet that arrived with a valid checksum must come back out of
// bytes taken before the call, unchanged.
want := append([]byte(nil), tt.pkt...)
consumeInside(f, tt.pkt)
if immediatelyForwardToSelf {
require.Len(t, tun.writes, 1)
assert.Equal(t, want, tun.writes[0])
} else {
assert.Empty(t, tun.writes, "self traffic reaches the tun over loopback here and must be dropped")
}
})
}
}
// TestConsumeInsidePacketSelfTrafficChecksum shows that the self-forward
// returns the bytes it was handed, so a packet that arrived with a wrong
// upper-layer checksum is written back with that same wrong checksum and the
// kernel drops it on re-entry.
//
// This is how a macOS host loses TCP and UDP to its own IPv6 overlay address:
// the kernel writes only the pseudo-header sum into the checksum field and
// defers completion to hardware offload, state that does not survive the
// crossing into userspace. Which kernels do this, for which protocols and IP
// versions, is a property of the kernel and belongs to a test against a live
// one; here the checksum is simply wrong, and the forward must make it right.
func TestConsumeInsidePacketSelfTrafficChecksum(t *testing.T) {
if !immediatelyForwardToSelf {
t.Skip("self traffic never reaches the tun on this platform")
}
versions := []struct {
name string
addr netip.Addr
build func(src, dst netip.Addr, p l4Proto) []byte
l4At int
valid func(pkt []byte) bool
}{
{"v4", netip.MustParseAddr("100.100.1.42"), buildIPv4, ipv4HeaderLen, l4ChecksumValid4},
{"v6", netip.MustParseAddr("fd00::42"), buildIPv6, ipv6HeaderLen, l4ChecksumValid6},
}
for _, v := range versions {
for _, p := range []l4Proto{tcpSyn, udpDatagram} {
t.Run(v.name+"/"+p.name, func(t *testing.T) {
pkt := v.build(v.addr, v.addr, p)
binary.BigEndian.PutUint16(pkt[v.l4At+p.cksumAt:], 0x1234)
require.False(t, v.valid(pkt), "the packet under test must start with a wrong checksum")
f, tun := newSelfForwardInterface(v.addr)
consumeInside(f, pkt)
require.Len(t, tun.writes, 1)
assert.True(t, v.valid(tun.writes[0]),
"a forwarded %s packet must carry a valid checksum, got 0x%04x",
p.name, binary.BigEndian.Uint16(tun.writes[0][v.l4At+p.cksumAt:]))
})
}
}
}
+14 -29
View File
@@ -215,9 +215,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
ifce.connectionManager.intf = ifce ifce.connectionManager.intf = ifce
// Held until Close so waiting on the interface blocks until the resources are actually released
ifce.wg.Add(1)
return ifce, nil return ifce, nil
} }
@@ -261,16 +258,17 @@ func (f *Interface) activate() error {
f.readers[i] = reader f.readers[i] = reader
} }
// On error the caller owns the cleanup, Control.Start cancels the service context f.wg.Add(1) // for us to wait on Close() to return
// before releasing our resources so a waiter never observes a live context
if err = f.inside.Activate(); err != nil { if err = f.inside.Activate(); err != nil {
f.wg.Done()
f.inside.Close()
return err return err
} }
return nil return nil
} }
func (f *Interface) run() { func (f *Interface) run() (func() error, error) {
// Launch n queues to read packets from udp // Launch n queues to read packets from udp
for i := 0; i < f.routines; i++ { for i := 0; i < f.routines; i++ {
f.wg.Go(func() { f.wg.Go(func() {
@@ -285,14 +283,13 @@ func (f *Interface) run() {
}) })
} }
} return func() error {
f.wg.Wait()
func (f *Interface) wait() error { if e := f.fatalErr.Load(); e != nil {
f.wg.Wait() return *e
if e := f.fatalErr.Load(); e != nil { }
return *e return nil
} }, nil
return nil
} }
// onFatal stores the first fatal reader error, and calls triggerShutdown if it was the first one // onFatal stores the first fatal reader error, and calls triggerShutdown if it was the first one
@@ -325,10 +322,7 @@ func (f *Interface) listenOut(i int) {
f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, plaintext[:0], payload, h, fwPacket, lhh, nb, i, ctCache.Get()) f.readOutsidePackets(ViaSender{UdpAddr: fromUdpAddr}, plaintext[:0], payload, h, fwPacket, lhh, nb, i, ctCache.Get())
}) })
// An error after teardown began is shutdown noise, the closed flag covers resources if err != nil && !f.closed.Load() {
// Close releases itself and the cancelled ctx covers ones torn down by their owners
// reacting to it, like the user device pipes
if err != nil && !f.closed.Load() && f.ctx.Err() == nil {
f.l.Error("Error while reading inbound packet, closing", "error", err) f.l.Error("Error while reading inbound packet, closing", "error", err)
f.onFatal(err) f.onFatal(err)
} }
@@ -347,8 +341,7 @@ func (f *Interface) listenIn(reader io.ReadWriteCloser, i int) {
for { for {
n, err := reader.Read(packet) n, err := reader.Read(packet)
if err != nil { if err != nil {
// Same shutdown noise handling as listenOut if !f.closed.Load() {
if !f.closed.Load() && f.ctx.Err() == nil {
f.l.Error("Error while reading outbound packet, closing", "error", err, "reader", i) f.l.Error("Error while reading outbound packet, closing", "error", err, "reader", i)
f.onFatal(err) f.onFatal(err)
} }
@@ -549,15 +542,9 @@ func (f *Interface) GetCertState() *CertState {
return f.pki.getCertState() return f.pki.getCertState()
} }
// Close releases the interface's resources: the udp sockets and the tun device.
// It is idempotent and safe to call at any point in the lifecycle, including on an interface that never activated,
// calls after the first return nil without doing anything.
func (f *Interface) Close() error { func (f *Interface) Close() error {
if !f.closed.CompareAndSwap(false, true) {
return nil
}
var errs []error var errs []error
f.closed.Store(true)
// Release the udp readers // Release the udp readers
for i, u := range f.writers { for i, u := range f.writers {
@@ -573,8 +560,6 @@ func (f *Interface) Close() error {
if closeErr != nil { if closeErr != nil {
errs = append(errs, closeErr) errs = append(errs, closeErr)
} }
// Release the construction token so waiters know the resources are gone
f.wg.Done() f.wg.Done()
return errors.Join(errs...) return errors.Join(errs...)
} }
-147
View File
@@ -1,147 +0,0 @@
package iputil
import (
"encoding/binary"
"github.com/google/gopacket/layers"
"golang.org/x/net/ipv4"
"golang.org/x/net/ipv6"
"gvisor.dev/gvisor/pkg/tcpip/checksum"
)
const udpHeaderLen = 8
// SetTransportChecksum recomputes the TCP or UDP checksum of an IPv4 or IPv6
// packet in place.
//
// A kernel that offloads checksums to the NIC hands a packet to a tun with the
// transport checksum unfinished: only the pseudo-header sum is in the field and
// the rest is left for hardware that a tun does not have. A packet written
// straight back to that tun is dropped on re-entry unless the checksum is
// completed first. ICMP is left alone; it arrived complete on the kernels this
// was measured against.
//
// So is any packet whose transport header cannot be located: fragments, unknown
// extension headers and truncated packets. An IPv6 fragment header is declined
// even when it carries the whole datagram (RFC 6946 atomic fragment), because
// the walk reports only that a fragment header was present.
func SetTransportChecksum(packet []byte) {
if len(packet) < 1 {
return
}
switch int(packet[0] >> 4) {
case ipv4.Version:
setTransportChecksum4(packet)
case ipv6.Version:
setTransportChecksum6(packet)
}
}
func setTransportChecksum4(packet []byte) {
if len(packet) < ipv4.HeaderLen {
return
}
ihl := int(packet[0]&0x0f) << 2
end := int(binary.BigEndian.Uint16(packet[2:4]))
if ihl < ipv4.HeaderLen || end < ihl || end > len(packet) {
return
}
// The checksum covers the whole datagram, which a fragment (MF set or a
// non-zero offset) does not carry.
if binary.BigEndian.Uint16(packet[6:8])&0x3fff != 0 {
return
}
transport, ok := transportExtent(packet[ihl:end], packet[9])
if !ok {
return
}
csum := ipv4PseudoheaderChecksum(packet[12:16], packet[16:20], uint32(packet[9]), uint32(len(transport)))
writeTransportChecksum(transport, layers.IPProtocol(packet[9]), csum)
}
func setTransportChecksum6(packet []byte) {
if len(packet) < ipv6.HeaderLen {
return
}
end := ipv6.HeaderLen + int(binary.BigEndian.Uint16(packet[4:6]))
if end > len(packet) {
return
}
// The checksum covers the whole datagram, which a fragment does not carry.
// An unknown extension header hides where the transport header starts. A
// chain longer than the walk's budget ends it early, at an offset that was
// never checked against the packet.
proto, offset, _, anyFragment, err := IPv6FindUpperProtocol(packet[:end])
if err != nil || anyFragment || offset >= end {
return
}
transport, ok := transportExtent(packet[offset:end], proto)
if !ok {
return
}
csum := ipv6PseudoheaderChecksum(packet[8:24], packet[24:40], uint32(proto), uint32(len(transport)))
writeTransportChecksum(transport, layers.IPProtocol(proto), csum)
}
// transportExtent narrows a segment to the length its own header declares. UDP
// carries a Length field, and RFC 768 and RFC 8200 section 8.1 both make that
// field, not the IP payload extent, the length the pseudo-header counts and the
// checksum covers; a datagram padded out to a link's minimum frame is the usual
// way the two differ. TCP has no such field, so its segment runs to the end of
// the IP payload. A Length that overruns the bytes IP delivered describes a
// datagram that is not there.
func transportExtent(transport []byte, proto uint8) ([]byte, bool) {
if layers.IPProtocol(proto) != layers.IPProtocolUDP {
return transport, true
}
if len(transport) < udpHeaderLen {
return nil, false
}
ulen := int(binary.BigEndian.Uint16(transport[4:6]))
if ulen < udpHeaderLen || ulen > len(transport) {
return nil, false
}
return transport[:ulen], true
}
// writeTransportChecksum stores the checksum of transport, taken over the
// pseudo-header sum csum, in the header's checksum field. A UDP checksum that
// computes to zero goes on the wire as 0xffff: zero means no checksum was
// computed (RFC 768), and over IPv6 the checksum is mandatory (RFC 8200
// section 8.1).
func writeTransportChecksum(transport []byte, proto layers.IPProtocol, csum uint32) {
var at, minLen int
switch proto {
case layers.IPProtocolTCP:
at, minLen = 16, 20
case layers.IPProtocolUDP:
at, minLen = 6, udpHeaderLen
default:
return
}
if len(transport) < minLen {
return
}
transport[at], transport[at+1] = 0, 0
sum := ^checksum.Checksum(transport, fold(csum))
if sum == 0 && proto == layers.IPProtocolUDP {
sum = 0xffff
}
binary.BigEndian.PutUint16(transport[at:], sum)
}
// fold reduces a pseudo-header sum to the 16 bit seed Checksum takes. Carrying
// the high half back into the low half is what keeps the reduction lossless, so
// the seed sums exactly as the wider value would; 0xffff is its fixed point.
// Every term of that sum comes from a 16 bit field, so it stays far below the
// width at which the accumulator would wrap.
func fold(csum uint32) uint16 {
for csum > 0xffff {
csum = (csum >> 16) + (csum & 0xffff)
}
return uint16(csum)
}
-242
View File
@@ -1,242 +0,0 @@
package iputil
import (
"encoding/binary"
"net"
"testing"
"github.com/google/gopacket"
"github.com/google/gopacket/layers"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/net/ipv6"
)
// serialize builds a packet with gopacket, whose checksums are computed
// independently of this package.
func serialize(t *testing.T, ls ...gopacket.SerializableLayer) []byte {
buf := gopacket.NewSerializeBuffer()
require.NoError(t, gopacket.SerializeLayers(buf, gopacket.SerializeOptions{FixLengths: true, ComputeChecksums: true}, ls...))
return append([]byte(nil), buf.Bytes()...)
}
// withExtensionHeader inserts an 8 byte IPv6 extension header of the given
// type between the IPv6 header and its payload. The transport checksum does not
// change: the pseudo-header counts only upper-layer bytes.
func withExtensionHeader(pkt []byte, typ layers.IPProtocol, hdr [8]byte) []byte {
hdr[0] = pkt[6]
out := make([]byte, 0, len(pkt)+8)
out = append(out, pkt[:40]...)
out = append(out, hdr[:]...)
out = append(out, pkt[40:]...)
out[6] = byte(typ)
binary.BigEndian.PutUint16(out[4:6], binary.BigEndian.Uint16(pkt[4:6])+8)
return out
}
// truncate copies the first n bytes into a buffer of exactly that capacity, so
// a read past the length panics instead of quietly succeeding.
func truncate(pkt []byte, n int) []byte {
out := make([]byte, n)
copy(out, pkt)
return out
}
// extChain builds an IPv6 packet fronted by n Destination Options headers. Each
// points at another one, so the walk spends its whole budget without reaching a
// transport header. lastExtLen inflates the final header's declared length,
// which is how the walk ends up past the end of the packet.
func extChain(n int, lastExtLen byte) []byte {
pkt := make([]byte, ipv6.HeaderLen)
pkt[0], pkt[6], pkt[7] = 0x60, 60, 64
for i := range n {
h := make([]byte, 8)
h[0] = 60
if i == n-1 {
h[1] = lastExtLen
}
pkt = append(pkt, h...)
}
pkt = append(pkt, make([]byte, 20)...)
binary.BigEndian.PutUint16(pkt[4:6], uint16(len(pkt)-ipv6.HeaderLen))
return pkt
}
func TestSetTransportChecksum(t *testing.T) {
// Source and destination differ so that a pseudo-header built from the wrong
// one, or from the two swapped, does not land on the same checksum anyway.
v4 := func(proto layers.IPProtocol) *layers.IPv4 {
return &layers.IPv4{Version: 4, TTL: 64, Id: 0x1234, Protocol: proto, SrcIP: net.IPv4(192, 0, 2, 1).To4(), DstIP: net.IPv4(198, 51, 100, 2).To4()}
}
v6 := func(proto layers.IPProtocol) *layers.IPv6 {
return &layers.IPv6{Version: 6, HopLimit: 64, NextHeader: proto, SrcIP: net.ParseIP("2001:db8::1"), DstIP: net.ParseIP("2001:db8:1::2")}
}
tcp := func(ip gopacket.NetworkLayer) *layers.TCP {
l := &layers.TCP{SrcPort: 49152, DstPort: 443, SYN: true, Window: 65535}
require.NoError(t, l.SetNetworkLayerForChecksum(ip))
return l
}
udp := func(ip gopacket.NetworkLayer) *layers.UDP {
l := &layers.UDP{SrcPort: 49152, DstPort: 53}
require.NoError(t, l.SetNetworkLayerForChecksum(ip))
return l
}
payload := gopacket.Payload("self")
nop := layers.IPv4Option{OptionType: 1, OptionLength: 1}
ip4tcp := v4(layers.IPProtocolTCP)
ip4opts := v4(layers.IPProtocolTCP)
ip4opts.Options = []layers.IPv4Option{nop, nop, nop, nop}
ip4udp := v4(layers.IPProtocolUDP)
ip6tcp := v6(layers.IPProtocolTCP)
ip6udp := v6(layers.IPProtocolUDP)
hopByHop := [8]byte{0, 0, 1, 4} // next header, length 0, PadN of 4
// Bytes past the length the IP header declares are not part of the
// datagram and must not be summed.
trailing4 := append(serialize(t, ip4tcp, tcp(ip4tcp), payload), []byte("trailing")...)
trailing6 := append(serialize(t, ip6tcp, tcp(ip6tcp), payload), []byte("trailing")...)
// A datagram padded out past the length UDP declares: the pseudo-header
// counts the UDP Length field, so the checksum is the unpadded one.
padded4 := append(serialize(t, ip4udp, udp(ip4udp), payload), []byte("pad!")...)
binary.BigEndian.PutUint16(padded4[2:4], uint16(len(padded4)))
padded6 := append(serialize(t, ip6udp, udp(ip6udp), payload), []byte("pad!")...)
binary.BigEndian.PutUint16(padded6[4:6], uint16(len(padded6)-ipv6.HeaderLen))
// Corrupting the checksum and asking for it back must yield gopacket's
// packet, byte for byte.
recomputed := []struct {
name string
pkt []byte
cksum int
}{
{"v4 tcp", serialize(t, ip4tcp, tcp(ip4tcp), payload), 20 + 16},
{"v4 tcp with ip options", serialize(t, ip4opts, tcp(ip4opts), payload), 24 + 16},
{"v4 udp", serialize(t, ip4udp, udp(ip4udp), payload), 20 + 6},
{"v4 tcp header only", serialize(t, ip4tcp, tcp(ip4tcp)), 20 + 16},
{"v4 udp header only", serialize(t, ip4udp, udp(ip4udp)), 20 + 6},
{"v6 tcp", serialize(t, ip6tcp, tcp(ip6tcp), payload), 40 + 16},
{"v6 udp", serialize(t, ip6udp, udp(ip6udp), payload), 40 + 6},
{"v6 udp header only", serialize(t, ip6udp, udp(ip6udp)), 40 + 6},
{"v6 tcp behind hop-by-hop", withExtensionHeader(serialize(t, ip6tcp, tcp(ip6tcp), payload), layers.IPProtocolIPv6HopByHop, hopByHop), 48 + 16},
{"v4 tcp with bytes past the total length", trailing4, 20 + 16},
{"v6 tcp with bytes past the payload length", trailing6, 40 + 16},
{"v4 udp padded past its declared length", padded4, 20 + 6},
{"v6 udp padded past its declared length", padded6, 40 + 6},
}
for _, tt := range recomputed {
t.Run(tt.name, func(t *testing.T) {
got := append([]byte(nil), tt.pkt...)
binary.BigEndian.PutUint16(got[tt.cksum:], 0x1234)
require.NotEqual(t, tt.pkt, got)
SetTransportChecksum(got)
assert.Equal(t, tt.pkt, got)
})
}
ip4frag := v4(layers.IPProtocolTCP)
ip4frag.Flags = layers.IPv4MoreFragments
ip4later := v4(layers.IPProtocolTCP)
ip4later.FragOffset = 1
ip4icmp := v4(layers.IPProtocolICMPv4)
badIHL := serialize(t, ip4tcp, tcp(ip4tcp), payload)
badIHL[0] = 0x44 // header length 16, shorter than an ipv4 header
shortTotalLen := serialize(t, ip4tcp, tcp(ip4tcp), payload)
binary.BigEndian.PutUint16(shortTotalLen[2:4], 10) // shorter than the header it introduces
cutTCP := serialize(t, ip4tcp, tcp(ip4tcp), payload)
binary.BigEndian.PutUint16(cutTCP[2:4], 20+19) // one byte short of a tcp header
cutTCP = truncate(cutTCP, 20+19)
cutUDP := serialize(t, ip4udp, udp(ip4udp), payload)
binary.BigEndian.PutUint16(cutUDP[2:4], 20+7) // one byte short of a udp header
cutUDP = truncate(cutUDP, 20+7)
// Two bytes short, so a transport header survives whole and the minimum
// length check cannot stand in for the bounds check.
cutV6 := truncate(serialize(t, ip6tcp, tcp(ip6tcp), payload), 62)
fragment := [8]byte{0, 0, 0, 1, 0, 0, 0, 1} // next header, reserved, offset 0 with M set, id
overrun4 := serialize(t, ip4udp, udp(ip4udp), payload)
binary.BigEndian.PutUint16(overrun4[24:26], uint16(len(overrun4)-20+1)) // one byte past what ip delivered
overrun6 := serialize(t, ip6udp, udp(ip6udp), payload)
binary.BigEndian.PutUint16(overrun6[44:46], uint16(len(overrun6)-ipv6.HeaderLen+1))
shortUDPLen := serialize(t, ip4udp, udp(ip4udp), payload)
binary.BigEndian.PutUint16(shortUDPLen[24:26], 7) // shorter than the header it counts
// Where the checksum cannot be completed the packet is left as it came.
untouched := []struct {
name string
pkt []byte
cksum int
}{
{"v4 first fragment", serialize(t, ip4frag, tcp(ip4frag), payload), 20 + 16},
{"v4 later fragment", serialize(t, ip4later, tcp(ip4later), payload), 20 + 16},
{"v4 icmp", serialize(t, ip4icmp, &layers.ICMPv4{TypeCode: layers.CreateICMPv4TypeCode(8, 0), Id: 1, Seq: 1}, payload), 20 + 2},
{"v4 header length below the minimum", badIHL, 20 + 16},
{"v4 total length below the header length", shortTotalLen, 20 + 16},
{"v4 truncated below its total length", truncate(serialize(t, ip4tcp, tcp(ip4tcp), payload), 30), -1},
{"v4 tcp header cut short", cutTCP, 20 + 16},
{"v4 udp header cut short", cutUDP, -1},
{"v6 fragment", withExtensionHeader(serialize(t, ip6tcp, tcp(ip6tcp), payload), layers.IPProtocolIPv6Fragment, fragment), 48 + 16},
{"v6 truncated below its payload length", truncate(serialize(t, ip6tcp, tcp(ip6tcp), payload), 50), -1},
{"v6 truncated with a whole transport header still present", cutV6, 40 + 16},
{"v6 extension header chain longer than the walk", extChain(9, 0), 112 + 16},
{"v6 extension header chain running past the packet", extChain(8, 255), 104 + 16},
{"v4 udp length past the end of the datagram", overrun4, 20 + 6},
{"v6 udp length past the end of the datagram", overrun6, 40 + 6},
{"v4 udp length below a udp header", shortUDPLen, 20 + 6},
}
for _, tt := range untouched {
t.Run(tt.name, func(t *testing.T) {
if tt.cksum >= 0 {
binary.BigEndian.PutUint16(tt.pkt[tt.cksum:], 0x1234)
}
want := append([]byte(nil), tt.pkt...)
SetTransportChecksum(tt.pkt)
assert.Equal(t, want, tt.pkt)
})
}
t.Run("too short to carry a header", func(t *testing.T) {
for _, pkt := range [][]byte{nil, {}, {0x45}, {0x60}} {
assert.NotPanics(t, func() { SetTransportChecksum(pkt) })
}
})
t.Run("tcp checksum of zero goes out as zero", func(t *testing.T) {
pkt := serialize(t, ip4tcp, tcp(ip4tcp), gopacket.Payload{0, 0})
c := binary.BigEndian.Uint16(pkt[36:38])
require.NotZero(t, c)
// Only udp reserves zero to mean "not computed", so tcp keeps it.
binary.BigEndian.PutUint16(pkt[40:42], c)
SetTransportChecksum(pkt)
assert.Zero(t, binary.BigEndian.Uint16(pkt[36:38]))
})
t.Run("udp checksum of zero goes out as 0xffff", func(t *testing.T) {
pkt := serialize(t, ip4udp, udp(ip4udp), gopacket.Payload{0, 0})
c := binary.BigEndian.Uint16(pkt[26:28])
require.NotZero(t, c)
// The one's complement sum is now 0xffff - c; adding c to the payload
// makes it 0xffff, whose complement is zero.
binary.BigEndian.PutUint16(pkt[28:30], c)
SetTransportChecksum(pkt)
assert.Equal(t, uint16(0xffff), binary.BigEndian.Uint16(pkt[26:28]))
})
}
func TestFold(t *testing.T) {
// 0xffff is the fold's fixed point, so a loop bound one notch tight never
// terminates on it.
for _, tt := range []struct {
in uint32
want uint16
}{
{0, 0},
{0xffff, 0xffff},
{0x10000, 1},
{0x1fffe, 0xffff},
{0xffffffff, 0xffff},
} {
assert.Equal(t, tt.want, fold(tt.in))
}
}
+8 -31
View File
@@ -2,16 +2,11 @@ package iputil
import ( import (
"encoding/binary" "encoding/binary"
"errors"
"golang.org/x/net/ipv4" "golang.org/x/net/ipv4"
"golang.org/x/net/ipv6" "golang.org/x/net/ipv6"
) )
// ErrIPv6CouldNotFindPayload is returned when the ipv6 extension header chain is truncated before a terminal
// upper layer protocol is reached.
var ErrIPv6CouldNotFindPayload = errors.New("could not find payload in ipv6 packet")
const ( const (
// MaxIPv4RejectPacketSize is the largest IPv4 reject packet: // MaxIPv4RejectPacketSize is the largest IPv4 reject packet:
// - 20 byte ipv4 header // - 20 byte ipv4 header
@@ -204,8 +199,8 @@ func ipv4CreateRejectTCPPacket(packet []byte, out []byte) []byte {
} }
func ipv6CreateRejectPacket(packet []byte, out []byte) []byte { func ipv6CreateRejectPacket(packet []byte, out []byte) []byte {
proto, offset, isFragment, _, err := IPv6FindUpperProtocol(packet) proto, offset, isFragment := ipv6FindUpperProtocol(packet)
if err != nil || isFragment { if isFragment {
return nil return nil
} }
switch proto { switch proto {
@@ -338,18 +333,7 @@ func ipv6CreateRejectTCPPacket(packet []byte, out []byte, offset int) []byte {
return out return out
} }
// IPv6FindUpperProtocol walks the ipv6 extension header chain and returns the upper layer protocol, the func ipv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragment bool) {
// offset it begins at, and whether the packet is a non-first fragment. Only the RFC 8200 and IANA extension
// headers below are walked. Everything else, including Mobility (135), HIP (139), Shim6 (140), experimental
// 253/254, and real upper layer protocols like SCTP or GRE, is terminal. Walking those as extension headers
// is a firewall bypass, so they fail closed. For a non-first fragment the returned protocol is the fragmented
// protocol and offset points at the fragment header, there is no transport header to locate. Returns
// ErrIPv6CouldNotFindPayload if packet is smaller than an ipv6 header or the chain is truncated before a
// terminal protocol is reached.
func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragment, anyFragment bool, err error) {
if len(packet) < ipv6.HeaderLen {
return 0, 0, false, false, ErrIPv6CouldNotFindPayload
}
nextHeader = packet[6] nextHeader = packet[6]
offset = ipv6.HeaderLen offset = ipv6.HeaderLen
@@ -357,37 +341,30 @@ func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragm
switch nextHeader { switch nextHeader {
case 0, 43, 60: // Hop-by-Hop, Routing, Destination case 0, 43, 60: // Hop-by-Hop, Routing, Destination
if len(packet) < offset+2 { if len(packet) < offset+2 {
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload return nextHeader, offset, isFragment
} }
nextHeader = packet[offset] nextHeader = packet[offset]
offset += (int(packet[offset+1]) + 1) << 3 offset += (int(packet[offset+1]) + 1) << 3
case 44: // Fragment case 44: // Fragment
if len(packet) < offset+8 { if len(packet) < offset+8 {
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload return nextHeader, offset, isFragment
} }
anyFragment = true
// Non-first fragments carry no transport header, report the fragmented protocol and stop
if packet[offset+2] != 0 || packet[offset+3]&0xf8 != 0 { if packet[offset+2] != 0 || packet[offset+3]&0xf8 != 0 {
return packet[offset], offset, true, anyFragment, nil isFragment = true
} }
nextHeader = packet[offset] nextHeader = packet[offset]
offset += 8 offset += 8
case 51: // AH case 51: // AH
if len(packet) < offset+2 { if len(packet) < offset+2 {
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload return nextHeader, offset, isFragment
} }
nextHeader = packet[offset] nextHeader = packet[offset]
offset += (int(packet[offset+1]) + 2) << 2 offset += (int(packet[offset+1]) + 2) << 2
default: default:
// A prior extension header can declare a length that advances offset past the packet. The terminal return nextHeader, offset, isFragment
// protocol's header isn't actually here, so treat the chain as truncated rather than classifying it.
if offset > len(packet) {
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload
}
return nextHeader, offset, isFragment, anyFragment, nil
} }
} }
} }
-61
View File
@@ -6,7 +6,6 @@ import (
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/net/ipv4" "golang.org/x/net/ipv4"
"golang.org/x/net/ipv6" "golang.org/x/net/ipv6"
) )
@@ -475,63 +474,3 @@ func TestCreateICMPEchoResponse_IPv6_NotICMPv6(t *testing.T) {
result := CreateICMPEchoResponse(packet, out) result := CreateICMPEchoResponse(packet, out)
assert.Nil(t, result) assert.Nil(t, result)
} }
func Test_IPv6FindUpperProtocol(t *testing.T) {
src := net.ParseIP("fd00::1")
dst := net.ParseIP("fd00::2")
// 8 byte extension/transport stand-ins, first byte is the next header, second is the length field
extToTCP := []byte{6, 0, 0, 0, 0, 0, 0, 0} // len 0 -> 8 bytes, next = TCP
extToUDP := []byte{17, 0, 0, 0, 0, 0, 0, 0} // len 0 -> 8 bytes, next = UDP
extToRouting := []byte{43, 0, 0, 0, 0, 0, 0, 0} // len 0 -> 8 bytes, next = Routing
ahToUDP := []byte{17, 0, 0, 0, 0, 0, 0, 0} // AH len 0 -> (0+2)<<2 = 8 bytes, next = UDP
firstFragToUDP := []byte{17, 0, 0, 1, 0, 0, 0, 1} // frag offset 0, M=1, next = UDP
nonFirstFrag := []byte{17, 0, 0, 9, 0, 0, 0, 1} // frag offset non-zero, next = UDP
transport := []byte{0, 80, 1, 187, 0, 0, 0, 0} // stand-in bytes, IPv6FindUpperProtocol never reads ports
tests := []struct {
name string
nextHeader uint8
payload []byte
wantProto uint8
wantOffset int
wantFragment bool
wantAnyFrag bool
wantErr error
}{
{"plain udp", 17, transport, 17, ipv6.HeaderLen, false, false, nil},
{"hop-by-hop then tcp", 0, append(extToTCP, transport...), 6, ipv6.HeaderLen + 8, false, false, nil},
{"routing then tcp", 43, append(extToTCP, transport...), 6, ipv6.HeaderLen + 8, false, false, nil},
{"destination then udp", 60, append(extToUDP, transport...), 17, ipv6.HeaderLen + 8, false, false, nil},
{"hop-by-hop, routing, then tcp", 0, append(append(extToRouting, extToTCP...), transport...), 6, ipv6.HeaderLen + 16, false, false, nil},
{"ah then udp", 51, append(ahToUDP, transport...), 17, ipv6.HeaderLen + 8, false, false, nil},
{"first fragment walks to transport", 44, append(firstFragToUDP, transport...), 17, ipv6.HeaderLen + 8, false, true, nil},
{"non-first fragment stops", 44, append(nonFirstFrag, transport...), 17, ipv6.HeaderLen, true, true, nil},
{"unknown protocol is terminal", 132, transport, 132, ipv6.HeaderLen, false, false, nil}, // SCTP
{"truncated extension header", 0, nil, 0, ipv6.HeaderLen, false, false, ErrIPv6CouldNotFindPayload},
// Destination Options with a declared length (255+1)*8 = 2048 that runs past the 48 byte buffer, next = SCTP
{"extension length past buffer", 60, []byte{132, 255, 0, 0, 0, 0, 0, 0}, 132, ipv6.HeaderLen + 2048, false, false, ErrIPv6CouldNotFindPayload},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
packet := makeIPv6Packet(src, dst, tt.nextHeader, tt.payload)
proto, offset, isFragment, anyFragment, err := IPv6FindUpperProtocol(packet)
if tt.wantErr != nil {
assert.ErrorIs(t, err, tt.wantErr)
return
}
require.NoError(t, err)
assert.Equal(t, tt.wantProto, proto)
assert.Equal(t, tt.wantOffset, offset)
assert.Equal(t, tt.wantFragment, isFragment)
assert.Equal(t, tt.wantAnyFrag, anyFragment)
})
}
// A packet smaller than an ipv6 header must error rather than panic reading byte 6
t.Run("shorter than ipv6 header", func(t *testing.T) {
_, _, _, _, err := IPv6FindUpperProtocol(make([]byte, 6))
assert.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
})
}
+1 -36
View File
@@ -19,7 +19,6 @@ import (
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/logging" "github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/nicfinder"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
"github.com/slackhq/nebula/util" "github.com/slackhq/nebula/util"
) )
@@ -37,14 +36,6 @@ type LightHouse struct {
myVpnNetworksTable *bart.Lite myVpnNetworksTable *bart.Lite
punchy *Punchy punchy *Punchy
// localAddrsFn enumerates the underlay addresses we advertise. It is a field so tests can supply simulated
// addresses rather than whatever this machine's NICs happen to be. Set it before Start.
localAddrsFn func(*LocalAllowList) []netip.Addr
// lastLocalAddrsErr is the previous localAddrsFn failure. Enumeration runs on every update, so an
// unchanged failure is demoted to Debug rather than warning every lighthouse.interval forever.
lastLocalAddrsErr atomic.Pointer[string]
// Local cache of answers from light houses // Local cache of answers from light houses
// map of vpn addr to answers // map of vpn addr to answers
addrMap map[netip.Addr]*RemoteList addrMap map[netip.Addr]*RemoteList
@@ -116,12 +107,6 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)), queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)),
l: l, l: l,
} }
h.localAddrsFn = func(al *LocalAllowList) []netip.Addr {
addrs, err := nicfinder.Find(ctx, h.l, al)
h.logLocalAddrsErr(err)
return addrs
}
lighthouses := make([]netip.Addr, 0) lighthouses := make([]netip.Addr, 0)
h.lighthouses.Store(&lighthouses) h.lighthouses.Store(&lighthouses)
staticList := make(map[netip.Addr]struct{}) staticList := make(map[netip.Addr]struct{})
@@ -920,26 +905,6 @@ func (lh *LightHouse) TriggerUpdate() {
} }
} }
// logLocalAddrsErr reports a localAddrs failure at Warn the first time it is seen and at Debug while
// it persists unchanged, so a permanent failure does not warn on every update forever.
func (lh *LightHouse) logLocalAddrsErr(err error) {
if err == nil {
lh.lastLocalAddrsErr.Store(nil)
return
}
msg := err.Error()
prev := lh.lastLocalAddrsErr.Swap(&msg)
if prev != nil && *prev == msg {
if lh.l.Enabled(context.Background(), slog.LevelDebug) {
lh.l.Debug("Failed to collect local addresses to advertise", "error", err)
}
return
}
lh.l.Warn("Failed to collect local addresses to advertise", "error", err)
}
func (lh *LightHouse) SendUpdate() { func (lh *LightHouse) SendUpdate() {
var v4 []*V4AddrPort var v4 []*V4AddrPort
var v6 []*V6AddrPort var v6 []*V6AddrPort
@@ -953,7 +918,7 @@ func (lh *LightHouse) SendUpdate() {
} }
lal := lh.GetLocalAllowList() lal := lh.GetLocalAllowList()
for _, e := range lh.localAddrsFn(lal) { for _, e := range localAddrs(lh.l, lal) {
if lh.myVpnNetworksTable.Contains(e) { if lh.myVpnNetworksTable.Contains(e) {
continue continue
} }
-31
View File
@@ -1,9 +1,7 @@
package nebula package nebula
import ( import (
"bytes"
"encoding/binary" "encoding/binary"
"errors"
"fmt" "fmt"
"net/netip" "net/netip"
"testing" "testing"
@@ -740,32 +738,3 @@ func TestLighthouse_DeletesWork(t *testing.T) {
out = lh.Query(testHost) out = lh.Query(testHost)
assert.Nil(t, out) assert.Nil(t, out)
} }
func TestLightHouse_logLocalAddrsErr(t *testing.T) {
out := &bytes.Buffer{}
lh := &LightHouse{l: test.NewLoggerWithOutput(out)}
// The first sighting of a failure warns.
lh.logLocalAddrsErr(errors.New("permission denied"))
assert.Contains(t, out.String(), "level=WARN")
assert.Contains(t, out.String(), "permission denied")
// Repeating unchanged does not warn again, which is what keeps a permanent failure from warning
// on every lighthouse.interval for the life of the process.
out.Reset()
lh.logLocalAddrsErr(errors.New("permission denied"))
assert.NotContains(t, out.String(), "level=WARN")
// A different failure is a new event and warns.
out.Reset()
lh.logLocalAddrsErr(errors.New("something else"))
assert.Contains(t, out.String(), "level=WARN")
assert.Contains(t, out.String(), "something else")
// Recovering resets, so the same failure returning later warns again.
out.Reset()
lh.logLocalAddrsErr(nil)
assert.Empty(t, out.String())
lh.logLocalAddrsErr(errors.New("something else"))
assert.Contains(t, out.String(), "level=WARN")
}
+3 -27
View File
@@ -11,7 +11,6 @@ import (
"time" "time"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay" "github.com/slackhq/nebula/overlay"
"github.com/slackhq/nebula/sshd" "github.com/slackhq/nebula/sshd"
"github.com/slackhq/nebula/udp" "github.com/slackhq/nebula/udp"
@@ -21,12 +20,6 @@ import (
type m = map[string]any type m = map[string]any
// maxRoutines caps routines below the RejectHeadroom nonce gap so concurrent senders can't race the counter past wrap.
const maxRoutines = 1 << 16
// The reject headroom must exceed every sender that can be mid-reservation at once, about two per routine.
const _ = noiseutil.RejectHeadroom - 4*maxRoutines
func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) { func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
// Automatically cancel the context if Main returns an error, to signal all created goroutines to quit. // Automatically cancel the context if Main returns an error, to signal all created goroutines to quit.
@@ -88,6 +81,9 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
if routines < 1 { if routines < 1 {
routines = 1 routines = 1
} }
if routines > 1 {
l.Info("Using multiple routines", "routines", routines)
}
} else { } else {
// deprecated and undocumented // deprecated and undocumented
tunQueues := c.GetInt("tun.routines", 1) tunQueues := c.GetInt("tun.routines", 1)
@@ -97,12 +93,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
l.Warn("Setting tun.routines and listen.routines is deprecated. Use `routines` instead", "routines", routines) l.Warn("Setting tun.routines and listen.routines is deprecated. Use `routines` instead", "routines", routines)
} }
} }
if routines > maxRoutines {
l.Warn("Using multiple routines", "routines", maxRoutines, "clamped", true, "requestedRoutines", routines)
routines = maxRoutines
} else if routines > 1 {
l.Info("Using multiple routines", "routines", routines)
}
// EXPERIMENTAL // EXPERIMENTAL
// Intentionally not documented yet while we do more testing and determine // Intentionally not documented yet while we do more testing and determine
@@ -140,17 +130,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
udpConns := make([]udp.Conn, routines) udpConns := make([]udp.Conn, routines)
port := c.GetInt("listen.port", 0) port := c.GetInt("listen.port", 0)
// Callers get no handle to these until the Control is returned, release them on any error.
defer func() {
if reterr != nil {
for _, u := range udpConns {
if u != nil {
_ = u.Close()
}
}
}
}()
if !configTest { if !configTest {
rawListenHost := c.GetString("listen.host", "0.0.0.0") rawListenHost := c.GetString("listen.host", "0.0.0.0")
var listenHost netip.Addr var listenHost netip.Addr
@@ -278,8 +257,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
attachCommands(l, c, ssh, ifce) attachCommands(l, c, ssh, ifce)
networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c)
return &Control{ return &Control{
state: StateReady, state: StateReady,
f: ifce, f: ifce,
@@ -290,7 +267,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
statsStart: stats.Start, statsStart: stats.Start,
dnsStart: ds.Start, dnsStart: ds.Start,
lighthouseStart: lightHouse.StartUpdateWorker, lighthouseStart: lightHouse.StartUpdateWorker,
networkChangeStart: networkChanges.Start,
connectionManagerStart: connManager.Start, connectionManagerStart: connManager.Start,
}, nil }, nil
} }
+4 -13
View File
@@ -14,8 +14,7 @@ type MessageMetrics struct {
rxUnknown metrics.Counter rxUnknown metrics.Counter
txUnknown metrics.Counter txUnknown metrics.Counter
rxInvalid metrics.Counter rxInvalid metrics.Counter
txExhausted metrics.Counter
} }
func (m *MessageMetrics) Rx(t header.MessageType, s header.MessageSubType, i int64) { func (m *MessageMetrics) Rx(t header.MessageType, s header.MessageSubType, i int64) {
@@ -42,13 +41,6 @@ func (m *MessageMetrics) RxInvalid(i int64) {
} }
} }
// TxExhausted counts outbound packets dropped because the tunnel's message counter is spent.
func (m *MessageMetrics) TxExhausted(i int64) {
if m != nil && m.txExhausted != nil {
m.txExhausted.Inc(i)
}
}
func newMessageMetrics() *MessageMetrics { func newMessageMetrics() *MessageMetrics {
gen := func(t string) [][]metrics.Counter { gen := func(t string) [][]metrics.Counter {
return [][]metrics.Counter{ return [][]metrics.Counter{
@@ -69,10 +61,9 @@ func newMessageMetrics() *MessageMetrics {
rx: gen("rx"), rx: gen("rx"),
tx: gen("tx"), tx: gen("tx"),
rxUnknown: metrics.GetOrRegisterCounter("messages.rx.other", nil), rxUnknown: metrics.GetOrRegisterCounter("messages.rx.other", nil),
txUnknown: metrics.GetOrRegisterCounter("messages.tx.other", nil), txUnknown: metrics.GetOrRegisterCounter("messages.tx.other", nil),
rxInvalid: metrics.GetOrRegisterCounter("messages.rx.invalid", nil), rxInvalid: metrics.GetOrRegisterCounter("messages.rx.invalid", nil),
txExhausted: metrics.GetOrRegisterCounter("messages.tx.exhausted", nil),
} }
} }
-39
View File
@@ -1,39 +0,0 @@
//go:build !android
package nicfinder
import (
"context"
"errors"
"fmt"
"log/slog"
"net"
"net/netip"
)
func localAddrs(ctx context.Context, l *slog.Logger, filter Filter) ([]netip.Addr, error) {
ifaces, err := net.Interfaces()
if err != nil {
return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err)
}
var out []netip.Addr
var errs []error
for _, i := range ifaces {
if !allowName(ctx, l, filter, i.Name) {
continue
}
addrs, err := i.Addrs()
if err != nil {
errs = append(errs, fmt.Errorf("failed to get addresses for %s: %w", i.Name, err))
continue
}
for _, rawAddr := range addrs {
if addr, ok := allowedAddr(ctx, l, filter, rawAddr); ok {
out = append(out, addr)
}
}
}
return out, errors.Join(errs...)
}
-104
View File
@@ -1,104 +0,0 @@
//go:build android
package nicfinder
import (
"context"
"errors"
"fmt"
"log/slog"
"net/netip"
"os"
"github.com/DefinedNet/netlink"
"github.com/DefinedNet/netlink/nl"
"golang.org/x/sys/unix"
)
// Android denies bind() on netlink route sockets and RTM_GETLINK, so net.Interfaces fails with EACCES.
// Enumerate with an RTM_GETADDR dump on an unbound socket instead.
// localInterfaces returns every interface that has an address, with its addresses. It lists
// addresses over a netlink socket that is never bound and names each index with SIOCGIFNAME, the
// same operations Bionic's getifaddrs uses for app UIDs, which the sandbox permits.
func localAddrs(ctx context.Context, l *slog.Logger, filter Filter) ([]netip.Addr, error) {
var errs []error
addrs, err := netlinkAddrList()
if err != nil {
if !errors.Is(err, netlink.ErrDumpInterrupted) {
return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err)
}
// The address table kept changing under the dump and netlink gave up retrying. What it
// returned is still worth advertising.
errs = append(errs, fmt.Errorf("local addresses may be incomplete: %w", err))
}
// Any socket will do for SIOCGIFNAME.
sfd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0)
if err != nil {
return nil, os.NewSyscallError("socket", err)
}
defer unix.Close(sfd)
var out []netip.Addr
nameAllowed := make(map[int]bool) // by interface index, once named
for _, a := range addrs {
allow, named := nameAllowed[a.LinkIndex]
if (named && !allow) || a.IPNet == nil {
continue
}
addr, ok := allowedAddr(ctx, l, filter, a.IPNet)
if !ok {
continue
}
if !named {
name, err := interfaceNameByIndex(sfd, a.LinkIndex)
// ENODEV means the interface went away after the dump. Any other failure is reported
// and the interface skipped, so the rest are still returned.
if err != nil && !errors.Is(err, unix.ENODEV) {
errs = append(errs, fmt.Errorf("failed to resolve name of interface %d: %w", a.LinkIndex, err))
}
allow = err == nil && allowName(ctx, l, filter, name)
nameAllowed[a.LinkIndex] = allow
}
if allow {
out = append(out, addr)
}
}
return out, errors.Join(errs...)
}
// netlinkAddrList is netlink.AddrList over a socket that is never bound. The kernel autobinds it
// on the first send, which the sandbox permits.
func netlinkAddrList() ([]netlink.Addr, error) {
fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE)
if err != nil {
return nil, os.NewSyscallError("socket", err)
}
s, err := nl.NewNetlinkSocketFromFd(fd)
if err != nil {
unix.Close(fd)
return nil, err
}
h := netlink.NewHandleFromSockets(
map[int]*nl.SocketHandle{unix.NETLINK_ROUTE: {Socket: s}},
netlink.HandleOptions{RetryInterrupted: true},
)
defer h.Close()
if err := h.SetSocketTimeout(netlink.GetSocketTimeout()); err != nil {
return nil, err
}
return h.AddrList(nil, netlink.FAMILY_ALL)
}
// interfaceNameByIndex resolves an interface index with SIOCGIFNAME.
func interfaceNameByIndex(fd int, index int) (string, error) {
var ifr unix.Ifreq
ifr.SetUint32(uint32(index)) // ifr_ifindex is in the ifr_ifru union
if err := unix.IoctlIfreq(fd, unix.SIOCGIFNAME, &ifr); err != nil {
return "", err
}
return ifr.Name(), nil
}
-59
View File
@@ -1,59 +0,0 @@
package nicfinder
import (
"context"
"log/slog"
"net"
"net/netip"
"github.com/slackhq/nebula/logging"
)
// Filter decides which interfaces, by name, and which of their addresses Find returns.
type Filter interface {
AllowName(string) bool
Allow(netip.Addr) bool
}
// Find returns the addresses of this host's network interfaces that allowList accepts, with
// loopback and link-local addresses left out. When only some interfaces could be read it returns
// their addresses alongside an error for the rest. allowList must not be nil; it is called as
// given, so a nil pointer of a type whose methods accept a nil receiver is fine.
func Find(ctx context.Context, l *slog.Logger, allowList Filter) ([]netip.Addr, error) {
return localAddrs(ctx, l, allowList)
}
// allowName asks filter about an interface name and traces the answer.
func allowName(ctx context.Context, l *slog.Logger, filter Filter, name string) bool {
allow := filter.AllowName(name)
if l.Enabled(ctx, logging.LevelTrace) {
l.Log(ctx, logging.LevelTrace, "localAllowList.AllowName", "interfaceName", name, "allow", allow)
}
return allow
}
// allowedAddr converts one address a platform reports and says whether to advertise it: it must
// be a *net.IPNet or *net.IPAddr carrying an IP, not be loopback or link-local, and be accepted by
// filter, whose answer is traced. IPv4 comes back unmapped.
func allowedAddr(ctx context.Context, l *slog.Logger, filter Filter, rawAddr net.Addr) (netip.Addr, bool) {
var ip net.IP
switch v := rawAddr.(type) {
case *net.IPNet:
ip = v.IP
case *net.IPAddr:
ip = v.IP
}
addr, ok := netip.AddrFromSlice(ip)
if !ok {
return netip.Addr{}, false
}
addr = addr.Unmap()
if addr.IsLoopback() || addr.IsLinkLocalUnicast() {
return netip.Addr{}, false
}
isAllowed := filter.Allow(addr)
if l.Enabled(ctx, logging.LevelTrace) {
l.Log(ctx, logging.LevelTrace, "localAllowList.Allow", "localAddr", addr, "allowed", isAllowed)
}
return addr, isAllowed
}
-117
View File
@@ -1,117 +0,0 @@
package nicfinder
import (
"context"
"net"
"net/netip"
"regexp"
"testing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
)
type allowAll struct{}
func (allowAll) Allow(netip.Addr) bool { return true }
func (allowAll) AllowName(string) bool { return true }
type testAllowList struct {
bannedName *regexp.Regexp
bannedAddr netip.Addr
}
func (l *testAllowList) Allow(addr netip.Addr) bool {
return addr != l.bannedAddr
}
func (l *testAllowList) AllowName(n string) bool {
return !l.bannedName.MatchString(n)
}
func newTestAllowList(bannedName string) *testAllowList {
return &testAllowList{
bannedName: regexp.MustCompile(bannedName),
}
}
func TestAllowedAddr(t *testing.T) {
ctx, l := context.Background(), test.NewLogger()
allowed := func(filter Filter, addrs []net.Addr) []netip.Addr {
var out []netip.Addr
for _, rawAddr := range addrs {
if addr, ok := allowedAddr(ctx, l, filter, rawAddr); ok {
out = append(out, addr)
}
}
return out
}
addrs := []net.Addr{
&net.IPNet{IP: net.ParseIP("127.0.0.1"), Mask: net.CIDRMask(8, 32)},
&net.IPNet{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)},
&net.IPNet{IP: net.ParseIP("10.0.0.5"), Mask: net.CIDRMask(24, 32)},
&net.IPNet{IP: net.ParseIP("169.254.10.10"), Mask: net.CIDRMask(16, 32)},
&net.IPNet{IP: net.ParseIP("fe80::1"), Mask: net.CIDRMask(64, 128)},
&net.IPAddr{IP: net.ParseIP("fd00::5")},
&net.IPNet{},
otherAddr{},
nil,
}
// Loopback, link local and anything without an IP are dropped, and IPv4 comes back unmapped
// even though net.ParseIP hands over a 16 byte slice.
assert.Equal(t, []netip.Addr{
netip.MustParseAddr("10.0.0.5"),
netip.MustParseAddr("fd00::5"),
}, allowed(allowAll{}, addrs))
// The filter sees the unmapped form and can drop a single address.
al := newTestAllowList("none")
al.bannedAddr = netip.MustParseAddr("10.0.0.5")
assert.Equal(t, []netip.Addr{netip.MustParseAddr("fd00::5")}, allowed(al, addrs))
}
// otherAddr is a net.Addr that is neither *net.IPNet nor *net.IPAddr. netlink.Addr is one such
// type: it satisfies net.Addr through an embedded *net.IPNet, and passing it directly once
// silently produced no addresses.
type otherAddr struct{}
func (otherAddr) Network() string { return "other" }
func (otherAddr) String() string { return "other" }
// hostFilter counts AllowName calls and can reject every name or every address.
type hostFilter struct {
rejectNames bool
rejectAddrs bool
asked map[string]int
}
func (f *hostFilter) Allow(netip.Addr) bool { return !f.rejectAddrs }
func (f *hostFilter) AllowName(name string) bool {
f.asked[name]++
return !f.rejectNames
}
// Runs this platform's localAddrs against the host and checks it honors the whole filter.
func TestLocalAddrsAppliesFilter(t *testing.T) {
ctx, l := context.Background(), test.NewLogger()
f := &hostFilter{asked: map[string]int{}}
all, err := localAddrs(ctx, l, f)
if len(f.asked) == 0 {
t.Skipf("no interfaces to enumerate here: %v", err)
}
for name, n := range f.asked {
assert.Equal(t, 1, n, "AllowName asked %d times for %s", n, name)
}
// Loopback and link-local never come back, whatever the filter says.
for _, addr := range all {
assert.False(t, addr.IsLoopback() || addr.IsLinkLocalUnicast(), "%s", addr)
}
none, _ := localAddrs(ctx, l, &hostFilter{rejectNames: true, asked: map[string]int{}})
assert.Empty(t, none, "addresses got past a filter that rejects every interface name")
none, _ = localAddrs(ctx, l, &hostFilter{rejectAddrs: true, asked: map[string]int{}})
assert.Empty(t, none, "addresses got past a filter that rejects every address")
}
-3
View File
@@ -25,9 +25,6 @@ func (s *CipherStateAESGCM) EncryptDanger(out, ad, plaintext []byte, n uint64, n
if s == nil { if s == nil {
return nil, errors.New("no cipher state available to encrypt") return nil, errors.New("no cipher state available to encrypt")
} }
if n >= RejectAfterMessages {
return nil, ErrMessageCounterExhausted
}
nb[0] = 0 nb[0] = 0
nb[1] = 0 nb[1] = 0
nb[2] = 0 nb[2] = 0
-3
View File
@@ -24,9 +24,6 @@ func (s *CipherStateChaChaPoly) EncryptDanger(out, ad, plaintext []byte, n uint6
if s == nil { if s == nil {
return nil, errors.New("no cipher state available to encrypt") return nil, errors.New("no cipher state available to encrypt")
} }
if n >= RejectAfterMessages {
return nil, ErrMessageCounterExhausted
}
nb[0] = 0 nb[0] = 0
nb[1] = 0 nb[1] = 0
nb[2] = 0 nb[2] = 0
-11
View File
@@ -1,22 +1,11 @@
package noiseutil package noiseutil
import ( import (
"errors"
"fmt" "fmt"
"math"
"github.com/flynn/noise" "github.com/flynn/noise"
) )
// RejectHeadroom is the wrap gap for senders racing the counter, sized large enough for any routine count.
const RejectHeadroom = uint64(1) << 40
// RejectAfterMessages is the nonce ceiling: encrypting stops RejectHeadroom short of the wrap.
const RejectAfterMessages = math.MaxUint64 - RejectHeadroom
// ErrMessageCounterExhausted is returned by EncryptDanger once the nonce reaches RejectAfterMessages.
var ErrMessageCounterExhausted = errors.New("message counter exhausted")
// CipherState is the post-handshake AEAD cipher used for the data plane. // CipherState is the post-handshake AEAD cipher used for the data plane.
// Each supported cipher has its own concrete implementation in this package with the nonce endianness hardcoded, // Each supported cipher has its own concrete implementation in this package with the nonce endianness hardcoded,
// so the encrypt/decrypt fast path avoids interface dispatch on the byte order. // so the encrypt/decrypt fast path avoids interface dispatch on the byte order.
-19
View File
@@ -1,7 +1,6 @@
package noiseutil package noiseutil
import ( import (
"math"
"testing" "testing"
"github.com/flynn/noise" "github.com/flynn/noise"
@@ -90,24 +89,6 @@ func roundtrip(t *testing.T, enc, dec CipherState) {
assert.Equal(t, 16, enc.Overhead()) assert.Equal(t, 16, enc.Overhead())
} }
func TestEncryptRejectsExhaustedCounter(t *testing.T) {
// Pin the headroom below the uint64 wrap so a typo can't silently move the ceiling.
require.Equal(t, uint64(1)<<40, RejectHeadroom)
require.Equal(t, math.MaxUint64-RejectHeadroom, RejectAfterMessages)
encA, _ := buildCipherStates(t, CipherAESGCM)
encC, _ := buildCipherStates(t, noise.CipherChaChaPoly)
nb := make([]byte, 12)
for _, cs := range []CipherState{NewCipherStateAESGCM(encA), NewCipherStateChaChaPoly(encC)} {
_, err := cs.EncryptDanger(nil, nil, []byte("x"), RejectAfterMessages-1, nb)
require.NoError(t, err)
_, err = cs.EncryptDanger(nil, nil, []byte("x"), RejectAfterMessages, nb)
require.ErrorIs(t, err, ErrMessageCounterExhausted)
}
}
func BenchmarkCipherStateEncryptAESGCM(b *testing.B) { func BenchmarkCipherStateEncryptAESGCM(b *testing.B) {
enc, _ := buildCipherStatesB(b, CipherAESGCM) enc, _ := buildCipherStatesB(b, CipherAESGCM)
benchEncryptCipherState(b, NewCipherState(enc, CipherAESGCM)) benchEncryptCipherState(b, NewCipherState(enc, CipherAESGCM))
+140 -69
View File
@@ -13,7 +13,6 @@ import (
"github.com/slackhq/nebula/firewall" "github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/iputil"
"golang.org/x/net/ipv4" "golang.org/x/net/ipv4"
) )
@@ -103,31 +102,27 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
return return
} }
if len(packet) < header.Len+hostinfo.ConnectionState.dKey.Overhead() {
f.messageMetrics.RxInvalid(1)
if f.l.Enabled(context.Background(), slog.LevelDebug) {
f.l.Debug("packet too small", "from", via, "length", len(packet))
}
return
}
// All remaining packets are encrypted // All remaining packets are encrypted
if isMessageRelay { ci := hostinfo.ConnectionState
// Relay packets are special, this branch should always early-return if !ci.window.Check(f.l, h.MessageCounter) {
if err = hostinfo.ConnectionState.VerifyRelay(f.l, h.MessageCounter, packet, nb); err != nil {
if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("Failed to verify relay packet", "error", err, "from", via, "header", h)
}
return
}
f.handleOutsideRelayPacket(hostinfo, via, out, packet, h, fwPacket, lhf, nb, q, localCache)
return return
} }
out, err = hostinfo.ConnectionState.Decrypt(f.l, h.MessageCounter, out, packet, nb) // Relay packets are special
if isMessageRelay {
f.handleOutsideRelayPacket(hostinfo, via, out, packet, h, fwPacket, lhf, nb, q, localCache)
return
}
out, err = f.decrypt(hostinfo, h.MessageCounter, out, packet, h, nb)
if err != nil { if err != nil {
if f.l.Enabled(context.Background(), slog.LevelDebug) { if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("Failed to decrypt packet", "error", err, "from", via, "header", h) hostinfo.logger(f.l).Debug("Failed to decrypt packet",
"error", err,
"from", via,
"header", h,
)
} }
return return
} }
@@ -156,7 +151,7 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
// No-op, useful for the Roaming and connectionManager side-effects above // No-op, useful for the Roaming and connectionManager side-effects above
case header.TestRequest: case header.TestRequest:
//recycle the input packet ciphertext as our output buffer //recycle the input packet ciphertext as our output buffer
f.send(header.Test, header.TestReply, hostinfo.ConnectionState, hostinfo, out, nb, packet) f.send(header.Test, header.TestReply, ci, hostinfo, out, nb, packet)
default: default:
hostinfo.logger(f.l).Error("IsValidSubType was true, but unexpected test subtype seen", "from", via, "header", h) hostinfo.logger(f.l).Error("IsValidSubType was true, but unexpected test subtype seen", "from", via, "header", h)
return return
@@ -175,8 +170,27 @@ func (f *Interface) readOutsidePackets(via ViaSender, out []byte, packet []byte,
} }
func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) { func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender, out []byte, packet []byte, h *header.H, fwPacket *firewall.Packet, lhf *LightHouseHandler, nb []byte, q int, localCache firewall.ConntrackCache) {
// Successfully validated the thing. Get rid of the Relay header and the AEAD tag // The entire body is sent as AD, not encrypted.
signedPayload := packet[header.Len : len(packet)-hostinfo.ConnectionState.dKey.Overhead()] // The packet consists of a 16-byte parsed Nebula header, Associated Data-protected payload, and a trailing 16-byte AEAD signature value.
// The packet is guaranteed to be at least 16 bytes at this point, b/c it got past the h.Parse() call above. If it's
// otherwise malformed (meaning, there is no trailing 16 byte AEAD value), then this will result in at worst a 0-length slice
// which will gracefully fail in the DecryptDanger call.
signedPayload := packet[:len(packet)-hostinfo.ConnectionState.dKey.Overhead()]
signatureValue := packet[len(packet)-hostinfo.ConnectionState.dKey.Overhead():]
var err error
out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, signedPayload, signatureValue, h.MessageCounter, nb)
if err != nil {
return
}
// Advance the replay window now that the frame is authenticated
if !hostinfo.ConnectionState.window.Update(f.l, h.MessageCounter) {
if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("dropping out of window relay packet", "header", h)
}
return
}
// Successfully validated the thing. Get rid of the Relay header.
signedPayload = signedPayload[header.Len:]
// Pull the Roaming parts up here, and return in all call paths. // Pull the Roaming parts up here, and return in all call paths.
f.handleHostRoaming(hostinfo, via) f.handleHostRoaming(hostinfo, via)
// Track usage of both the HostInfo and the Relay for the received & authenticated packet // Track usage of both the HostInfo and the Relay for the received & authenticated packet
@@ -200,6 +214,7 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
via = ViaSender{ via = ViaSender{
UdpAddr: via.UdpAddr, UdpAddr: via.UdpAddr,
relayHI: hostinfo, relayHI: hostinfo,
remoteIdx: relay.RemoteIndex,
relay: relay, relay: relay,
IsRelayed: true, IsRelayed: true,
} }
@@ -220,10 +235,9 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
if targetRelay.State == Established { if targetRelay.State == Established {
switch targetRelay.Type { switch targetRelay.Type {
case ForwardingType: case ForwardingType:
// Forward this packet through the relay tunnel, rebuilding it in place. // Forward this packet through the relay tunnel
// Encode overwrites the old outer header, and the new AEAD tag lands where the old one was // Find the target HostInfo
fwdBuf := packet[:0:len(packet)] // Cap to len(packet) to protect memory from a larger parent buffer f.SendVia(targetHI, targetRelay, signedPayload, nb, out, false)
f.SendVia(targetHI, targetRelay, signedPayload, nb, fwdBuf, true)
case TerminalType: case TerminalType:
hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal") hostinfo.logger(f.l).Error("Unexpected Relay Type of Terminal")
return return
@@ -300,6 +314,7 @@ var (
ErrIPv4InvalidHeaderLength = errors.New("invalid ipv4 header length") ErrIPv4InvalidHeaderLength = errors.New("invalid ipv4 header length")
ErrIPv4PacketTooShort = errors.New("ipv4 packet is too short") ErrIPv4PacketTooShort = errors.New("ipv4 packet is too short")
ErrIPv6PacketTooShort = errors.New("ipv6 packet is too short") ErrIPv6PacketTooShort = errors.New("ipv6 packet is too short")
ErrIPv6CouldNotFindPayload = errors.New("could not find payload in ipv6 packet")
) )
// newPacket validates and parses the interesting bits for the firewall out of the ip and sub protocol headers // newPacket validates and parses the interesting bits for the firewall out of the ip and sub protocol headers
@@ -332,59 +347,101 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
fp.RemoteAddr, _ = netip.AddrFromSlice(data[24:40]) fp.RemoteAddr, _ = netip.AddrFromSlice(data[24:40])
} }
// Walk the extension header chain to the upper layer protocol. iputil.IPv6FindUpperProtocol is the single protoAt := 6 // NextHeader is at 6 bytes into the ipv6 header
// source of truth for which headers are extension headers, so this stays in lockstep with the reject path offset := ipv6.HeaderLen // Start at the end of the ipv6 header
// and cannot drift into misreading an unknown protocol (SCTP, GRE, etc.) as a forged transport. next := 0
proto, offset, isFragment, _, err := iputil.IPv6FindUpperProtocol(data) for {
if err != nil { if protoAt >= dataLen {
return ErrIPv6PacketTooShort break
}
fp.Protocol = proto
fp.Fragment = isFragment
if isFragment {
// Non-first fragments carry no transport header, so we have no ports to read
fp.RemotePort = 0
fp.LocalPort = 0
return nil
}
switch layers.IPProtocol(proto) {
case layers.IPProtocolICMPv6:
// An ICMPv6 message is at least type, code and checksum, 4 bytes. Only echo carries more than we read.
if dataLen < offset+4 {
return ErrIPv6PacketTooShort
} }
fp.LocalPort = 0 //incoming vs outgoing doesn't matter for icmpv6 proto := layers.IPProtocol(data[protoAt])
switch data[offset] { //icmp type
case layers.ICMPv6TypeEchoRequest, layers.ICMPv6TypeEchoReply: switch proto {
case layers.IPProtocolESP, layers.IPProtocolNoNextHeader:
fp.Protocol = uint8(proto)
fp.RemotePort = 0
fp.LocalPort = 0
fp.Fragment = false
return nil
case layers.IPProtocolICMPv6:
if dataLen < offset+6 { if dataLen < offset+6 {
return ErrIPv6PacketTooShort return ErrIPv6PacketTooShort
} }
fp.RemotePort = binary.BigEndian.Uint16(data[offset+4 : offset+6]) //identifier fp.Protocol = uint8(proto)
fp.LocalPort = 0 //incoming vs outgoing doesn't matter for icmpv6
icmptype := data[offset+1]
switch icmptype {
case layers.ICMPv6TypeEchoRequest, layers.ICMPv6TypeEchoReply:
fp.RemotePort = binary.BigEndian.Uint16(data[offset+4 : offset+6]) //identifier
default:
fp.RemotePort = 0
}
fp.Fragment = false
return nil
case layers.IPProtocolTCP, layers.IPProtocolUDP:
if dataLen < offset+4 {
return ErrIPv6PacketTooShort
}
fp.Protocol = uint8(proto)
if incoming {
fp.RemotePort = binary.BigEndian.Uint16(data[offset : offset+2])
fp.LocalPort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
} else {
fp.LocalPort = binary.BigEndian.Uint16(data[offset : offset+2])
fp.RemotePort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
}
fp.Fragment = false
return nil
case layers.IPProtocolIPv6Fragment:
// Fragment header is 8 bytes, need at least offset+4 to read the offset field
if dataLen < offset+8 {
return ErrIPv6PacketTooShort
}
// Check if this is the first fragment
fragmentOffset := binary.BigEndian.Uint16(data[offset+2:offset+4]) &^ uint16(0x7) // Remove the reserved and M flag bits
if fragmentOffset != 0 {
// Non-first fragment, use what we have now and stop processing
fp.Protocol = data[offset]
fp.Fragment = true
fp.RemotePort = 0
fp.LocalPort = 0
return nil
}
// The next loop should be the transport layer since we are the first fragment
next = 8 // Fragment headers are always 8 bytes
case layers.IPProtocolAH:
// Auth headers, used by IPSec, have a different meaning for header length
if dataLen <= offset+1 {
break
}
next = (int(data[offset+1]) + 2) << 2
default: default:
fp.RemotePort = 0 // Normal ipv6 header length processing
if dataLen <= offset+1 {
break
}
next = (int(data[offset+1]) + 1) << 3
} }
case layers.IPProtocolTCP, layers.IPProtocolUDP: if next <= 0 {
if dataLen < offset+4 { // Safety check, each ipv6 header has to be at least 8 bytes
return ErrIPv6PacketTooShort next = 8
}
if incoming {
fp.RemotePort = binary.BigEndian.Uint16(data[offset : offset+2])
fp.LocalPort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
} else {
fp.LocalPort = binary.BigEndian.Uint16(data[offset : offset+2])
fp.RemotePort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
} }
default: protoAt = offset
// don't set ports for protocols Nebula doesn't inspect offset = offset + next
fp.RemotePort = 0
fp.LocalPort = 0
} }
return nil return ErrIPv6CouldNotFindPayload
} }
func parseV4(data []byte, incoming bool, fp *firewall.Packet) error { func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
@@ -447,6 +504,20 @@ func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
return nil return nil
} }
func (f *Interface) decrypt(hostinfo *HostInfo, mc uint64, out []byte, packet []byte, h *header.H, nb []byte) ([]byte, error) {
var err error
out, err = hostinfo.ConnectionState.dKey.DecryptDanger(out, packet[:header.Len], packet[header.Len:], mc, nb)
if err != nil {
return nil, err
}
if !hostinfo.ConnectionState.window.Update(f.l, mc) {
return nil, ErrOutOfWindow
}
return out, nil
}
func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache) { func (f *Interface) handleOutsideMessagePacket(hostinfo *HostInfo, out []byte, packet []byte, fwPacket *firewall.Packet, nb []byte, q int, localCache firewall.ConntrackCache) {
err := newPacket(out, true, fwPacket) err := newPacket(out, true, fwPacket)
if err != nil { if err != nil {
+9 -89
View File
@@ -14,7 +14,6 @@ import (
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"golang.org/x/net/ipv4" "golang.org/x/net/ipv4"
"golang.org/x/net/ipv6"
) )
func Test_newPacket(t *testing.T) { func Test_newPacket(t *testing.T) {
@@ -116,12 +115,12 @@ func Test_newPacket_v6(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
err = newPacket(buffer.Bytes(), true, p) err = newPacket(buffer.Bytes(), true, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort) require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
// A v6 packet with a hop-by-hop extension // A v6 packet with a hop-by-hop extension
// ICMPv6 Payload (Echo Request) // ICMPv6 Payload (Echo Request)
icmpLayer := layers.ICMPv6{ icmpLayer := layers.ICMPv6{
TypeCode: layers.CreateICMPv6TypeCode(layers.ICMPv6TypeEchoRequest, 0), TypeCode: layers.ICMPv6TypeEchoRequest,
} }
// Hop-by-Hop Extension Header // Hop-by-Hop Extension Header
hopOption := layers.IPv6HopByHopOption{} hopOption := layers.IPv6HopByHopOption{}
@@ -150,12 +149,12 @@ func Test_newPacket_v6(t *testing.T) {
// A full IPv6 header and 1 byte in the first extension, but missing // A full IPv6 header and 1 byte in the first extension, but missing
// the length byte. // the length byte.
err = newPacket(buffer.Bytes()[:41], true, p) err = newPacket(buffer.Bytes()[:41], true, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort) require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
// A full IPv6 header plus 1 full extension, but only 1 byte of the // A full IPv6 header plus 1 full extension, but only 1 byte of the
// next layer, missing length byte // next layer, missing length byte
err = newPacket(buffer.Bytes()[:49], true, p) err = newPacket(buffer.Bytes()[:49], true, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort) require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
err = nil err = nil
// A good ICMP packet // A good ICMP packet
@@ -168,7 +167,7 @@ func Test_newPacket_v6(t *testing.T) {
} }
icmp := layers.ICMPv6{ icmp := layers.ICMPv6{
TypeCode: layers.CreateICMPv6TypeCode(layers.ICMPv6TypeEchoRequest, 0), TypeCode: layers.ICMPv6TypeEchoRequest,
Checksum: 0x1234, Checksum: 0x1234,
} }
@@ -190,18 +189,6 @@ func Test_newPacket_v6(t *testing.T) {
assert.Equal(t, uint16(0), p.LocalPort) assert.Equal(t, uint16(0), p.LocalPort)
assert.False(t, p.Fragment) assert.False(t, p.Fragment)
// A minimal 4 byte non-echo ICMPv6 message (type, code, checksum), no identifier to read
icmpMin := make([]byte, ipv6.HeaderLen+4)
copy(icmpMin, buffer.Bytes()[:ipv6.HeaderLen])
icmpMin[6] = byte(layers.IPProtocolICMPv6)
icmpMin[ipv6.HeaderLen] = 1 // type 1, destination unreachable, not echo
err = newPacket(icmpMin, true, p)
require.NoError(t, err)
assert.Equal(t, uint8(layers.IPProtocolICMPv6), p.Protocol)
assert.Equal(t, uint16(0), p.RemotePort)
assert.Equal(t, uint16(0), p.LocalPort)
assert.False(t, p.Fragment)
// A good ESP packet // A good ESP packet
b := buffer.Bytes() b := buffer.Bytes()
b[6] = byte(layers.IPProtocolESP) b[6] = byte(layers.IPProtocolESP)
@@ -226,15 +213,11 @@ func Test_newPacket_v6(t *testing.T) {
assert.Equal(t, uint16(0), p.LocalPort) assert.Equal(t, uint16(0), p.LocalPort)
assert.False(t, p.Fragment) assert.False(t, p.Fragment)
// An unknown protocol packet, we don't dissect it so we fail closed on its true protocol with no ports // An unknown protocol packet
b = buffer.Bytes() b = buffer.Bytes()
b[6] = 255 // 255 is a reserved protocol number b[6] = 255 // 255 is a reserved protocol number
err = newPacket(b, true, p) err = newPacket(b, true, p)
require.NoError(t, err) require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
assert.Equal(t, uint8(255), p.Protocol)
assert.Equal(t, uint16(0), p.RemotePort)
assert.Equal(t, uint16(0), p.LocalPort)
assert.False(t, p.Fragment)
// A good UDP packet // A good UDP packet
ip = layers.IPv6{ ip = layers.IPv6{
@@ -351,14 +334,14 @@ func Test_newPacket_v6(t *testing.T) {
assert.Equal(t, uint16(22), p.LocalPort) assert.Equal(t, uint16(22), p.LocalPort)
assert.False(t, p.Fragment) assert.False(t, p.Fragment)
// Ensure buffer bounds checking during processing, a truncated AH header can't reach the payload // Ensure buffer bounds checking during processing
err = newPacket(b[:41], true, p) err = newPacket(b[:41], true, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort) require.ErrorIs(t, err, ErrIPv6PacketTooShort)
// Invalid AH header // Invalid AH header
b = buffer.Bytes() b = buffer.Bytes()
err = newPacket(b, true, p) err = newPacket(b, true, p)
require.ErrorIs(t, err, ErrIPv6PacketTooShort) require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
} }
func Test_newPacket_ipv6Fragment(t *testing.T) { func Test_newPacket_ipv6Fragment(t *testing.T) {
@@ -692,66 +675,3 @@ func Test_newPacket_v6ExtHeaderOverflow(t *testing.T) {
// the host delivers to, not the forged 443 at the overflowed offset. // the host delivers to, not the forged 443 at the overflowed offset.
assert.Equal(t, uint16(22), p.LocalPort, "firewall must parse the real transport header, not the overflowed offset") assert.Equal(t, uint16(22), p.LocalPort, "firewall must parse the real transport header, not the overflowed offset")
} }
// Test_newPacket_v6ExtHeaderPastBuffer is a regression test for an extension header whose declared length
// advances the walk past the end of the packet. The upper layer protocol's header isn't actually present,
// so parseV6 must drop the packet rather than classify it as the terminal protocol with no ports.
func Test_newPacket_v6ExtHeaderPastBuffer(t *testing.T) {
p := &firewall.Packet{}
pkt := make([]byte, 48)
pkt[0] = 0x60
pkt[6] = byte(layers.IPProtocolIPv6Destination) // Destination Options
pkt[7] = 64 // hop limit
pkt[40] = byte(layers.IPProtocolSCTP) // Dest Options next header = SCTP
pkt[41] = 255 // declared length (255+1)*8 = 2048, past the 48 byte buffer
require.ErrorIs(t, newPacket(pkt, true, p), ErrIPv6PacketTooShort)
}
// Test_newPacket_v6ExtHeaderConfusion is a regression test for parseV6 walking any unrecognized
// Next Header as if it were an ipv6 extension header. A real upper layer protocol Nebula doesn't
// dissect (SCTP here) is not walkable, so applying the (len+1)*8 formula marched into the SCTP
// payload and landed on a byte that looked like UDP, forging a protocol/port pair the firewall
// would trust while the host delivered the real SCTP datagram. The fix fails closed: the packet
// is classified as its true protocol with no ports, so it only matches an `any` rule.
func Test_newPacket_v6ExtHeaderConfusion(t *testing.T) {
p := &firewall.Packet{}
pkt := make([]byte, 52)
pkt[0] = 0x60 // version 6
pkt[6] = byte(layers.IPProtocolSCTP) // NextHeader = SCTP, a real protocol, not an extension header
pkt[7] = 64 // hop limit
// Real SCTP header at offset 40. Pre-fix parseV6 walked SCTP as an extension header: byte 41 (0x00, the
// low byte of the src port below) was read as the header length, giving next=(0+1)*8=8, which landed the
// walk on byte 40 (0x11), misread as NextHeader=UDP, then bytes 48-51 as ports.
binary.BigEndian.PutUint16(pkt[40:42], 0x1100) // SCTP src port; byte 40=0x11, byte 41=0x00
binary.BigEndian.PutUint16(pkt[42:44], 445) // SCTP dst port, never read by parseV6
binary.BigEndian.PutUint16(pkt[48:50], 53) // SCTP checksum bytes, pre-fix forged RemotePort
binary.BigEndian.PutUint16(pkt[50:52], 53) // pre-fix forged LocalPort
require.NoError(t, newPacket(pkt, true, p))
assert.Equal(t, uint8(layers.IPProtocolSCTP), p.Protocol, "must classify as the true protocol, not the forged UDP")
assert.Equal(t, uint16(0), p.RemotePort)
assert.Equal(t, uint16(0), p.LocalPort)
assert.False(t, p.Fragment)
// Same confusion, but the unknown protocol sits after a real extension header. The HopByHop is walked
// correctly, then SCTP must still fail closed instead of being walked into its own payload. Protocol is
// the only assertion that discriminates the fix here, a regression that walked SCTP would misclassify it.
chained := make([]byte, 60)
chained[0] = 0x60 // version 6
chained[6] = byte(layers.IPProtocolIPv6HopByHop) // NextHeader = HopByHop extension
chained[7] = 64 // hop limit
chained[40] = byte(layers.IPProtocolSCTP) // HopByHop NextHeader = SCTP
chained[41] = 0 // HopByHop length 0 -> 8 bytes, SCTP begins at offset 48
binary.BigEndian.PutUint16(chained[48:50], 0x1100) // SCTP src port, pre-fix forged NextHeader/length bait
binary.BigEndian.PutUint16(chained[50:52], 445) // SCTP dst port, never read by parseV6
require.NoError(t, newPacket(chained, true, p))
assert.Equal(t, uint8(layers.IPProtocolSCTP), p.Protocol, "must fail closed on the unknown protocol after the extension header")
assert.Equal(t, uint16(0), p.RemotePort)
assert.Equal(t, uint16(0), p.LocalPort)
assert.False(t, p.Fragment)
}
-1
View File
@@ -40,7 +40,6 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
err := t.reload(c, true) err := t.reload(c, true)
if err != nil { if err != nil {
_ = file.Close()
return nil, err return nil, err
} }
+1
View File
@@ -659,6 +659,7 @@ func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
return fmt.Errorf("failed to create route.RouteMessage for change: %w", err) return fmt.Errorf("failed to create route.RouteMessage for change: %w", err)
} }
_, err = unix.Write(sock, data[:]) _, err = unix.Write(sock, data[:])
fmt.Println("DOING CHANGE")
return err return err
} }
return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err) return fmt.Errorf("failed to write route.RouteMessage to socket: %w", err)
-8
View File
@@ -18,7 +18,6 @@ import (
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/util" "github.com/slackhq/nebula/util"
"golang.org/x/sys/unix"
) )
type tun struct { type tun struct {
@@ -34,12 +33,6 @@ func newTun(_ *config.C, _ *slog.Logger, _ []netip.Prefix, _ bool) (*tun, error)
} }
func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) { func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
if err := unix.SetNonblock(deviceFd, true); err != nil {
// We own the fd from the moment it is handed to us, same as the reload error path below
_ = unix.Close(deviceFd)
return nil, fmt.Errorf("failed to set the tun fd to non-blocking mode: %w", err)
}
file := os.NewFile(uintptr(deviceFd), "/dev/tun") file := os.NewFile(uintptr(deviceFd), "/dev/tun")
t := &tun{ t := &tun{
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
@@ -49,7 +42,6 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
err := t.reload(c, true) err := t.reload(c, true)
if err != nil { if err != nil {
_ = file.Close()
return nil, err return nil, err
} }
+36 -15
View File
@@ -18,11 +18,11 @@ import (
"time" "time"
"unsafe" "unsafe"
"github.com/DefinedNet/netlink"
"github.com/gaissmai/bart" "github.com/gaissmai/bart"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing" "github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/util" "github.com/slackhq/nebula/util"
"github.com/vishvananda/netlink"
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
) )
@@ -251,6 +251,17 @@ func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip
} }
func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) { func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue bool) (*tun, error) {
// Validate the device name up front so a bad tun.dev fails fast, before we
// open /dev/net/tun or leak a file descriptor. A single %d in the name is
// substituted by the kernel during TUNSETIFF (dev_alloc_name) with the
// lowest number that yields an unused device name. Resolving the template
// in the kernel keeps the pick-a-name/create-the-device pair atomic, so
// concurrent callers can never race each other to the same name.
tunName := c.GetString("tun.dev", "nebula%d")
if err := validateTunName(tunName); err != nil {
return nil, err
}
fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0) fd, err := unix.Open("/dev/net/tun", os.O_RDWR, 0)
if err != nil { if err != nil {
// If /dev/net/tun doesn't exist, try to create it (will happen in docker) // If /dev/net/tun doesn't exist, try to create it (will happen in docker)
@@ -278,12 +289,11 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue
if multiqueue { if multiqueue {
req.Flags |= unix.IFF_MULTI_QUEUE req.Flags |= unix.IFF_MULTI_QUEUE
} }
nameStr := c.GetString("tun.dev", "") copy(req.Name[:], tunName)
copy(req.Name[:], nameStr)
if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil { if err = ioctl(uintptr(fd), uintptr(unix.TUNSETIFF), uintptr(unsafe.Pointer(&req))); err != nil {
_ = unix.Close(fd) _ = unix.Close(fd)
return nil, &NameError{ return nil, &NameError{
Name: nameStr, Name: tunName,
Underlying: err, Underlying: err,
} }
} }
@@ -299,6 +309,27 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, multiqueue
return t, nil return t, nil
} }
func validateTunName(tunName string) error {
if !strings.Contains(tunName, "%d") {
if len(tunName) >= unix.IFNAMSIZ {
return fmt.Errorf("tun.dev %q is not shorter than the maximum device name length of %d", tunName, unix.IFNAMSIZ)
}
return nil
}
if strings.Count(tunName, "%d") > 1 {
return fmt.Errorf("tun.dev template %q may only contain a single %%d", tunName)
}
if tunName == "%d" {
return errors.New("please don't name your tun device '%d'")
}
// The kernel substitutes the %d itself and requires the template, like a
// literal name, to be NUL-terminated within IFNAMSIZ bytes.
if len(tunName) >= unix.IFNAMSIZ {
return fmt.Errorf("tun.dev template %q is not shorter than the maximum device name length of %d", tunName, unix.IFNAMSIZ)
}
return nil
}
// newTunGeneric does all the stuff common to different tun initialization paths. It will close your files on error. // newTunGeneric does all the stuff common to different tun initialization paths. It will close your files on error.
func newTunGeneric(c *config.C, l *slog.Logger, fd int, vpnNetworks []netip.Prefix) (*tun, error) { func newTunGeneric(c *config.C, l *slog.Logger, fd int, vpnNetworks []netip.Prefix) (*tun, error) {
tfd, err := newTunFd(fd) tfd, err := newTunFd(fd)
@@ -484,16 +515,7 @@ func (t *tun) addIPs(link netlink.Link) error {
//iterate over remainder, remove whoever shouldn't be there //iterate over remainder, remove whoever shouldn't be there
al, err := netlink.AddrList(link, netlink.FAMILY_ALL) al, err := netlink.AddrList(link, netlink.FAMILY_ALL)
if err != nil { if err != nil {
//RTM_GETADDR dumps the whole system, so any concurrent address change return fmt.Errorf("failed to get tun address list: %s", err)
//interrupts it - including the kernel's async tentative->preferred
//flip of an IPv6 address the AddrReplace calls above just added,
//which makes this a race against our own setup. Partial results are
//still returned; the worst case is a stale address surviving until
//the next config reload, which beats failing startup over it.
if !errors.Is(err, netlink.ErrDumpInterrupted) {
return fmt.Errorf("failed to get tun address list: %s", err)
}
t.l.Warn("tun address list dump was interrupted, stale addresses may remain")
} }
for i := range al { for i := range al {
@@ -778,7 +800,6 @@ func (t *tun) isGatewayInVpnNetworks(gwAddr netip.Addr) bool {
func (t *tun) getGatewaysFromRoute(r *netlink.Route) routing.Gateways { func (t *tun) getGatewaysFromRoute(r *netlink.Route) routing.Gateways {
var gateways routing.Gateways var gateways routing.Gateways
link, err := netlink.LinkByName(t.Device) link, err := netlink.LinkByName(t.Device)
if err != nil { if err != nil {
t.l.Error("Ignoring route update: failed to get link by name", "deviceName", t.Device) t.l.Error("Ignoring route update: failed to get link by name", "deviceName", t.Device)
+42 -1
View File
@@ -3,7 +3,12 @@
package overlay package overlay
import "testing" import (
"strings"
"testing"
"golang.org/x/sys/unix"
)
var runAdvMSSTests = []struct { var runAdvMSSTests = []struct {
name string name string
@@ -32,3 +37,39 @@ func TestTunAdvMSS(t *testing.T) {
}) })
} }
} }
func TestValidateTunName(t *testing.T) {
// A device name must be shorter than IFNAMSIZ (i.e. IFNAMSIZ-1 chars max).
maxLenName := strings.Repeat("a", unix.IFNAMSIZ-1)
tests := []struct {
name string
tmpl string
wantErr bool
}{
{"short literal name is fine", "nebula1", false},
{"literal name at the max length is fine", maxLenName, false},
{"literal name at IFNAMSIZ is rejected", strings.Repeat("a", unix.IFNAMSIZ), true},
{"trailing template is fine", "nebula%d", false},
{"mid-string template is fine", "neb%dprod", false},
{"leading template is fine", "%dnebula", false},
{"template at the max length is fine", strings.Repeat("a", unix.IFNAMSIZ-3) + "%d", false},
{"template at IFNAMSIZ is rejected", strings.Repeat("a", unix.IFNAMSIZ-2) + "%d", true},
{"bare %d is rejected", "%d", true},
{"multiple %d is rejected", "neb%d%dprod", true},
{"over-long template is rejected", strings.Repeat("a", unix.IFNAMSIZ-1) + "%d", true},
{"over-long mid-string template is rejected", "neb%d" + strings.Repeat("a", unix.IFNAMSIZ-3), true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := validateTunName(tt.tmpl)
if tt.wantErr && err == nil {
t.Fatalf("expected an error for %q, got none", tt.tmpl)
}
if !tt.wantErr && err != nil {
t.Fatalf("unexpected error for %q: %v", tt.tmpl, err)
}
})
}
}
-29
View File
@@ -12,7 +12,6 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"runtime" "runtime"
"slices"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -179,7 +178,6 @@ func (t *winTun) addRoutes(logErrors bool) error {
luid := winipcfg.LUID(t.tun.LUID()) luid := winipcfg.LUID(t.tun.LUID())
routes := *t.Routes.Load() routes := *t.Routes.Load()
foundDefault4 := false foundDefault4 := false
carriesV6 := slices.ContainsFunc(t.vpnNetworks, func(p netip.Prefix) bool { return p.Addr().Is6() })
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if len(r.Via) == 0 || !r.Install {
@@ -187,9 +185,6 @@ func (t *winTun) addRoutes(logErrors bool) error {
continue continue
} }
// A v6 unsafe_route is legal under a v4-only cert; uninstalled ones put nothing on the adapter.
carriesV6 = carriesV6 || r.Cidr.Addr().Is6()
// Add our unsafe route as an on-link route to the nebula tun device. // Add our unsafe route as an on-link route to the nebula tun device.
err := luid.AddRoute(r.Cidr, unspecifiedNextHop(r.Cidr), uint32(r.Metric)) err := luid.AddRoute(r.Cidr, unspecifiedNextHop(r.Cidr), uint32(r.Metric))
if err != nil { if err != nil {
@@ -211,11 +206,6 @@ func (t *winTun) addRoutes(logErrors bool) error {
} }
} }
return t.setMTU(luid, foundDefault4, carriesV6)
}
// setMTU applies tun.mtu per address family. The default route metric rides along on the v4 handle.
func (t *winTun) setMTU(luid winipcfg.LUID, foundDefault4, carriesV6 bool) error {
ipif, err := luid.IPInterface(windows.AF_INET) ipif, err := luid.IPInterface(windows.AF_INET)
if err != nil { if err != nil {
return fmt.Errorf("failed to get ip interface: %w", err) return fmt.Errorf("failed to get ip interface: %w", err)
@@ -230,25 +220,6 @@ func (t *winTun) setMTU(luid winipcfg.LUID, foundDefault4, carriesV6 bool) error
if err := ipif.Set(); err != nil { if err := ipif.Set(); err != nil {
return fmt.Errorf("failed to set ip interface: %w", err) return fmt.Errorf("failed to set ip interface: %w", err)
} }
// Windows tracks NLMTU per family and wintun sets neither, so v6 keeps the adapter default of 65535.
// Gated so a v4-only overlay under 1280 boots; a v6 one deliberately does not, as linux also refuses.
if !carriesV6 {
return nil
}
ipif6, err := luid.IPInterface(windows.AF_INET6)
if err != nil {
// No v6 on the adapter means there is no NLMTU to get wrong. A failed Set below is not the same thing.
t.l.Info("Skipping ipv6 MTU, no ipv6 interface on this adapter", "error", err)
return nil
}
ipif6.NLMTU = uint32(t.MTU)
if err := ipif6.Set(); err != nil {
return fmt.Errorf("failed to set ipv6 interface: %w", err)
}
return nil return nil
} }
+1 -9
View File
@@ -107,10 +107,7 @@ func (rm *relayManager) StartRelays(f *Interface, vpnIp netip.Addr, hh *Handshak
if relayHostInfo.GetRemote().IsValid() { if relayHostInfo.GetRemote().IsValid() {
idx, err := AddRelay(rm.l, relayHostInfo, rm.hostmap, vpnIp, nil, TerminalType, Requested) idx, err := AddRelay(rm.l, relayHostInfo, rm.hostmap, vpnIp, nil, TerminalType, Requested)
if err != nil { if err != nil {
// No local relay state was installed, so a CreateRelayRequest would hand the
// peer an index we could never resolve. Skip it.
hl.Info("Failed to add relay to hostmap", "relay", relay.String(), "error", err) hl.Info("Failed to add relay to hostmap", "relay", relay.String(), "error", err)
continue
} }
m := NebulaControl{ m := NebulaControl{
@@ -240,12 +237,7 @@ func AddRelay(l *slog.Logger, relayHostInfo *HostInfo, hm *HostMap, vpnIp netip.
// Avoid standing up a relay that can't be used since only the primary hostinfo // Avoid standing up a relay that can't be used since only the primary hostinfo
// will be pointed to by the relay logic // will be pointed to by the relay logic
//TODO: if there was an existing primary and it had relay state, should we merge? //TODO: if there was an existing primary and it had relay state, should we merge?
if !hm.unlockedMakePrimary(relayHostInfo) { hm.unlockedMakePrimary(relayHostInfo)
// The tunnel was torn down after the caller grabbed relayHostInfo. A relay standing
// on an unlinked hostinfo would never carry traffic, and its Relays entry could
// never be reclaimed since the delete-time cleanup has already run.
return 0, errors.New("relay hostinfo is no longer in the hostmap")
}
hm.Relays[index] = relayHostInfo hm.Relays[index] = relayHostInfo
newRelay := Relay{ newRelay := Relay{
+8 -16
View File
@@ -43,25 +43,12 @@ type Service struct {
} }
} }
func New(control *nebula.Control) (_ *Service, reterr error) { func New(control *nebula.Control) (*Service, error) {
// Check this before Start so a failure doesn't leave a running nebula wait, err := control.Start()
device, ok := control.Device().(*overlay.UserDevice)
if !ok {
return nil, errors.New("must be using user device")
}
err := control.Start()
if err != nil { if err != nil {
return nil, err return nil, err
} }
// Anything that fails after a successful Start must tear nebula back down
defer func() {
if reterr != nil {
control.Stop()
}
}()
ctx := control.Context() ctx := control.Context()
eg, ctx := errgroup.WithContext(ctx) eg, ctx := errgroup.WithContext(ctx)
s := Service{ s := Service{
@@ -70,6 +57,11 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
} }
s.mu.listeners = map[uint16]*tcpListener{} s.mu.listeners = map[uint16]*tcpListener{}
device, ok := control.Device().(*overlay.UserDevice)
if !ok {
return nil, errors.New("must be using user device")
}
s.ipstack = stack.New(stack.Options{ s.ipstack = stack.New(stack.Options{
NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol}, NetworkProtocols: []stack.NetworkProtocolFactory{ipv4.NewProtocol, ipv6.NewProtocol},
TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4, icmp.NewProtocol6}, TransportProtocols: []stack.TransportProtocolFactory{tcp.NewProtocol, udp.NewProtocol, icmp.NewProtocol4, icmp.NewProtocol6},
@@ -155,7 +147,7 @@ func New(control *nebula.Control) (_ *Service, reterr error) {
// Add the nebula wait function to the group so a fatal reader error // Add the nebula wait function to the group so a fatal reader error
// propagates out through errgroup.Wait(). // propagates out through errgroup.Wait().
eg.Go(func() error { eg.Go(func() error {
return control.Wait() return wait()
}) })
return &s, nil return &s, nil
+3 -2
View File
@@ -13,6 +13,7 @@ import (
"sync/atomic" "sync/atomic"
"time" "time"
graphite "github.com/cyberdelia/go-metrics-graphite"
mp "github.com/nbrownus/go-metrics-prometheus" mp "github.com/nbrownus/go-metrics-prometheus"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp" "github.com/prometheus/client_golang/prometheus/promhttp"
@@ -252,7 +253,7 @@ func (s *statsServer) buildRuntime(cfg statsConfig) ([]func(), *http.Server) {
// loadStatsConfig already resolved and validated the address; re-parse // loadStatsConfig already resolved and validated the address; re-parse
// the resolved form (no DNS lookup) to get a *net.TCPAddr. // the resolved form (no DNS lookup) to get a *net.TCPAddr.
addr, _ := net.ResolveTCPAddr(cfg.graphite.protocol, cfg.graphite.resolvedAddr) addr, _ := net.ResolveTCPAddr(cfg.graphite.protocol, cfg.graphite.resolvedAddr)
gcfg := graphiteConfigExport{ gcfg := graphite.Config{
Addr: addr, Addr: addr,
Registry: metrics.DefaultRegistry, Registry: metrics.DefaultRegistry,
FlushInterval: cfg.interval, FlushInterval: cfg.interval,
@@ -261,7 +262,7 @@ func (s *statsServer) buildRuntime(cfg statsConfig) ([]func(), *http.Server) {
Percentiles: []float64{0.5, 0.75, 0.95, 0.99, 0.999}, Percentiles: []float64{0.5, 0.75, 0.95, 0.99, 0.999},
} }
captureFns = append(captureFns, func() { captureFns = append(captureFns, func() {
if err := graphiteOnce(gcfg); err != nil { if err := graphite.Once(gcfg); err != nil {
s.l.Error("Graphite export failed", "error", err) s.l.Error("Graphite export failed", "error", err)
} }
}) })
+1 -1
View File
@@ -371,7 +371,7 @@ func waitForListening(t *testing.T, addr string) {
}) })
} }
// graphiteSink is a minimal TCP accept-and-discard server so graphiteOnce // graphiteSink is a minimal TCP accept-and-discard server so graphite.Once
// calls in tests don't spam error logs or wedge on connection refused. // calls in tests don't spam error logs or wedge on connection refused.
type graphiteSink struct { type graphiteSink struct {
ln net.Listener ln net.Listener
-61
View File
@@ -1,61 +0,0 @@
package udp
import (
"context"
"log/slog"
"github.com/slackhq/nebula/config"
)
// NetworkChangeMonitor rebinds the udp listener when the local network moves out from under it.
//
// Detection lives here in the udp package, next to the socket it concerns and the platform matrix that already knows
// which sockets go stale. What to do about a change — updating the lighthouse, requerying tunnels — is not the udp
// package's business, so Start takes the reaction as a plain function. Passing it at Start rather than holding it
// keeps this package from referencing whatever owns the rebind.
//
// On platforms whose sockets do not go stale, watchNetworkChanges hands back a nil channel and Start returns.
type NetworkChangeMonitor struct {
l *slog.Logger
ctx context.Context
enabled bool
}
// NewNetworkChangeMonitor builds a monitor for local network changes. The returned monitor is always usable: Start
// is safe to call unconditionally, it no-ops when disabled or on a platform that does not need it.
func NewNetworkChangeMonitor(ctx context.Context, l *slog.Logger, c *config.C) *NetworkChangeMonitor {
return &NetworkChangeMonitor{
l: l,
ctx: ctx,
enabled: c.GetBool("listen.rebind_on_network_change", true),
}
}
// Start watches for network changes until the context is cancelled, calling rebind once per settled change. It
// blocks, so callers run it in a goroutine, and it no-ops when disabled, unsupported, or with nothing to rebind.
func (m *NetworkChangeMonitor) Start(rebind func()) {
if !m.enabled || rebind == nil || m.ctx.Err() != nil {
return
}
changes, err := watchNetworkChanges(m.ctx, m.l)
if err != nil {
// Not fatal. Everything else still works, we just won't notice a network change on our own.
m.l.Error("Failed to watch for network changes, will not rebind the udp listener when the network moves",
"error", err,
)
return
}
if changes == nil {
// This platform's sockets don't go stale, so there is nothing to watch for.
return
}
m.l.Info("Watching for network changes to rebind the udp listener")
for range changes {
m.l.Info("Local network changed, rebinding the udp listener")
rebind()
}
}
-164
View File
@@ -1,164 +0,0 @@
//go:build darwin && !ios && !e2e_testing
// +build darwin,!ios,!e2e_testing
package udp
import (
"context"
"encoding/binary"
"errors"
"log/slog"
"os"
"time"
"golang.org/x/sys/unix"
)
const (
// netChangeSettleWindow is how long we keep swallowing routing messages after the first interesting one. A
// single network change is never a single message, it is a burst: the link drops, addresses go away, new ones
// arrive, routes get rewritten. Reporting part way through that just means reporting again.
netChangeSettleWindow = time.Second
// netChangeReadBuffer is sized well past any rt_msghdr plus its addresses. A short read would be discarded by
// the kernel, so being generous here is how we avoid missing a message.
netChangeReadBuffer = 4096
)
// watchNetworkChanges reports when the local network moves out from under us, so the listener can be rebound.
//
// Darwin scopes a udp socket to whatever interface it came up on. Move between networks and we keep sending out an
// interface that no longer has a route, which surfaces as an instant "no route to host" with no packet ever leaving
// the box. Rebind clears that, but only if something notices the change and calls it. iOS has always been told by
// the host app off NWPathMonitor. This is the equivalent for everything else that runs on darwin.
//
// The returned channel is buffered and coalescing: a send is dropped if one is already pending, since both mean the
// same thing to a reader. It is closed when ctx is cancelled or the routing socket fails, so a caller can simply
// range over it. Platforms whose sockets do not need rebinding return a nil channel and no error.
func watchNetworkChanges(ctx context.Context, l *slog.Logger) (<-chan struct{}, error) {
sock, err := openRouteSocket()
if err != nil {
return nil, err
}
changes := make(chan struct{}, 1)
go func() {
defer close(changes)
defer func() { _ = sock.Close() }()
// Closing the socket is what unblocks the read in watchRouteSocket, so this turns cancellation into a
// close. It is scoped to this call so it cannot outlive the watch it belongs to.
done := make(chan struct{})
defer close(done)
go func() {
select {
case <-ctx.Done():
_ = sock.Close()
case <-done:
}
}()
watchRouteSocket(l, sock, changes)
}()
return changes, nil
}
// watchRouteSocket blocks reading the routing socket, reporting once per settled burst of changes. It returns when
// the socket is closed, which is how cancellation gets us out of here.
func watchRouteSocket(l *slog.Logger, sock *os.File, changes chan<- struct{}) {
buf := make([]byte, netChangeReadBuffer)
for {
n, err := sock.Read(buf)
if err != nil {
logRouteSocketError(l, err)
return
}
if !isNetworkChange(buf[:n]) {
continue
}
// Swallow the rest of the burst. The deadline is absolute and not extended by what arrives, so this always
// ends after the settle window no matter how chatty the socket is. Changes that land after the window
// simply produce another report, which is the correct outcome anyway.
deadline := time.Now().Add(netChangeSettleWindow)
for {
if err = sock.SetReadDeadline(deadline); err != nil {
logRouteSocketError(l, err)
return
}
if _, err = sock.Read(buf); err != nil {
if os.IsTimeout(err) {
break
}
logRouteSocketError(l, err)
return
}
}
if err = sock.SetReadDeadline(time.Time{}); err != nil {
logRouteSocketError(l, err)
return
}
select {
case changes <- struct{}{}:
default:
// One already pending, and a second "the network moved" tells the reader nothing new.
}
}
}
// logRouteSocketError reports a routing socket failure unless it is just us shutting the socket down.
func logRouteSocketError(l *slog.Logger, err error) {
if errors.Is(err, os.ErrClosed) {
return
}
l.Error("Error reading the routing socket, will no longer notice local network changes", "error", err)
}
// openRouteSocket returns the routing socket as a non blocking os.File. Going through os.File puts reads on the go
// poller, which buys us both a working read deadline and a Close that unblocks a read in progress.
func openRouteSocket() (*os.File, error) {
fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
if err != nil {
return nil, err
}
if err = unix.SetNonblock(fd, true); err != nil {
_ = unix.Close(fd)
return nil, err
}
return os.NewFile(uintptr(fd), "route"), nil
}
// isNetworkChange reports whether a routing message means our local addressing may have moved out from under us.
//
// We read the header instead of parsing the message because the type is the only part we need, and a full parse can
// fail on shapes we don't care about, which would turn "a message I can't parse" into "a change I missed".
// rt_msghdr, if_msghdr and ifa_msghdr all begin with the same three fields, so this is the same for every type.
func isNetworkChange(msg []byte) bool {
if len(msg) < 4 {
return false
}
// u_short msglen, u_char version, u_char type
if int(binary.NativeEndian.Uint16(msg[0:2])) > len(msg) || msg[2] != unix.RTM_VERSION {
return false
}
switch msg[3] {
case unix.RTM_NEWADDR, unix.RTM_DELADDR, unix.RTM_IFINFO:
// An address arrived or left, or a link changed state. Anything else on this socket is either a route
// churning underneath us, which a rebind doesn't help with, or unrelated traffic.
return true
default:
return false
}
}
-244
View File
@@ -1,244 +0,0 @@
//go:build darwin && !ios && !e2e_testing
// +build darwin,!ios,!e2e_testing
package udp
import (
"context"
"encoding/binary"
"os"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
"golang.org/x/sys/unix"
)
// routeMsg builds the first four bytes of a routing message, which is all isNetworkChange reads.
func routeMsg(msgType uint8, extra int) []byte {
msg := make([]byte, 4+extra)
binary.NativeEndian.PutUint16(msg[0:2], uint16(len(msg)))
msg[2] = unix.RTM_VERSION
msg[3] = msgType
return msg
}
func TestIsNetworkChange(t *testing.T) {
// The three that mean our addressing may have moved
assert.True(t, isNetworkChange(routeMsg(unix.RTM_NEWADDR, 0)))
assert.True(t, isNetworkChange(routeMsg(unix.RTM_DELADDR, 0)))
assert.True(t, isNetworkChange(routeMsg(unix.RTM_IFINFO, 0)))
// Route churn is not something a rebind helps with
assert.False(t, isNetworkChange(routeMsg(unix.RTM_ADD, 0)))
assert.False(t, isNetworkChange(routeMsg(unix.RTM_DELETE, 0)))
assert.False(t, isNetworkChange(routeMsg(unix.RTM_GET, 0)))
// Garbage must not be mistaken for a change
assert.False(t, isNetworkChange(nil), "empty")
assert.False(t, isNetworkChange([]byte{0, 0, 0}), "short header")
wrongVersion := routeMsg(unix.RTM_NEWADDR, 0)
wrongVersion[2] = unix.RTM_VERSION + 1
assert.False(t, isNetworkChange(wrongVersion), "wrong rtm_version")
lying := routeMsg(unix.RTM_NEWADDR, 0)
binary.NativeEndian.PutUint16(lying[0:2], 512)
assert.False(t, isNetworkChange(lying), "msglen longer than what we read")
}
// socketPair returns a connected pair of datagram sockets, the first wrapped the same way the routing socket is. It
// stands in for the kernel so the watch loop can be driven with synthetic messages.
func socketPair(t *testing.T) (*os.File, int) {
t.Helper()
fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_DGRAM, 0)
require.NoError(t, err)
require.NoError(t, unix.SetNonblock(fds[0], true))
f := os.NewFile(uintptr(fds[0]), "route")
t.Cleanup(func() {
_ = f.Close()
_ = unix.Close(fds[1])
})
return f, fds[1]
}
func TestWatchRouteSocketCoalescesABurst(t *testing.T) {
sock, kernel := socketPair(t)
changes := make(chan struct{}, 1)
done := make(chan struct{})
go func() {
watchRouteSocket(test.NewLogger(), sock, changes)
close(done)
}()
// One network change is a burst of messages. All of these land inside the settle window, so they must produce
// exactly one report rather than one apiece.
for range 5 {
_, err := unix.Write(kernel, routeMsg(unix.RTM_NEWADDR, 8))
require.NoError(t, err)
}
// Uninteresting messages in the middle of a burst must not add a report of their own either.
_, err := unix.Write(kernel, routeMsg(unix.RTM_ADD, 8))
require.NoError(t, err)
select {
case <-changes:
case <-time.After(netChangeSettleWindow * 4):
t.Fatal("a burst should have reported a change")
}
// Nothing more from that burst
select {
case <-changes:
t.Fatal("a burst should report exactly once")
case <-time.After(netChangeSettleWindow):
}
// A change after the window has closed is a separate event and gets its own report.
_, err = unix.Write(kernel, routeMsg(unix.RTM_IFINFO, 8))
require.NoError(t, err)
select {
case <-changes:
case <-time.After(netChangeSettleWindow * 4):
t.Fatal("a later change should report again")
}
// Closing the socket is how the real thing shuts down
require.NoError(t, sock.Close())
select {
case <-done:
case <-time.After(time.Second * 5):
t.Fatal("watchRouteSocket did not return after the socket was closed")
}
}
func TestWatchRouteSocketIgnoresUninterestingMessages(t *testing.T) {
sock, kernel := socketPair(t)
changes := make(chan struct{}, 1)
done := make(chan struct{})
go func() {
watchRouteSocket(test.NewLogger(), sock, changes)
close(done)
}()
for _, msgType := range []uint8{unix.RTM_ADD, unix.RTM_DELETE, unix.RTM_GET, unix.RTM_MISS} {
_, err := unix.Write(kernel, routeMsg(msgType, 8))
require.NoError(t, err)
}
select {
case <-changes:
t.Fatal("route churn alone must not report a change")
case <-time.After(netChangeSettleWindow * 2):
}
require.NoError(t, sock.Close())
select {
case <-done:
case <-time.After(time.Second * 5):
t.Fatal("watchRouteSocket did not return after the socket was closed")
}
}
// TestWatchRouteSocketDropsRatherThanBlocks covers the coalescing send. A reader that is busy rebinding must not
// wedge the watcher, and a second pending "the network moved" tells it nothing new anyway.
func TestWatchRouteSocketDropsRatherThanBlocks(t *testing.T) {
sock, kernel := socketPair(t)
changes := make(chan struct{}, 1)
done := make(chan struct{})
go func() {
watchRouteSocket(test.NewLogger(), sock, changes)
close(done)
}()
// Nobody is reading changes, so after the first report the buffer is full for the rest of this test
for range 3 {
_, err := unix.Write(kernel, routeMsg(unix.RTM_NEWADDR, 8))
require.NoError(t, err)
time.Sleep(netChangeSettleWindow + time.Millisecond*250)
}
// The watcher must still be alive and responsive to a close
require.NoError(t, sock.Close())
select {
case <-done:
case <-time.After(time.Second * 5):
t.Fatal("watchRouteSocket wedged on a full channel")
}
assert.Len(t, changes, 1, "the pending report should have coalesced, not queued")
}
// TestWatchNetworkChangesStopsWithContext covers the detection path against a real routing socket, including that
// cancelling the context closes the channel so a ranging caller falls out of its loop.
func TestWatchNetworkChangesStopsWithContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
changes, err := watchNetworkChanges(ctx, test.NewLogger())
require.NoError(t, err)
require.NotNil(t, changes, "darwin should support watching")
drained := make(chan struct{})
go func() {
for range changes {
}
close(drained)
}()
cancel()
select {
case <-drained:
case <-time.After(time.Second * 5):
t.Fatal("cancelling the context should close the changes channel")
}
}
// TestNetworkChangeMonitorStopsWithContext drives the whole monitor against a real routing socket: Start must block
// watching, and cancelling the context (which is all Control does on shutdown, it never stops the monitor directly)
// must return it and clean up the watch goroutines.
func TestNetworkChangeMonitorStopsWithContext(t *testing.T) {
// IgnoreCurrent because other tests in this package leave readers running; we only care about what this test
// leaks itself.
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
ctx, cancel := context.WithCancel(context.Background())
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("listen:\n rebind_on_network_change: true\n"))
m := NewNetworkChangeMonitor(ctx, l, c)
done := make(chan struct{})
go func() {
m.Start(func() {})
close(done)
}()
// Start should be sitting on the routing socket, not have fallen out. If it returned early it either failed to
// watch or no-op'd, both of which we want to catch.
select {
case <-done:
t.Fatal("Start returned instead of watching")
case <-time.After(time.Millisecond * 250):
}
cancel()
select {
case <-done:
case <-time.After(time.Second * 5):
t.Fatal("Start did not return after the context was cancelled")
}
// Starting again after the context is dead must not open anything.
m.Start(func() {})
}
-22
View File
@@ -1,22 +0,0 @@
//go:build !darwin || ios || e2e_testing
// +build !darwin ios e2e_testing
package udp
import (
"context"
"log/slog"
)
// watchNetworkChanges is a no-op outside of darwin.
//
// Darwin is the platform that scopes a udp socket to the interface it came up on, so it is the platform whose socket
// goes stale when the local network changes. Everywhere else Rebind has nothing to do, so there is nothing to watch
// for. iOS is excluded on purpose even though it is darwin: the host app already drives the rebind off NWPathMonitor,
// and two things racing to rebind the same socket is worse than one.
//
// A nil channel means "not supported here", which callers must treat as "do not start a watcher" rather than
// selecting on it, since a receive from a nil channel blocks forever.
func watchNetworkChanges(_ context.Context, _ *slog.Logger) (<-chan struct{}, error) {
return nil, nil
}
-39
View File
@@ -1,39 +0,0 @@
package udp
import (
"context"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func newMonitor(t *testing.T, ctx context.Context, cfg string) *NetworkChangeMonitor {
t.Helper()
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString(cfg))
return NewNetworkChangeMonitor(ctx, l, c)
}
func TestNetworkChangeMonitorDefaultsOn(t *testing.T) {
// Says nothing about rebinding, so this covers the default.
m := newMonitor(t, context.Background(), "listen:\n host: 0.0.0.0\n")
assert.True(t, m.enabled, "should default to on")
}
func TestNetworkChangeMonitorDisabledIsANoOp(t *testing.T) {
m := newMonitor(t, context.Background(), "listen:\n rebind_on_network_change: false\n")
require.False(t, m.enabled)
// Must return without opening a socket. If it watched anything this would block.
m.Start(func() {})
}
func TestNetworkChangeMonitorNilRebindIsANoOp(t *testing.T) {
// Nothing to rebind, so there is no point watching, on any platform.
m := newMonitor(t, context.Background(), "listen:\n rebind_on_network_change: true\n")
m.Start(nil)
}
+5 -4
View File
@@ -187,9 +187,6 @@ func (u *StdConn) SupportsMultipleReaders() bool {
return false return false
} }
// Rebind clears the interface the kernel scoped this socket to, so that sends are routed against the current
// routing table instead of the interface we happened to be on when the socket was created. Darwin pins sockets
// this way on its own, which is what strands us after the underlying network changes.
func (u *StdConn) Rebind() error { func (u *StdConn) Rebind() error {
var err error var err error
if u.isV4 { if u.isV4 {
@@ -198,5 +195,9 @@ func (u *StdConn) Rebind() error {
err = syscall.SetsockoptInt(int(u.sysFd), syscall.IPPROTO_IPV6, syscall.IPV6_BOUND_IF, 0) err = syscall.SetsockoptInt(int(u.sysFd), syscall.IPPROTO_IPV6, syscall.IPV6_BOUND_IF, 0)
} }
return err if err != nil {
u.l.Error("Failed to rebind udp socket", "error", err)
}
return nil
} }
+166 -167
View File
@@ -4,13 +4,12 @@
package udp package udp
import ( import (
"context"
"encoding/binary" "encoding/binary"
"errors"
"fmt" "fmt"
"log/slog" "log/slog"
"net" "net"
"net/netip" "net/netip"
"sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -20,51 +19,58 @@ import (
) )
type StdConn struct { type StdConn struct {
sysFd int udpConn *net.UDPConn
closed atomic.Bool rawConn syscall.RawConn
isV4 bool isV4 bool
l *slog.Logger l *slog.Logger
batch int batch int
}
func setReusePort(network, address string, c syscall.RawConn) error {
var opErr error
err := c.Control(func(fd uintptr) {
opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_REUSEPORT, 1)
//CloseOnExec already set by the runtime
})
if err != nil {
return err
}
return opErr
} }
func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) { func NewListener(l *slog.Logger, ip netip.Addr, port int, multi bool, batch int) (Conn, error) {
af := unix.AF_INET6 listen := netip.AddrPortFrom(ip, uint16(port))
if ip.Is4() { lc := net.ListenConfig{}
af = unix.AF_INET
}
syscall.ForkLock.RLock()
fd, err := unix.Socket(af, unix.SOCK_DGRAM, unix.IPPROTO_UDP)
if err == nil {
unix.CloseOnExec(fd)
}
syscall.ForkLock.RUnlock()
if err != nil {
return nil, fmt.Errorf("unable to open socket: %w", err)
}
if multi { if multi {
if err = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1); err != nil { lc.Control = setReusePort
_ = unix.Close(fd) }
return nil, fmt.Errorf("unable to set SO_REUSEPORT: %w", err) //this context is only used during the bind operation, you can't cancel it to kill the socket
} pc, err := lc.ListenPacket(context.Background(), "udp", listen.String())
if err != nil {
return nil, fmt.Errorf("unable to open socket: %s", err)
}
udpConn := pc.(*net.UDPConn)
rawConn, err := udpConn.SyscallConn()
if err != nil {
_ = udpConn.Close()
return nil, err
}
//gotta find out if we got an AF_INET6 socket or not:
out := &StdConn{
udpConn: udpConn,
rawConn: rawConn,
l: l,
batch: batch,
} }
var sa unix.Sockaddr af, err := out.getSockOptInt(unix.SO_DOMAIN)
if ip.Is4() { if err != nil {
sa4 := &unix.SockaddrInet4{Port: port} _ = out.Close()
sa4.Addr = ip.As4() return nil, err
sa = sa4
} else {
sa6 := &unix.SockaddrInet6{Port: port}
sa6.Addr = ip.As16()
sa = sa6
}
if err = unix.Bind(fd, sa); err != nil {
_ = unix.Close(fd)
return nil, fmt.Errorf("unable to bind to socket: %w", err)
} }
out.isV4 = af == unix.AF_INET
return &StdConn{sysFd: fd, isV4: ip.Is4(), l: l, batch: batch}, nil return out, nil
} }
func (u *StdConn) SupportsMultipleReaders() bool { func (u *StdConn) SupportsMultipleReaders() bool {
@@ -75,111 +81,134 @@ func (u *StdConn) Rebind() error {
return nil return nil
} }
func (u *StdConn) getSockOptInt(opt int) (int, error) {
if u.rawConn == nil {
return 0, fmt.Errorf("no UDP connection")
}
var out int
var opErr error
err := u.rawConn.Control(func(fd uintptr) {
out, opErr = unix.GetsockoptInt(int(fd), unix.SOL_SOCKET, opt)
})
if err != nil {
return 0, err
}
return out, opErr
}
func (u *StdConn) setSockOptInt(opt int, n int) error {
if u.rawConn == nil {
return fmt.Errorf("no UDP connection")
}
var opErr error
err := u.rawConn.Control(func(fd uintptr) {
opErr = unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, opt, n)
})
if err != nil {
return err
}
return opErr
}
func (u *StdConn) SetRecvBuffer(n int) error { func (u *StdConn) SetRecvBuffer(n int) error {
return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUFFORCE, n) return u.setSockOptInt(unix.SO_RCVBUFFORCE, n)
} }
func (u *StdConn) SetSendBuffer(n int) error { func (u *StdConn) SetSendBuffer(n int) error {
return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUFFORCE, n) return u.setSockOptInt(unix.SO_SNDBUFFORCE, n)
} }
func (u *StdConn) SetSoMark(mark int) error { func (u *StdConn) SetSoMark(mark int) error {
return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK, mark) return u.setSockOptInt(unix.SO_MARK, mark)
} }
func (u *StdConn) GetRecvBuffer() (int, error) { func (u *StdConn) GetRecvBuffer() (int, error) {
return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUF) return u.getSockOptInt(unix.SO_RCVBUF)
} }
func (u *StdConn) GetSendBuffer() (int, error) { func (u *StdConn) GetSendBuffer() (int, error) {
return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUF) return u.getSockOptInt(unix.SO_SNDBUF)
} }
func (u *StdConn) GetSoMark() (int, error) { func (u *StdConn) GetSoMark() (int, error) {
return unix.GetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_MARK) return u.getSockOptInt(unix.SO_MARK)
} }
func (u *StdConn) LocalAddr() (netip.AddrPort, error) { func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
sa, err := unix.Getsockname(u.sysFd) a := u.udpConn.LocalAddr()
if err != nil {
return netip.AddrPort{}, err switch v := a.(type) {
} case *net.UDPAddr:
switch sa := sa.(type) { addr, ok := netip.AddrFromSlice(v.IP)
case *unix.SockaddrInet4: if !ok {
return netip.AddrPortFrom(netip.AddrFrom4(sa.Addr), uint16(sa.Port)), nil return netip.AddrPort{}, fmt.Errorf("LocalAddr returned invalid IP address: %s", v.IP)
case *unix.SockaddrInet6: }
return netip.AddrPortFrom(netip.AddrFrom16(sa.Addr), uint16(sa.Port)), nil return netip.AddrPortFrom(addr, uint16(v.Port)), nil
default: default:
return netip.AddrPort{}, fmt.Errorf("unsupported sock type: %T", sa) return netip.AddrPort{}, fmt.Errorf("LocalAddr returned: %#v", a)
} }
} }
// recvmmsg does one blocking recvmmsg (MSG_WAITFORONE), reading up to len(msgs) datagrams func recvmmsg(fd uintptr, msgs []rawMessage) (int, bool, error) {
func (u *StdConn) recvmmsg(msgs []rawMessage) (int, error) { var errno syscall.Errno
r, _, errno := unix.Syscall6( n, _, errno := unix.Syscall6(
unix.SYS_RECVMMSG, unix.SYS_RECVMMSG,
uintptr(u.sysFd), fd,
uintptr(unsafe.Pointer(&msgs[0])), uintptr(unsafe.Pointer(&msgs[0])),
uintptr(len(msgs)), uintptr(len(msgs)),
unix.MSG_WAITFORONE, unix.MSG_WAITFORONE,
0, 0,
0, 0,
) )
if errno == syscall.EAGAIN || errno == syscall.EWOULDBLOCK {
// No data available, block for I/O and try again.
return int(n), false, nil
}
if errno != 0 { if errno != 0 {
if u.closed.Load() { return int(n), true, &net.OpError{Op: "recvmmsg", Err: errno}
return 0, net.ErrClosed
}
return 0, &net.OpError{Op: "recvmmsg", Err: errno}
} }
n := int(r) return int(n), true, nil
if (n == 0 || msgs[0].Len == 0) && u.closed.Load() {
return 0, net.ErrClosed
}
return n, nil
} }
// recvmsg does one blocking recvmsg into msgs[0] func (u *StdConn) listenOutSingle(r EncReader) error {
func (u *StdConn) recvmsg(msgs []rawMessage) (int, error) { var err error
r, _, errno := unix.Syscall6( var n int
unix.SYS_RECVMSG, var from netip.AddrPort
uintptr(u.sysFd), buffer := make([]byte, MTU)
uintptr(unsafe.Pointer(&msgs[0].Hdr)),
0, for {
0, n, from, err = u.udpConn.ReadFromUDPAddrPort(buffer)
0, if err != nil {
0, return err
)
if errno != 0 {
if u.closed.Load() {
return 0, net.ErrClosed
} }
return 0, &net.OpError{Op: "recvmsg", Err: errno} from = netip.AddrPortFrom(from.Addr().Unmap(), from.Port())
r(from, buffer[:n])
} }
if r == 0 && u.closed.Load() {
return 0, net.ErrClosed
}
msgs[0].Len = uint32(r)
return 1, nil
} }
func (u *StdConn) ListenOut(r EncReader) error { func (u *StdConn) listenOutBatch(r EncReader) error {
var ip netip.Addr var ip netip.Addr
var n int
var operr error
msgs, buffers, names := u.PrepareRawMessages(u.batch) msgs, buffers, names := u.PrepareRawMessages(u.batch)
read := u.recvmmsg
if u.batch == 1 { //reader needs to capture variables from this function, since it's used as a lambda with rawConn.Read
read = u.recvmsg //defining it outside the loop so it gets re-used
reader := func(fd uintptr) (done bool) {
n, done, operr = recvmmsg(fd, msgs)
return done
} }
for { for {
n, err := read(msgs) err := u.rawConn.Read(reader)
if err != nil { if err != nil {
if errors.Is(err, unix.EINTR) {
continue // interrupted by a signal, retry the read
}
// net.ErrClosed after Close() is teardown, absorbed by the caller's
// closed flag like the other platforms; anything else is a real error.
return err return err
} }
if operr != nil {
return operr
}
for i := 0; i < n; i++ { for i := 0; i < n; i++ {
// Its ok to skip the ok check here, the slicing is the only error that can occur and it will panic // Its ok to skip the ok check here, the slicing is the only error that can occur and it will panic
@@ -193,68 +222,26 @@ func (u *StdConn) ListenOut(r EncReader) error {
} }
} }
func (u *StdConn) ListenOut(r EncReader) error {
if u.batch == 1 {
return u.listenOutSingle(r)
} else {
return u.listenOutBatch(r)
}
}
func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error { func (u *StdConn) WriteTo(b []byte, ip netip.AddrPort) error {
if u.isV4 { _, err := u.udpConn.WriteToUDPAddrPort(b, ip)
return u.writeTo4(b, ip) return err
}
return u.writeTo6(b, ip)
}
func (u *StdConn) writeTo6(b []byte, ip netip.AddrPort) error {
var rsa unix.RawSockaddrInet6
rsa.Family = unix.AF_INET6
rsa.Addr = ip.Addr().As16()
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
for {
_, _, err := unix.Syscall6(
unix.SYS_SENDTO,
uintptr(u.sysFd),
uintptr(unsafe.Pointer(&b[0])),
uintptr(len(b)),
uintptr(0),
uintptr(unsafe.Pointer(&rsa)),
uintptr(unix.SizeofSockaddrInet6),
)
if err != 0 {
return &net.OpError{Op: "sendto", Err: err}
}
return nil
}
}
func (u *StdConn) writeTo4(b []byte, ip netip.AddrPort) error {
if !ip.Addr().Is4() {
return ErrInvalidIPv6RemoteForSocket
}
var rsa unix.RawSockaddrInet4
rsa.Family = unix.AF_INET
rsa.Addr = ip.Addr().As4()
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&rsa.Port))[:], ip.Port())
for {
_, _, err := unix.Syscall6(
unix.SYS_SENDTO,
uintptr(u.sysFd),
uintptr(unsafe.Pointer(&b[0])),
uintptr(len(b)),
uintptr(0),
uintptr(unsafe.Pointer(&rsa)),
uintptr(unix.SizeofSockaddrInet4),
)
if err != 0 {
return &net.OpError{Op: "sendto", Err: err}
}
return nil
}
} }
func (u *StdConn) ReloadConfig(c *config.C) { func (u *StdConn) ReloadConfig(c *config.C) {
b := c.GetInt("listen.read_buffer", 0) b := c.GetInt("listen.read_buffer", 0)
if b > 0 { if b > 0 {
if err := u.SetRecvBuffer(b); err == nil { err := u.SetRecvBuffer(b)
if s, err := u.GetRecvBuffer(); err == nil { if err == nil {
s, err := u.GetRecvBuffer()
if err == nil {
u.l.Info("listen.read_buffer was set", "size", s) u.l.Info("listen.read_buffer was set", "size", s)
} else { } else {
u.l.Warn("Failed to get listen.read_buffer", "error", err) u.l.Warn("Failed to get listen.read_buffer", "error", err)
@@ -266,8 +253,10 @@ func (u *StdConn) ReloadConfig(c *config.C) {
b = c.GetInt("listen.write_buffer", 0) b = c.GetInt("listen.write_buffer", 0)
if b > 0 { if b > 0 {
if err := u.SetSendBuffer(b); err == nil { err := u.SetSendBuffer(b)
if s, err := u.GetSendBuffer(); err == nil { if err == nil {
s, err := u.GetSendBuffer()
if err == nil {
u.l.Info("listen.write_buffer was set", "size", s) u.l.Info("listen.write_buffer was set", "size", s)
} else { } else {
u.l.Warn("Failed to get listen.write_buffer", "error", err) u.l.Warn("Failed to get listen.write_buffer", "error", err)
@@ -280,8 +269,10 @@ func (u *StdConn) ReloadConfig(c *config.C) {
b = c.GetInt("listen.so_mark", 0) b = c.GetInt("listen.so_mark", 0)
s, err := u.GetSoMark() s, err := u.GetSoMark()
if b > 0 || (err == nil && s != 0) { if b > 0 || (err == nil && s != 0) {
if err := u.SetSoMark(b); err == nil { err := u.SetSoMark(b)
if s, err := u.GetSoMark(); err == nil { if err == nil {
s, err := u.GetSoMark()
if err == nil {
u.l.Info("listen.so_mark was set", "mark", s) u.l.Info("listen.so_mark was set", "mark", s)
} else { } else {
u.l.Warn("Failed to get listen.so_mark", "error", err) u.l.Warn("Failed to get listen.so_mark", "error", err)
@@ -294,20 +285,28 @@ func (u *StdConn) ReloadConfig(c *config.C) {
func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error { func (u *StdConn) getMemInfo(meminfo *[unix.SK_MEMINFO_VARS]uint32) error {
var vallen uint32 = 4 * unix.SK_MEMINFO_VARS var vallen uint32 = 4 * unix.SK_MEMINFO_VARS
_, _, err := unix.Syscall6(unix.SYS_GETSOCKOPT, uintptr(u.sysFd), uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
if err != 0 { if u.rawConn == nil {
return fmt.Errorf("no UDP connection")
}
var opErr error
err := u.rawConn.Control(func(fd uintptr) {
_, _, syserr := unix.Syscall6(unix.SYS_GETSOCKOPT, fd, uintptr(unix.SOL_SOCKET), uintptr(unix.SO_MEMINFO), uintptr(unsafe.Pointer(meminfo)), uintptr(unsafe.Pointer(&vallen)), 0)
if syserr != 0 {
opErr = syserr
}
})
if err != nil {
return err return err
} }
return nil return opErr
} }
func (u *StdConn) Close() error { func (u *StdConn) Close() error {
u.closed.Store(true) if u.udpConn != nil {
// Wake the reader parked in recvmmsg/recvmsg. shutdown(2) on an unconnected socket return u.udpConn.Close()
// returns ENOTCONN but still wakes it, so ignore the error. }
// The reader then sees closed and stops touching the fd, making the Close below safe. return nil
_ = unix.Shutdown(u.sysFd, unix.SHUT_RDWR)
return unix.Close(u.sysFd)
} }
func NewUDPStatsEmitter(udpConns []Conn) func() { func NewUDPStatsEmitter(udpConns []Conn) func() {
-179
View File
@@ -1,179 +0,0 @@
//go:build linux && !android && !e2e_testing
package udp
import (
"errors"
"fmt"
"log/slog"
"net"
"net/netip"
"os"
"runtime"
"sync/atomic"
"testing"
"time"
"golang.org/x/sys/unix"
)
func testLogger() *slog.Logger {
return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{Level: slog.LevelError}))
}
// TestShutdownWakesAfterRx_Mechanism exercises the kernel quirk our teardown
// relies on: once a socket has received a packet, shutdown(2) wakes a blocked
// recvmmsg with n>=1/Len==0 (not n==0). recvmmsg must turn that into net.ErrClosed
// once Close set closed, so a parked reader exits instead of spinning.
func TestShutdownWakesAfterRx_Mechanism(t *testing.T) {
c, err := NewListener(testLogger(), netip.MustParseAddr("127.0.0.1"), 0, true, 64)
if err != nil {
t.Fatalf("NewListener: %v", err)
}
sc := c.(*StdConn)
addr, err := sc.LocalAddr()
if err != nil {
t.Fatalf("LocalAddr: %v", err)
}
msgs, _, _ := sc.PrepareRawMessages(sc.batch)
// Receive a real packet so the socket has carried data.
send, err := net.Dial("udp", addr.String())
if err != nil {
t.Fatalf("dial: %v", err)
}
if _, err := send.Write([]byte("hello")); err != nil {
t.Fatalf("write: %v", err)
}
time.Sleep(50 * time.Millisecond)
n, err := sc.recvmmsg(msgs)
t.Logf("drain of real packet: n=%d err=%v msgs[0].Len=%d", n, err, msgs[0].Len)
_ = send.Close()
// Block a reader on the now-empty queue, then tear down as Close() does.
// recvmmsg must return net.ErrClosed (not hang, not spin) even post-rx.
done := make(chan error, 1)
go func() {
_, err := sc.recvmmsg(msgs)
done <- err
}()
time.Sleep(150 * time.Millisecond) // let it park in recvmmsg
sc.closed.Store(true)
if serr := unix.Shutdown(sc.sysFd, unix.SHUT_RDWR); serr != nil {
t.Logf("shutdown returned %v (expected ENOTCONN on unconnected UDP)", serr)
}
select {
case err := <-done:
if !errors.Is(err, net.ErrClosed) {
t.Errorf("recvmmsg after post-rx shutdown returned %v, want net.ErrClosed", err)
}
case <-time.After(2 * time.Second):
t.Fatalf("HANG: recvmmsg did not return after shutdown following a received packet")
}
_ = unix.Close(sc.sysFd)
}
// TestListenOutTeardown_TrafficPatterns reproduces the field report: a blocking
// reader must tear down cleanly on Close() regardless of what the socket has
// carried. The three cases the report called out:
//
// no traffic ever -> works (shutdown wakes recvmmsg with n==0)
// ping once, then idle -> historically HUNG: once the socket has received a
// packet, shutdown(2) wakes recvmmsg with n>=1/Len==0,
// which an n==0-only teardown check misses
// continuous traffic -> works (a real packet is always arriving)
//
// All three must return within the deadline; a hang dumps goroutines so the
// stuck reader is visible.
func TestListenOutTeardown_TrafficPatterns(t *testing.T) {
cases := []struct {
name string
traffic func(send net.Conn, stop <-chan struct{})
}{
{"no_traffic_ever", func(net.Conn, <-chan struct{}) {}},
{"ping_once_then_idle", func(send net.Conn, _ <-chan struct{}) {
_, _ = send.Write([]byte("hello"))
}},
{"continuous", func(send net.Conn, stop <-chan struct{}) {
for {
select {
case <-stop:
return
default:
_, _ = send.Write([]byte("hello"))
time.Sleep(2 * time.Millisecond)
}
}
}},
}
// batch 1 exercises the recvmsg path, batch 64 the recvmmsg path; both must
// tear down cleanly.
for _, batch := range []int{1, 64} {
for _, tc := range cases {
t.Run(fmt.Sprintf("batch%d/%s", batch, tc.name), func(t *testing.T) {
runTeardownCase(t, batch, tc.name, tc.traffic)
})
}
}
}
func runTeardownCase(t *testing.T, batch int, name string, traffic func(send net.Conn, stop <-chan struct{})) {
c, err := NewListener(testLogger(), netip.MustParseAddr("127.0.0.1"), 0, true, batch)
if err != nil {
t.Fatalf("NewListener: %v", err)
}
sc := c.(*StdConn)
addr, err := sc.LocalAddr()
if err != nil {
t.Fatalf("LocalAddr: %v", err)
}
var received atomic.Int64
loopDone := make(chan error, 1)
go func() {
loopDone <- sc.ListenOut(func(netip.AddrPort, []byte) {
received.Add(1)
})
}()
send, err := net.Dial("udp", addr.String())
if err != nil {
t.Fatalf("dial: %v", err)
}
defer send.Close()
stop := make(chan struct{})
trafficDone := make(chan struct{})
go func() {
traffic(send, stop)
close(trafficDone)
}()
// Let the pattern run and, for the idle case, the reader park again on an
// empty queue with the socket already having received a packet.
time.Sleep(500 * time.Millisecond)
start := time.Now()
if err := sc.Close(); err != nil {
t.Fatalf("Close: %v", err)
}
close(stop)
select {
case err := <-loopDone:
// Clean teardown surfaces as net.ErrClosed (propagated like the other
// platforms); the caller absorbs it via its closed flag.
if err != nil && !errors.Is(err, net.ErrClosed) {
t.Fatalf("%s: ListenOut returned unexpected error on teardown: %v", name, err)
}
t.Logf("%s: closed in %v (received %d packets)", name, time.Since(start), received.Load())
case <-time.After(3 * time.Second):
buf := make([]byte, 1<<20)
n := runtime.Stack(buf, true)
t.Fatalf("%s: HANG, ListenOut did not return within 3s of Close\n%s", name, buf[:n])
}
<-trafficDone
}
+5 -18
View File
@@ -31,8 +31,7 @@ func procyield(cycles uint32)
const ( const (
packetsPerRing = 1024 packetsPerRing = 1024
// Caps tun.mtu at MTU-32 direct, MTU-64 relayed, unenforced anywhere else. 17.6MB page locked per socket. bytesPerPacket = 2048 - 32
bytesPerPacket = MTU
receiveSpins = 15 receiveSpins = 15
) )
@@ -70,14 +69,12 @@ func NewRIOListener(l *slog.Logger, addr netip.Addr, port int) (*RIOConn, error)
err := u.bind(l, &windows.SockaddrInet6{Addr: addr.As16(), Port: port}) err := u.bind(l, &windows.SockaddrInet6{Addr: addr.As16(), Port: port})
if err != nil { if err != nil {
u.close()
return nil, fmt.Errorf("bind: %w", err) return nil, fmt.Errorf("bind: %w", err)
} }
for i := 0; i < packetsPerRing; i++ { for i := 0; i < packetsPerRing; i++ {
err = u.insertReceiveRequest() err = u.insertReceiveRequest()
if err != nil { if err != nil {
u.close()
return nil, fmt.Errorf("init rx ring: %w", err) return nil, fmt.Errorf("init rx ring: %w", err)
} }
} }
@@ -345,25 +342,15 @@ func (u *RIOConn) Close() error {
return nil return nil
} }
u.close()
return nil
}
// Also unwinds a partial build from NewRIOListener, where isOpen is false and Close would no-op.
// Socket first, unlike wireguard-go: receive() re-arms every slot, so freeing the rings under a live socket
// hands the kernel freed pages for all packetsPerRing outstanding receives.
func (u *RIOConn) close() {
// WSASocket reports failure as InvalidHandle, not zero.
if u.sock != 0 && u.sock != windows.InvalidHandle {
windows.CloseHandle(u.sock)
}
u.sock = 0
windows.PostQueuedCompletionStatus(u.rx.iocp, 0, 0, nil) windows.PostQueuedCompletionStatus(u.rx.iocp, 0, 0, nil)
windows.PostQueuedCompletionStatus(u.tx.iocp, 0, 0, nil) windows.PostQueuedCompletionStatus(u.tx.iocp, 0, 0, nil)
u.rx.CloseAndZero() u.rx.CloseAndZero()
u.tx.CloseAndZero() u.tx.CloseAndZero()
if u.sock != 0 {
windows.CloseHandle(u.sock)
}
return nil
} }
func (ring *ringBuffer) Push() *ringPacket { func (ring *ringBuffer) Push() *ringPacket {
+6 -20
View File
@@ -10,7 +10,6 @@ import (
"net/netip" "net/netip"
"os" "os"
"sync" "sync"
"sync/atomic"
"github.com/slackhq/nebula/config" "github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header" "github.com/slackhq/nebula/header"
@@ -65,9 +64,7 @@ func acquirePacket() *Packet {
} }
type TesterConn struct { type TesterConn struct {
// addr is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can Addr netip.AddrPort
// move it mid-run to simulate roaming, so it is atomic rather than a plain field.
addr atomic.Pointer[netip.AddrPort]
RxPackets chan *Packet // Packets to receive into nebula RxPackets chan *Packet // Packets to receive into nebula
TxPackets chan *Packet // Packets transmitted outside by nebula TxPackets chan *Packet // Packets transmitted outside by nebula
@@ -85,24 +82,13 @@ type TesterConn struct {
} }
func NewListener(l *slog.Logger, ip netip.Addr, port int, _ bool, _ int) (Conn, error) { func NewListener(l *slog.Logger, ip netip.Addr, port int, _ bool, _ int) (Conn, error) {
c := &TesterConn{ return &TesterConn{
Addr: netip.AddrPortFrom(ip, uint16(port)),
RxPackets: make(chan *Packet, 10), RxPackets: make(chan *Packet, 10),
TxPackets: make(chan *Packet, 10), TxPackets: make(chan *Packet, 10),
done: make(chan struct{}), done: make(chan struct{}),
l: l, l: l,
} }, nil
c.SetAddr(netip.AddrPortFrom(ip, uint16(port)))
return c, nil
}
// GetAddr returns the underlay address this conn currently sends from.
func (u *TesterConn) GetAddr() netip.AddrPort {
return *u.addr.Load()
}
// SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network.
func (u *TesterConn) SetAddr(addr netip.AddrPort) {
u.addr.Store(&addr)
} }
// Send will place a UdpPacket onto the receive queue for nebula to consume // Send will place a UdpPacket onto the receive queue for nebula to consume
@@ -161,7 +147,7 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
p.Data = p.Data[:len(b)] p.Data = p.Data[:len(b)]
} }
copy(p.Data, b) copy(p.Data, b)
p.From = u.GetAddr() p.From = u.Addr
p.To = addr p.To = addr
select { select {
case <-u.done: case <-u.done:
@@ -192,7 +178,7 @@ func NewUDPStatsEmitter(_ []Conn) func() {
} }
func (u *TesterConn) LocalAddr() (netip.AddrPort, error) { func (u *TesterConn) LocalAddr() (netip.AddrPort, error) {
return u.GetAddr(), nil return u.Addr, nil
} }
func (u *TesterConn) SupportsMultipleReaders() bool { func (u *TesterConn) SupportsMultipleReaders() bool {