mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-16 23:06:52 +01:00
Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4dbeff889c | ||
|
|
673f59b2b2 | ||
|
|
3871faf26e | ||
|
|
b66c9bb362 | ||
|
|
501fda7a40 | ||
|
|
992e36b678 | ||
|
|
5d7c8c905b | ||
|
|
c2ad75e459 | ||
|
|
0a916b1d4b | ||
|
|
33abaf95cd | ||
|
|
20c68ef3c4 | ||
|
|
f46abae87e | ||
|
|
ae688d038d | ||
|
|
1617897043 | ||
|
|
f8775bb6ca | ||
|
|
15f0f0d5d0 | ||
|
|
7902ce674e |
No files matched your search
@@ -43,8 +43,15 @@ runs:
|
||||
with:
|
||||
role-to-assume: ${{ inputs.role }}
|
||||
aws-region: ${{ inputs.region }}
|
||||
# Default is 12 retries to ride out IAM trust-policy propagation; once
|
||||
# the role is stable we want a real misconfiguration to fail fast.
|
||||
# An STS secret key with special characters does not survive the
|
||||
# pwsh -> make -> MSYS sh -> aws.exe chain, and SigV4 then signs with a
|
||||
# key that no longer matches, so the first S3 upload fails with
|
||||
# SignatureDoesNotMatch. Retries the assume until it comes back clean.
|
||||
# Same fix as DefinedNet/dnclient#867.
|
||||
special-characters-workaround: true
|
||||
# Overridden by the workaround above and kept for whenever that goes:
|
||||
# the default 12 rides out IAM trust-policy propagation, and once the
|
||||
# role is stable a real misconfiguration should fail fast.
|
||||
retry-max-attempts: 5
|
||||
|
||||
- name: Sign .exe files
|
||||
|
||||
@@ -14,7 +14,7 @@ jobs:
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '1.25'
|
||||
go-version: '1.26'
|
||||
check-latest: true
|
||||
|
||||
- name: Build
|
||||
@@ -40,7 +40,7 @@ jobs:
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '1.25'
|
||||
go-version: '1.26'
|
||||
check-latest: true
|
||||
|
||||
- name: Build
|
||||
@@ -73,27 +73,81 @@ jobs:
|
||||
build-darwin:
|
||||
name: Build Universal Darwin
|
||||
env:
|
||||
HAS_SIGNING_CREDS: ${{ secrets.AC_USERNAME != '' }}
|
||||
HAS_SIGNING_CREDS: ${{ secrets.APPLE_SIGNING_ROLE_ARN != '' }}
|
||||
runs-on: macos-latest
|
||||
permissions:
|
||||
id-token: write
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '1.25'
|
||||
go-version: '1.26'
|
||||
check-latest: true
|
||||
|
||||
# GitHub holds ARNs, not credentials, and ARNs outlive a rotation
|
||||
- name: Configure AWS credentials
|
||||
if: env.HAS_SIGNING_CREDS == 'true'
|
||||
uses: aws-actions/configure-aws-credentials@v6
|
||||
with:
|
||||
role-to-assume: ${{ secrets.APPLE_SIGNING_ROLE_ARN }}
|
||||
aws-region: us-east-2
|
||||
|
||||
# parse-json-secrets unpacks into SIGNING_* and ASC_*, masked on the way in
|
||||
- name: Fetch signing credentials
|
||||
if: env.HAS_SIGNING_CREDS == 'true'
|
||||
uses: aws-actions/aws-secretsmanager-get-secrets@v3
|
||||
with:
|
||||
parse-json-secrets: true
|
||||
secret-ids: |
|
||||
SIGNING,${{ secrets.APPLE_SIGNING_DEVELOPER_ID_ARN }}
|
||||
ASC,${{ secrets.APPLE_NOTARY_KEY_ARN }}
|
||||
|
||||
- name: Import certificates
|
||||
if: env.HAS_SIGNING_CREDS == 'true'
|
||||
uses: Apple-Actions/import-codesign-certs@v7
|
||||
with:
|
||||
p12-file-base64: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_P12_BASE64 }}
|
||||
p12-password: ${{ secrets.APPLE_DEVELOPER_CERTIFICATE_PASSWORD }}
|
||||
p12-file-base64: ${{ env.SIGNING_P12_BASE64 }}
|
||||
p12-password: ${{ env.SIGNING_PASSWORD }}
|
||||
|
||||
# The action imports but does not check the chain validates, which is how a p12
|
||||
# missing its intermediate reaches a failing codesign
|
||||
- name: Check the identity is usable
|
||||
if: env.HAS_SIGNING_CREDS == 'true'
|
||||
run: |
|
||||
: "${SIGNING_IDENTITY_SHA1:?empty, so the secret has no identity_sha1}"
|
||||
identities=$(security find-identity -v -p codesigning signing_temp.keychain)
|
||||
case "$identities" in
|
||||
*"$SIGNING_IDENTITY_SHA1"*) ;;
|
||||
*) printf '%s\n' "$identities" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
# notarytool wants the key as a file
|
||||
- name: Write the App Store Connect key
|
||||
if: env.HAS_SIGNING_CREDS == 'true'
|
||||
run: |
|
||||
mkdir -p ~/private_keys
|
||||
chmod 700 ~/private_keys
|
||||
key_path="$HOME/private_keys/AuthKey_${ASC_KEY_ID}.p8"
|
||||
(umask 077; printf '%s\n' "$ASC_PRIVATE_KEY" > "$key_path")
|
||||
echo "ASC_P8=$key_path" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Drop the credentials from the environment
|
||||
if: env.HAS_SIGNING_CREDS == 'true'
|
||||
run: |
|
||||
# The action's own inventory, so a new field in a secret is covered
|
||||
python3 -c '
|
||||
import json, os
|
||||
raw = os.environ.get("SECRETS_LIST_CLEAN_UP")
|
||||
if raw is None and os.environ.get("SIGNING_P12_BASE64"):
|
||||
raise SystemExit("SECRETS_LIST_CLEAN_UP is gone, fetched secrets are not being scrubbed")
|
||||
keep = {"SIGNING_IDENTITY_SHA1", "ASC_KEY_ID", "ASC_ISSUER_ID"}
|
||||
names = [n for n in json.loads(raw or "[]") if n not in keep]
|
||||
print("\n".join(f"{n}=" for n in dict.fromkeys(names)))
|
||||
' >> "$GITHUB_ENV"
|
||||
|
||||
- name: Build, sign, and notarize
|
||||
env:
|
||||
AC_USERNAME: ${{ secrets.AC_USERNAME }}
|
||||
AC_PASSWORD: ${{ secrets.AC_PASSWORD }}
|
||||
run: |
|
||||
rm -rf release
|
||||
mkdir release
|
||||
@@ -102,17 +156,34 @@ jobs:
|
||||
lipo -create -output ./release/nebula ./build/darwin-amd64/nebula ./build/darwin-arm64/nebula
|
||||
lipo -create -output ./release/nebula-cert ./build/darwin-amd64/nebula-cert ./build/darwin-arm64/nebula-cert
|
||||
|
||||
if [ -n "$AC_USERNAME" ]; then
|
||||
codesign -s "10BC1FDDEB6CE753550156C0669109FAC49E4D1E" -f -v --timestamp --options=runtime -i "net.defined.nebula" ./release/nebula
|
||||
codesign -s "10BC1FDDEB6CE753550156C0669109FAC49E4D1E" -f -v --timestamp --options=runtime -i "net.defined.nebula-cert" ./release/nebula-cert
|
||||
# Unset in a fork, which has no credentials to sign with
|
||||
if [ -n "$SIGNING_IDENTITY_SHA1" ]; then
|
||||
codesign -s "$SIGNING_IDENTITY_SHA1" -f -v --timestamp --options=runtime -i "net.defined.nebula" ./release/nebula
|
||||
codesign -s "$SIGNING_IDENTITY_SHA1" -f -v --timestamp --options=runtime -i "net.defined.nebula-cert" ./release/nebula-cert
|
||||
fi
|
||||
|
||||
zip -j release/nebula-darwin.zip release/nebula-cert release/nebula
|
||||
|
||||
if [ -n "$AC_USERNAME" ]; then
|
||||
xcrun notarytool submit ./release/nebula-darwin.zip --team-id "576H3XS7FP" --apple-id "$AC_USERNAME" --password "$AC_PASSWORD" --wait
|
||||
if [ -n "$ASC_P8" ]; then
|
||||
xcrun notarytool submit ./release/nebula-darwin.zip --key "$ASC_P8" --key-id "$ASC_KEY_ID" --issuer "$ASC_ISSUER_ID" --wait
|
||||
fi
|
||||
|
||||
- name: Drop the signing key
|
||||
if: always() && env.HAS_SIGNING_CREDS == 'true'
|
||||
run: |
|
||||
# Locked, not deleted: import-codesign-certs deletes it in its own post
|
||||
# step and fails the job if it is already gone. Locked is unusable.
|
||||
security lock-keychain signing_temp.keychain || true
|
||||
rm -f "$ASC_P8"
|
||||
# Nothing later in this job needs AWS
|
||||
python3 -c '
|
||||
import json, os
|
||||
names = json.loads(os.environ.get("SECRETS_LIST_CLEAN_UP") or "[]")
|
||||
names += ["ASC_P8", "SIGNING_IDENTITY_SHA1", "ASC_KEY_ID", "ASC_ISSUER_ID",
|
||||
"AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
|
||||
print("\n".join(f"{n}=" for n in dict.fromkeys(names)))
|
||||
' >> "$GITHUB_ENV"
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
|
||||
@@ -34,7 +34,7 @@ jobs:
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '1.25'
|
||||
go-version: '1.26'
|
||||
check-latest: true
|
||||
|
||||
- name: add hashicorp source
|
||||
@@ -66,7 +66,7 @@ jobs:
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '1.25'
|
||||
go-version: '1.26'
|
||||
check-latest: true
|
||||
|
||||
- name: add hashicorp source
|
||||
@@ -92,7 +92,7 @@ jobs:
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '1.25'
|
||||
go-version: '1.26'
|
||||
check-latest: true
|
||||
|
||||
# WSL2 + Ubuntu so the smoke can run a real linux peer with its own
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '1.25'
|
||||
go-version: '1.26'
|
||||
check-latest: true
|
||||
|
||||
- name: build
|
||||
@@ -61,3 +61,24 @@ jobs:
|
||||
run: NAME="smoke-p256" ./smoke.sh
|
||||
|
||||
timeout-minutes: 10
|
||||
|
||||
smoke-self:
|
||||
name: Run self traffic smoke test on macOS
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '1.26'
|
||||
check-latest: true
|
||||
|
||||
- name: build
|
||||
run: make bin
|
||||
|
||||
- name: run smoke-self
|
||||
working-directory: ./.github/workflows/smoke
|
||||
run: ./smoke-self.sh
|
||||
|
||||
timeout-minutes: 10
|
||||
Executable
+130
@@ -0,0 +1,130 @@
|
||||
#!/bin/bash
|
||||
|
||||
# A host must be able to reach its own overlay address. Where the kernel sends
|
||||
# that traffic through the tun rather than over loopback, nebula sees it and
|
||||
# hands it straight back (immediatelyForwardToSelf), and whether the kernel
|
||||
# accepts what comes back is only answerable against a real kernel. Runs one
|
||||
# nebula on this machine as root and aims every probe at its own address.
|
||||
|
||||
set -e -x
|
||||
|
||||
set -o pipefail
|
||||
|
||||
V4=192.0.2.1
|
||||
V6=2001:db8::1
|
||||
|
||||
case "$(uname -s)" in
|
||||
Darwin) TUN_DEV=utun ;;
|
||||
*) TUN_DEV=tun0 ;;
|
||||
esac
|
||||
|
||||
ROOT="$(cd ../../.. && pwd)"
|
||||
|
||||
rm -rf build/self
|
||||
mkdir -p build/self
|
||||
cd build/self
|
||||
|
||||
cleanup() {
|
||||
echo
|
||||
echo " *** cleanup"
|
||||
echo
|
||||
|
||||
set +e
|
||||
if [ -n "$NEBULA_PID" ]
|
||||
then
|
||||
sudo kill "$NEBULA_PID"
|
||||
fi
|
||||
{ kill $(jobs -p); wait; } 2>/dev/null
|
||||
sed 's/^/ [self] /' nebula.log
|
||||
}
|
||||
|
||||
trap cleanup EXIT
|
||||
|
||||
# perl is on every platform this runs on; timeout(1) is not.
|
||||
alarm() {
|
||||
perl -e 'alarm shift; exec @ARGV' "$@"
|
||||
}
|
||||
|
||||
RESULTS=""
|
||||
FAILED=""
|
||||
probe() {
|
||||
local name="$1"
|
||||
shift
|
||||
if "$@"
|
||||
then
|
||||
RESULTS="$RESULTS $name=ok"
|
||||
else
|
||||
RESULTS="$RESULTS $name=FAIL"
|
||||
FAILED="$FAILED $name"
|
||||
fi
|
||||
}
|
||||
|
||||
# Send one datagram, then wait for the listener to have written it out.
|
||||
udp_probe() {
|
||||
echo self | alarm 5 nc -u -w1 "$1" 3000 || true
|
||||
set +x
|
||||
for _ in $(seq 1 20)
|
||||
do
|
||||
if grep -q self "$2"
|
||||
then
|
||||
set -x
|
||||
return 0
|
||||
fi
|
||||
sleep 0.25
|
||||
done
|
||||
set -x
|
||||
return 1
|
||||
}
|
||||
|
||||
"$ROOT/nebula-cert" ca -name "Smoke Test"
|
||||
"$ROOT/nebula-cert" sign -name self -networks "$V4/24,$V6/64"
|
||||
|
||||
HOST=self AM_LIGHTHOUSE=true TUN_DEV="$TUN_DEV" ../../genconfig.sh >self.yml
|
||||
|
||||
"$ROOT/nebula" -config self.yml -test
|
||||
|
||||
sudo -v
|
||||
sudo "$ROOT/nebula" -config self.yml >nebula.log 2>&1 &
|
||||
NEBULA_PID=$!
|
||||
|
||||
for _ in $(seq 1 40)
|
||||
do
|
||||
ifconfig | grep "inet6 $V6 " >/dev/null && break
|
||||
sleep 0.25
|
||||
done
|
||||
ifconfig | grep "inet $V4 "
|
||||
ifconfig | grep "inet6 $V6 "
|
||||
|
||||
nc -l "$V4" 2000 >/dev/null &
|
||||
nc -l "$V6" 2000 >/dev/null &
|
||||
nc -u -l "$V4" 3000 >udp4.txt &
|
||||
nc -u -l "$V6" 3000 >udp6.txt &
|
||||
sleep 1
|
||||
|
||||
set +x
|
||||
echo
|
||||
echo " *** Testing self traffic from $V4"
|
||||
echo
|
||||
set -x
|
||||
probe icmp4 alarm 5 ping -c1 "$V4"
|
||||
probe tcp4 alarm 5 nc -z "$V4" 2000
|
||||
probe udp4 udp_probe "$V4" udp4.txt
|
||||
|
||||
set +x
|
||||
echo
|
||||
echo " *** Testing self traffic from $V6"
|
||||
echo
|
||||
set -x
|
||||
probe icmp6 alarm 5 ping6 -c1 "$V6"
|
||||
probe tcp6 alarm 5 nc -z "$V6" 2000
|
||||
probe udp6 udp_probe "$V6" udp6.txt
|
||||
|
||||
set +x
|
||||
echo
|
||||
echo " *** self traffic:$RESULTS"
|
||||
echo
|
||||
if [ -n "$FAILED" ]
|
||||
then
|
||||
echo "self traffic failed:$FAILED" >&2
|
||||
exit 1
|
||||
fi
|
||||
@@ -51,15 +51,19 @@ wsl -d $Distro -- bash -c "rm -rf $WslDir && mkdir -p $WslDir" | Out-Null
|
||||
$DevName = 'nebula-smoke'
|
||||
$Ip1 = '192.168.241.1'
|
||||
$Ip2 = '192.168.241.2'
|
||||
# Dual stack on purpose: a v4-only overlay never exercises the v6 side of tun.mtu.
|
||||
$Ip6_1 = 'fd42:4242:241::1'
|
||||
$Ip6_2 = 'fd42:4242:241::2'
|
||||
$Mtu = 1300
|
||||
$Port = 4242
|
||||
|
||||
& $NebulaCert ca -name 'smoke-ca' -out-crt "$WorkDir\ca.crt" -out-key "$WorkDir\ca.key"
|
||||
if ($LASTEXITCODE -ne 0) { throw "nebula-cert ca failed (exit $LASTEXITCODE)" }
|
||||
|
||||
& $NebulaCert sign -name 'lighthouse' -networks "$Ip1/24" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\lighthouse.crt" -out-key "$WorkDir\lighthouse.key"
|
||||
& $NebulaCert sign -name 'lighthouse' -networks "$Ip1/24,$Ip6_1/64" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\lighthouse.crt" -out-key "$WorkDir\lighthouse.key"
|
||||
if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign lighthouse failed (exit $LASTEXITCODE)" }
|
||||
|
||||
& $NebulaCert sign -name 'peer' -networks "$Ip2/24" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\peer.crt" -out-key "$WorkDir\peer.key"
|
||||
& $NebulaCert sign -name 'peer' -networks "$Ip2/24,$Ip6_2/64" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\peer.crt" -out-key "$WorkDir\peer.key"
|
||||
if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign peer failed (exit $LASTEXITCODE)" }
|
||||
|
||||
# Windows lighthouse config.
|
||||
@@ -82,7 +86,7 @@ tun:
|
||||
drop_local_broadcast: false
|
||||
drop_multicast: false
|
||||
tx_queue: 500
|
||||
mtu: 1300
|
||||
mtu: $Mtu
|
||||
network_category: private
|
||||
logging:
|
||||
level: info
|
||||
@@ -126,7 +130,7 @@ tun:
|
||||
drop_local_broadcast: false
|
||||
drop_multicast: false
|
||||
tx_queue: 500
|
||||
mtu: 1300
|
||||
mtu: $Mtu
|
||||
logging:
|
||||
level: info
|
||||
format: text
|
||||
@@ -169,7 +173,7 @@ Write-Host '=== WSL diagnostic ==='
|
||||
wsl --version 2>&1 | Out-Host
|
||||
wsl --list --verbose 2>&1 | Out-Host
|
||||
wsl -d $Distro -u root -- uname -a | Out-Host
|
||||
wsl -d $Distro -u root -- bash -c "modprobe tun 2>&1 || true; mkdir -p /dev/net; [ -c /dev/net/tun ] || mknod /dev/net/tun c 10 200; chmod 600 /dev/net/tun; ls -l /dev/net/tun"
|
||||
wsl -d $Distro -u root -- bash -c "modprobe tun 2>&1 || true; mkdir -p /dev/net; [ -c /dev/net/tun ] || mknod /dev/net/tun c 10 200; chmod 600 /dev/net/tun; { echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6; echo 0 > /proc/sys/net/ipv6/conf/default/disable_ipv6; } 2>/dev/null || true; ls -l /dev/net/tun"
|
||||
if ($LASTEXITCODE -ne 0) { throw "failed to prepare /dev/net/tun in WSL (TUN support missing?)" }
|
||||
|
||||
# Deliberately no New-NetFirewallRule calls here -- nebula's windows_bypass_wdf
|
||||
@@ -214,6 +218,16 @@ try {
|
||||
}
|
||||
Write-Host "OK: $DevName NetworkCategory=Private"
|
||||
|
||||
# v6 silently kept the adapter default of 65535 while v4 was correct.
|
||||
foreach ($family in @('IPv4', 'IPv6')) {
|
||||
Wait-Until -TimeoutSec 30 -What "$DevName $family NlMtu=$Mtu" -Predicate {
|
||||
if ($lhProc.HasExited) { throw "lighthouse exited (code $($lhProc.ExitCode)) before $family mtu was set" }
|
||||
$rows = @(Get-NetIPInterface -InterfaceAlias $DevName -AddressFamily $family -ErrorAction SilentlyContinue)
|
||||
$rows.Count -gt 0 -and -not ($rows | Where-Object { $_.NlMtu -ne $Mtu })
|
||||
}
|
||||
Write-Host "OK: $DevName $family NlMtu=$Mtu"
|
||||
}
|
||||
|
||||
Wait-Until -TimeoutSec 30 -What "WSL nebula1 with $Ip2" -Predicate {
|
||||
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before tun was ready" }
|
||||
$r = wsl -d $Distro -u root -- bash -c "ip -o addr show nebula1 2>/dev/null | grep -q 'inet $Ip2' && echo yes"
|
||||
@@ -221,6 +235,13 @@ try {
|
||||
}
|
||||
Write-Host "OK: WSL nebula1 has $Ip2"
|
||||
|
||||
Wait-Until -TimeoutSec 30 -What "WSL nebula1 with $Ip6_2" -Predicate {
|
||||
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the v6 address was up" }
|
||||
$r = wsl -d $Distro -u root -- bash -c "ip -o addr show nebula1 2>/dev/null | grep -q 'inet6 $Ip6_2' && echo yes"
|
||||
("$r").Trim() -eq 'yes'
|
||||
}
|
||||
Write-Host "OK: WSL nebula1 has $Ip6_2"
|
||||
|
||||
Wait-Until -TimeoutSec 30 -What "ping from WSL peer to windows lighthouse ($Ip1)" -Predicate {
|
||||
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before ping succeeded" }
|
||||
$r = wsl -d $Distro -u root -- bash -c "ping -c1 -W1 $Ip1 >/dev/null 2>&1 && echo OK"
|
||||
@@ -234,6 +255,28 @@ try {
|
||||
}
|
||||
Write-Host "OK: windows lighthouse -> WSL peer"
|
||||
|
||||
# Otherwise the v6 networks only prove the interface exists, not that it forwards.
|
||||
Wait-Until -TimeoutSec 30 -What "v6 ping from WSL peer to windows lighthouse ($Ip6_1)" -Predicate {
|
||||
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the v6 ping succeeded" }
|
||||
$r = wsl -d $Distro -u root -- bash -c "ping -6 -c1 -W1 $Ip6_1 >/dev/null 2>&1 && echo OK"
|
||||
("$r").Trim() -eq 'OK'
|
||||
}
|
||||
Write-Host "OK: WSL peer -> windows lighthouse over v6"
|
||||
|
||||
# The NlMtu check above proves the value was written, not that windows honors it. A payload this size only
|
||||
# crosses the tunnel as fragments cut at NlMtu on the way out of the adapter. Left at the adapter default it
|
||||
# reaches nebula whole, overflows the udp.MTU (9001) write buffer and is dropped, so the ping never answers.
|
||||
# Anything smaller would ride out as one oversized datagram and survive on IP fragmentation of the underlay.
|
||||
$BigPayload = 9000
|
||||
foreach ($target in @(@{ Family = 'v4'; Ip = $Ip2 }, @{ Family = 'v6'; Ip = $Ip6_2 })) {
|
||||
Wait-Until -TimeoutSec 30 -What "$($target.Family) ping with a $BigPayload byte payload from windows lighthouse to WSL peer ($($target.Ip))" -Predicate {
|
||||
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the large $($target.Family) ping succeeded" }
|
||||
$null = & ping.exe -n 1 -w 1000 -l $BigPayload $target.Ip
|
||||
$LASTEXITCODE -eq 0
|
||||
}
|
||||
Write-Host "OK: windows lighthouse -> WSL peer, $BigPayload byte $($target.Family) payload"
|
||||
}
|
||||
|
||||
Write-Host ''
|
||||
Write-Host 'All smoke checks passed.'
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ jobs:
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '1.25'
|
||||
go-version: '1.26'
|
||||
check-latest: true
|
||||
|
||||
- name: Install goimports
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
- name: golangci-lint
|
||||
uses: golangci/golangci-lint-action@v9
|
||||
with:
|
||||
version: v2.5
|
||||
version: v2.12
|
||||
|
||||
test:
|
||||
name: Test ${{ matrix.name }}
|
||||
@@ -82,7 +82,7 @@ jobs:
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '1.25'
|
||||
go-version: '1.26'
|
||||
check-latest: true
|
||||
|
||||
- name: Build
|
||||
@@ -127,7 +127,7 @@ jobs:
|
||||
|
||||
- uses: actions/setup-go@v7
|
||||
with:
|
||||
go-version: '1.25'
|
||||
go-version: '1.26'
|
||||
check-latest: true
|
||||
|
||||
- name: Build ${{ matrix.name }}
|
||||
|
||||
+128
-1
@@ -7,6 +7,130 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [1.11.2] - 2026-09-21
|
||||
|
||||
See the [v1.11.2](https://github.com/slackhq/nebula/milestone/31?closed=1) milestone for a complete list of changes.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Windows tracks NLMTU per address family and Nebula only ever set it for `AF_INET`, so the IPv6 MTU on every
|
||||
Windows adapter sat at the adapter default of 65535 regardless of `tun.mtu` and the stack could hand Nebula
|
||||
packets far larger than configured. An IPv6 enabled overlay with `tun.mtu` under 1280 now refuses to start,
|
||||
matching the other platforms. (#1871)
|
||||
- Android 11+ denies `bind()` on netlink route sockets inside the app sandbox, so local address discovery
|
||||
failed and an Android node advertised no underlay addresses, leaving peers on the same LAN to reach it only
|
||||
at the address a lighthouse observed. (#1881)
|
||||
- Recompute the transport checksum on packets a host sends to its own overlay address. On macOS the kernel
|
||||
leaves the checksum unfinished for hardware offload, which does not survive the trip through userspace, so
|
||||
TCP and UDP to a host's own IPv6 overlay address were dropped as corrupt. (#1862)
|
||||
|
||||
## [1.11.1] - 2026-08-21
|
||||
|
||||
See the [v1.11.1](https://github.com/slackhq/nebula/milestone/30?closed=1) milestone for a complete list of changes.
|
||||
|
||||
### Changed
|
||||
|
||||
- IPv6 packets whose next header is a protocol Nebula does not parse (SCTP, GRE, IP-in-IP, etc.) are now
|
||||
classified as that protocol with no ports, closing a firewall bypass where a crafted payload could steer
|
||||
the classifier into reading one as TCP/UDP and matching a TCP/UDP rule. These packets are now matched as
|
||||
their true protocol, so only a `proto: any` rule allows them. If you carry one of these protocols over the
|
||||
overlay, confirm a `proto: any` rule covers it before upgrading, it may have been passing only through this
|
||||
bypass. (#1840)
|
||||
- Drop the dependency on `github.com/cyberdelia/go-metrics-graphite`, which has been unmaintained for over ten
|
||||
years, by inlining the small amount of code Nebula used. (#1832)
|
||||
|
||||
### Fixed
|
||||
|
||||
- The ICMPv6 type was read from the wrong byte when classifying IPv6 packets, so the echo identifier used
|
||||
for conntrack was never picked up. (#1840)
|
||||
- Enforce outbound message counter limits so a tunnel is rehandshaked before the counter can wrap, preventing
|
||||
nonce reuse. This is unreachable in practice, but is enforced as a defense-in-depth measure. (#1841)
|
||||
- Prevent `nebula-cert ca` from running out of memory on 32bit systems when generating encrypted private keys. (#1834)
|
||||
- Tolerate `ErrDumpInterrupted` when listing tun addresses on Linux, so a transient interrupted netlink dump
|
||||
no longer aborts startup. (#1835)
|
||||
|
||||
## [1.11.0] - 2026-07-23
|
||||
|
||||
See the [v1.11.0](https://github.com/slackhq/nebula/milestone/25?closed=1) milestone for a complete list of changes.
|
||||
|
||||
### Breaking
|
||||
|
||||
- Logging has switched from logrus to Go's structured `slog`. Log output changes: levels are upper case
|
||||
(`level=INFO`), trace prints as `level=DEBUG-4`, timestamps are always RFC3339Nano and `logging.timestamp_format`
|
||||
is ignored, and some messages were reworded. Review any log parsing before upgrading. This is also an API break
|
||||
for embedders, as constructors now take a `*slog.Logger`. (#1672, #1734, #1621)
|
||||
- `firewall.inbound_action` and `firewall.outbound_action` (used to set reject vs. drop policy) were each being
|
||||
applied to the opposite direction, that is now corrected. This only affects how blocked packets are answered, not
|
||||
which packets the firewall allows or denies. If you set either of these you are getting the behavior of the other
|
||||
one today and likely want to swap them before upgrading. (#1798)
|
||||
- On Windows, Nebula now installs WFP PERMIT filters for the nebula adapter and the listener port by default. WFP
|
||||
sits below Windows Defender Firewall, so any WDF inbound rules you rely on for either will no longer apply. Set
|
||||
`tun.windows_bypass_wdf` and `listen.windows_bypass_wdf` to false to leave WDF in charge. (#1710)
|
||||
- On Windows, the nebula device is now set to the `private` network category instead of whatever Windows decided,
|
||||
which is usually `Public`. This makes the host firewall less restrictive on the overlay. Set
|
||||
`tun.network_category` to `unset` to keep the old behavior. (#1710)
|
||||
- Reject packets for non-TCP now use ICMP code 13, communication administratively prohibited, instead of code 3,
|
||||
port unreachable. Anything keying off the old code needs updating. (#1766, #1768)
|
||||
- The SSH debug server's profiling commands are now confined to `sshd.sandbox_dir`, which defaults to
|
||||
`$TMP/nebula-debug`. Relative paths resolve inside it and absolute paths outside it are rejected, so anything
|
||||
scripting `start-cpu-profile`, `save-heap-profile`, or `save-mutex-profile` with a path elsewhere needs the
|
||||
directory set. The directory is not created for you. (#1622)
|
||||
|
||||
### Added
|
||||
|
||||
- Sign the Windows release binaries. (#1718)
|
||||
- Generate IPv6 reject packets, matching the existing IPv4 behavior. (#1766, #1767, #1768)
|
||||
- Accept `-` in `nebula-cert` to read from stdin or write to stdout. (#1714)
|
||||
- Search for both `config.yml` and `config.yaml` in service and command line modes. (#1717)
|
||||
- Add version labels to the Docker/OCI images. (#1772)
|
||||
- Rebind the listener and re-query lighthouses on macOS when the underlay network changes, so devices moving
|
||||
between wifi and wired or between networks recover without waiting for dead tunnel detection. Controlled by
|
||||
`listen.rebind_on_network_change` (default `true`, not reloadable). (#1816)
|
||||
|
||||
### Changed
|
||||
|
||||
- Reload the firewall when the unsafe networks in the certificate change. (#1719)
|
||||
- Reconfigure, start, and stop the stats listener on a config reload instead of requiring a restart. (#1670)
|
||||
- Update a static host's addresses when they change on reload. (#1713)
|
||||
- Don't require a port on ICMP firewall rules. (#1609)
|
||||
- Connection track ICMP traffic. (#1602)
|
||||
- Return `NODATA` instead of `NXDOMAIN` from the DNS server for a name that exists but has no record of the
|
||||
requested type, so clients that query `AAAA` first (busybox/Alpine) fall through to `A`. (#1668)
|
||||
- Record the local host's details in the DNS server. (#1716)
|
||||
- Install Windows unsafe routes as link routes. (#1709)
|
||||
- Reduce relay handshake log spam, and only log a handshake send error at error level when the remote list
|
||||
changes. (#1733, #1765, #1810)
|
||||
- Start, stop, and reload subsystems (DNS, stats, conntrack, ssh, punchy) cleanly without leaking goroutines. (#1640, #1654, #1661, #1667, #1669, #1708, #1806, #1815)
|
||||
- `Control` is now safe to stop and wait on from any lifecycle state, and a new `Control.Wait` blocks until nebula
|
||||
has fully stopped and returns the first fatal reader error. Failed starts release the udp sockets and tun fd
|
||||
instead of leaking them. (#1794)
|
||||
- Trigger an immediate lighthouse update when reconnecting to or adding a lighthouse instead of waiting for the next update tick. (#1645)
|
||||
- Bring the Darwin and OpenBSD tun implementations in line with the other BSDs. (#1703)
|
||||
- Update to build against go v1.26. (#1818)
|
||||
- Various dependency updates. (#1586, #1587, #1604, #1617, #1618, #1627, #1628, #1629, #1652, #1664, #1665, #1697, #1721, #1732, #1742, #1743, #1750, #1763, #1771, #1782, #1800, #1807)
|
||||
|
||||
### Fixed
|
||||
|
||||
- Fix a data race on a host's remote address that could send packets to the wrong address during a roam. (#1773)
|
||||
- Fix tunnels that could permanently escape connection manager monitoring. (#1752)
|
||||
- Fix a crash when reloading the SSH server's trusted keys. (#1787)
|
||||
- Fix hostmap corruption when a host has multiple overlay addresses. Each address now gets its own list instead of
|
||||
a single shared chain, which also fixes two latent bugs on the add and makePrimary paths. (#1788, #1790)
|
||||
- Apply `remote_allow_list` IPv4 rules to 4-in-6 mapped addresses. (#1786)
|
||||
- Don't panic in the DNS server on a short or empty query name. (#1635)
|
||||
- Advance the replay window on relayed packets so a relay drops replayed frames instead of re-forwarding them. (#1751)
|
||||
- Fix a race in relay state handling. (#1753)
|
||||
- Lock replay window updates so concurrent readers can't corrupt it. (#1802)
|
||||
- Reject malformed handshakes more reliably, including invalid ed25519 key lengths. (#1601, #1756)
|
||||
- Properly handle `closetunnel` packets. (#1638)
|
||||
- Fix an IPv6 extension-header length overflow that could make the firewall parse the wrong protocol and ports. (#1789)
|
||||
- Fix relay re-establishment when a handshake arrives over a relay entry that a one-sided teardown left
|
||||
`Disestablished`, which silently dropped every send until dead tunnel detection forced a re-handshake. (#1805)
|
||||
- Don't build new relay state on a tunnel that was just discarded. (#1796)
|
||||
- Don't delete the wrong pending hostinfo in the handshake manager. (#1811)
|
||||
- Don't call the packet reader after a UDP error on Darwin. (#1755)
|
||||
- Open the FreeBSD tun device non blocking. (#1666)
|
||||
|
||||
## [1.10.3] - 2026-02-06
|
||||
|
||||
### Security
|
||||
@@ -788,7 +912,10 @@ created.)
|
||||
|
||||
- Initial public release.
|
||||
|
||||
[Unreleased]: https://github.com/slackhq/nebula/compare/v1.10.3...HEAD
|
||||
[Unreleased]: https://github.com/slackhq/nebula/compare/v1.11.2...HEAD
|
||||
[1.11.2]: https://github.com/slackhq/nebula/releases/tag/v1.11.2
|
||||
[1.11.1]: https://github.com/slackhq/nebula/releases/tag/v1.11.1
|
||||
[1.11.0]: https://github.com/slackhq/nebula/releases/tag/v1.11.0
|
||||
[1.10.3]: https://github.com/slackhq/nebula/releases/tag/v1.10.3
|
||||
[1.10.2]: https://github.com/slackhq/nebula/releases/tag/v1.10.2
|
||||
[1.10.1]: https://github.com/slackhq/nebula/releases/tag/v1.10.1
|
||||
|
||||
@@ -222,11 +222,14 @@ test-cov-html:
|
||||
go test -coverprofile=coverage.out
|
||||
go tool cover -html=coverage.out
|
||||
|
||||
# The package builds only compile. The final line links an android binary so a linker-only failure
|
||||
# cannot pass CI.
|
||||
build-test-mobile:
|
||||
GOARCH=amd64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
|
||||
GOARCH=arm64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
|
||||
GOARCH=amd64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
|
||||
GOARCH=arm64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
|
||||
GOARCH=arm64 GOOS=android go build -o /dev/null ${NEBULA_CMD_PATH}
|
||||
|
||||
bench:
|
||||
go test -bench=.
|
||||
@@ -275,10 +278,13 @@ smoke-docker-race: BUILD_ARGS = -race
|
||||
smoke-docker-race: CGO_ENABLED = 1
|
||||
smoke-docker-race: smoke-docker
|
||||
|
||||
smoke-self: bin
|
||||
cd .github/workflows/smoke/ && ./smoke-self.sh
|
||||
|
||||
smoke-vagrant/%: bin-docker build/%/nebula
|
||||
cd .github/workflows/smoke/ && ./build.sh $*
|
||||
cd .github/workflows/smoke/ && ./smoke-vagrant.sh $*
|
||||
|
||||
.FORCE:
|
||||
.PHONY: all all-linux all-freebsd all-openbsd all-netbsd all-darwin all-windows all-cross-linux all-cross-linux-arm all-cross-linux-mips all-cross-linux-other all-cross-darwin all-cross-windows bench bench-cpu bench-cpu-long bin build-test-mobile e2e e2ev e2evv e2evvv e2evvvv proto release service smoke-docker smoke-docker-race test test-cov-html smoke-vagrant/%
|
||||
.PHONY: all all-linux all-freebsd all-openbsd all-netbsd all-darwin all-windows all-cross-linux all-cross-linux-arm all-cross-linux-mips all-cross-linux-other all-cross-darwin all-cross-windows bench bench-cpu bench-cpu-long bin build-test-mobile e2e e2ev e2evv e2evvv e2evvvv proto release service smoke-docker smoke-docker-race smoke-self test test-cov-html smoke-vagrant/%
|
||||
.DEFAULT_GOAL := bin
|
||||
+17
-2
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"math"
|
||||
"math/bits"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strings"
|
||||
@@ -44,6 +45,20 @@ type caFlags struct {
|
||||
}
|
||||
|
||||
func newCaFlags() *caFlags {
|
||||
// prevent running out of memory on 32-bit systems by defaulting to
|
||||
// RFC9106's recommendation for memory-constrained environments
|
||||
var (
|
||||
defaultArgonMemory uint
|
||||
defaultArgonIterations uint
|
||||
)
|
||||
if bits.UintSize == 32 {
|
||||
defaultArgonMemory = 64 * 1024
|
||||
defaultArgonIterations = 3
|
||||
} else {
|
||||
defaultArgonMemory = 2 * 1024 * 1024
|
||||
defaultArgonIterations = 1
|
||||
}
|
||||
|
||||
cf := caFlags{set: flag.NewFlagSet("ca", flag.ContinueOnError)}
|
||||
cf.set.Usage = func() {}
|
||||
cf.name = cf.set.String("name", "", "Required: name of the certificate authority")
|
||||
@@ -55,9 +70,9 @@ func newCaFlags() *caFlags {
|
||||
cf.groups = cf.set.String("groups", "", "Optional: comma separated list of groups. This will limit which groups subordinate certs can use")
|
||||
cf.networks = cf.set.String("networks", "", "Optional: comma separated list of ip address and network in CIDR notation. This will limit which ip addresses and networks subordinate certs can use in networks")
|
||||
cf.unsafeNetworks = cf.set.String("unsafe-networks", "", "Optional: comma separated list of ip address and network in CIDR notation. This will limit which ip addresses and networks subordinate certs can use in unsafe networks")
|
||||
cf.argonMemory = cf.set.Uint("argon-memory", 2*1024*1024, "Optional: Argon2 memory parameter (in KiB) used for encrypted private key passphrase")
|
||||
cf.argonMemory = cf.set.Uint("argon-memory", defaultArgonMemory, "Optional: Argon2 memory parameter (in KiB) used for encrypted private key passphrase")
|
||||
cf.argonParallelism = cf.set.Uint("argon-parallelism", 4, "Optional: Argon2 parallelism parameter used for encrypted private key passphrase")
|
||||
cf.argonIterations = cf.set.Uint("argon-iterations", 1, "Optional: Argon2 iterations parameter used for encrypted private key passphrase")
|
||||
cf.argonIterations = cf.set.Uint("argon-iterations", defaultArgonIterations, "Optional: Argon2 iterations parameter used for encrypted private key passphrase")
|
||||
cf.encryption = cf.set.Bool("encrypt", false, "Optional: prompt for passphrase and write out-key in an encrypted format")
|
||||
cf.curve = cf.set.String("curve", "25519", "EdDSA/ECDSA Curve (25519, P256)")
|
||||
cf.p11url = p11Flag(cf.set)
|
||||
|
||||
@@ -7,7 +7,9 @@ import (
|
||||
"bytes"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"math/bits"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -22,6 +24,18 @@ func Test_caSummary(t *testing.T) {
|
||||
}
|
||||
|
||||
func Test_caHelp(t *testing.T) {
|
||||
var (
|
||||
defaultArgonMemory string
|
||||
defaultArgonIterations string
|
||||
)
|
||||
if bits.UintSize == 32 {
|
||||
defaultArgonMemory = strconv.Itoa(64 * 1024)
|
||||
defaultArgonIterations = strconv.Itoa(3)
|
||||
} else {
|
||||
defaultArgonMemory = strconv.Itoa(2 * 1024 * 1024)
|
||||
defaultArgonIterations = strconv.Itoa(1)
|
||||
}
|
||||
|
||||
ob := &bytes.Buffer{}
|
||||
caHelp(ob)
|
||||
assert.Equal(
|
||||
@@ -29,9 +43,9 @@ func Test_caHelp(t *testing.T) {
|
||||
"Usage of "+os.Args[0]+" ca <flags>: create a self signed certificate authority\n"+
|
||||
" Pass \"-\" to any path flag to read from stdin or write to stdout.\n"+
|
||||
" -argon-iterations uint\n"+
|
||||
" \tOptional: Argon2 iterations parameter used for encrypted private key passphrase (default 1)\n"+
|
||||
" \tOptional: Argon2 iterations parameter used for encrypted private key passphrase (default "+defaultArgonIterations+")\n"+
|
||||
" -argon-memory uint\n"+
|
||||
" \tOptional: Argon2 memory parameter (in KiB) used for encrypted private key passphrase (default 2097152)\n"+
|
||||
" \tOptional: Argon2 memory parameter (in KiB) used for encrypted private key passphrase (default "+defaultArgonMemory+")\n"+
|
||||
" -argon-parallelism uint\n"+
|
||||
" \tOptional: Argon2 parallelism parameter used for encrypted private key passphrase (default 4)\n"+
|
||||
" -curve string\n"+
|
||||
@@ -188,10 +202,16 @@ func Test_ca(t *testing.T) {
|
||||
k, _ := pem.Decode(rb)
|
||||
ned, err := cert.UnmarshalNebulaEncryptedData(k.Bytes)
|
||||
require.NoError(t, err)
|
||||
// we won't know salt in advance, so just check start of string
|
||||
assert.Equal(t, uint32(2*1024*1024), ned.EncryptionMetadata.Argon2Parameters.Memory)
|
||||
|
||||
if bits.UintSize == 32 {
|
||||
assert.Equal(t, uint32(64*1024), ned.EncryptionMetadata.Argon2Parameters.Memory)
|
||||
assert.Equal(t, uint32(3), ned.EncryptionMetadata.Argon2Parameters.Iterations)
|
||||
} else {
|
||||
assert.Equal(t, uint32(2*1024*1024), ned.EncryptionMetadata.Argon2Parameters.Memory)
|
||||
assert.Equal(t, uint32(1), ned.EncryptionMetadata.Argon2Parameters.Iterations)
|
||||
}
|
||||
|
||||
assert.Equal(t, uint8(4), ned.EncryptionMetadata.Argon2Parameters.Parallelism)
|
||||
assert.Equal(t, uint32(1), ned.EncryptionMetadata.Argon2Parameters.Iterations)
|
||||
|
||||
// verify the key is valid and decrypt-able
|
||||
var curve cert.Curve
|
||||
|
||||
+20
-100
@@ -44,11 +44,6 @@ type connectionManager struct {
|
||||
inactivityTimeout atomic.Int64
|
||||
dropInactive atomic.Bool
|
||||
|
||||
// Wake-from-sleep handling, sampled once per tick in Start
|
||||
wakeDetector *wakeDetector
|
||||
clearOnWake atomic.Bool
|
||||
wakeClearThreshold atomic.Int64
|
||||
|
||||
l *slog.Logger
|
||||
}
|
||||
|
||||
@@ -59,7 +54,6 @@ func newConnectionManagerFromConfig(l *slog.Logger, c *config.C, hm *HostMap, p
|
||||
punchy: p,
|
||||
relayUsed: make(map[uint32]struct{}),
|
||||
relayUsedLock: &sync.RWMutex{},
|
||||
wakeDetector: newWakeDetector(),
|
||||
}
|
||||
|
||||
cm.reload(c, true)
|
||||
@@ -104,38 +98,12 @@ func (cm *connectionManager) reload(c *config.C, initial bool) {
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if initial || c.HasChanged("tunnels.clear_on_wake") {
|
||||
old := cm.clearOnWake.Load()
|
||||
cm.clearOnWake.Store(c.GetBool("tunnels.clear_on_wake", true))
|
||||
if !initial {
|
||||
cm.l.Info("Clear on wake setting has changed",
|
||||
"oldBool", old,
|
||||
"newBool", cm.clearOnWake.Load(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
if initial || c.HasChanged("tunnels.wake_clear_threshold") {
|
||||
old := cm.getWakeClearThreshold()
|
||||
cm.wakeClearThreshold.Store((int64)(c.GetDuration("tunnels.wake_clear_threshold", 30*time.Second)))
|
||||
if !initial {
|
||||
cm.l.Info("Wake clear threshold has changed",
|
||||
"oldDuration", old,
|
||||
"newDuration", cm.getWakeClearThreshold(),
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (cm *connectionManager) getInactivityTimeout() time.Duration {
|
||||
return (time.Duration)(cm.inactivityTimeout.Load())
|
||||
}
|
||||
|
||||
func (cm *connectionManager) getWakeClearThreshold() time.Duration {
|
||||
return (time.Duration)(cm.wakeClearThreshold.Load())
|
||||
}
|
||||
|
||||
func (cm *connectionManager) In(h *HostInfo) {
|
||||
h.in.Store(true)
|
||||
}
|
||||
@@ -168,73 +136,6 @@ func (cm *connectionManager) getAndResetTrafficCheck(h *HostInfo, now time.Time)
|
||||
return in, out
|
||||
}
|
||||
|
||||
// checkWake runs once per tick and clears every tunnel when the machine has just returned from system sleep.
|
||||
// Tunnels rarely survive a suspend: our NAT mappings have expired and our address has usually changed, so every
|
||||
// established hostinfo is a corpse that will eat 15-20s of traffic checks before the wheel declares it dead.
|
||||
// Clearing now means the first packet after wake starts a fresh handshake immediately.
|
||||
//
|
||||
// The suspend itself costs nothing here: the ticker driving us is frozen with the rest of the process and this
|
||||
// fires within one tick of resume.
|
||||
func (cm *connectionManager) checkWake() {
|
||||
slept, ok := cm.wakeDetector.Sample()
|
||||
if !ok || slept == 0 {
|
||||
return
|
||||
}
|
||||
|
||||
// The clock pair is read non-atomically, so scheduling jitter shows up as tiny sub-millisecond "sleeps".
|
||||
// Keep the floor well above that so a zero/nonsense threshold can't clear tunnels on every tick.
|
||||
threshold := max(cm.getWakeClearThreshold(), time.Second)
|
||||
|
||||
if slept < threshold {
|
||||
// Short suspends (lid closed and quickly reopened) often come back before NAT state expires; those
|
||||
// tunnels may well be alive, leave them to the normal traffic checks.
|
||||
if slept >= time.Second {
|
||||
cm.l.Debug("Woke from sleep below the clear threshold, leaving tunnels alone",
|
||||
"sleptFor", slept,
|
||||
"threshold", threshold,
|
||||
)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if !cm.clearOnWake.Load() {
|
||||
cm.l.Info("Woke from sleep, tunnels.clear_on_wake is disabled so tunnels are left to the normal traffic checks", "sleptFor", slept)
|
||||
return
|
||||
}
|
||||
|
||||
closed := cm.clearAllTunnels()
|
||||
cm.l.Info("Woke from sleep, cleared tunnels", "sleptFor", slept, "tunnelsCleared", closed)
|
||||
|
||||
// Our public address almost certainly changed; get it to the lighthouses as soon as possible so peers can
|
||||
// find us again. The update rides over a fresh lighthouse handshake. If the network isn't back up yet these
|
||||
// sends fail harmlessly and the periodic update worker retries within lighthouse.interval.
|
||||
cm.intf.lightHouse.TriggerUpdate()
|
||||
}
|
||||
|
||||
// clearAllTunnels closes every tunnel in the hostmap locally, without notifying the remotes. It is the wake-from-
|
||||
// sleep counterpart to Control.CloseAllTunnels: after a suspend the remotes stopped hearing from us long ago, and
|
||||
// close packets fired into a network that may not even be up yet are wasted, so we only tear down our own state
|
||||
// and let the next packet to each host start a fresh handshake.
|
||||
func (cm *connectionManager) clearAllTunnels() int {
|
||||
cm.hostMap.RLock()
|
||||
hostinfos := make([]*HostInfo, 0, len(cm.hostMap.Indexes))
|
||||
for _, h := range cm.hostMap.Indexes {
|
||||
hostinfos = append(hostinfos, h)
|
||||
}
|
||||
cm.hostMap.RUnlock()
|
||||
|
||||
for _, h := range hostinfos {
|
||||
cm.intf.closeTunnel(h)
|
||||
}
|
||||
|
||||
// With every tunnel gone no relay can be in use, drop the usage tracking wholesale.
|
||||
cm.relayUsedLock.Lock()
|
||||
clear(cm.relayUsed)
|
||||
cm.relayUsedLock.Unlock()
|
||||
|
||||
return len(hostinfos)
|
||||
}
|
||||
|
||||
func (cm *connectionManager) Start(ctx context.Context) {
|
||||
clockSource := time.NewTicker(cm.trafficTimer.t.tickDuration)
|
||||
defer clockSource.Stop()
|
||||
@@ -249,7 +150,6 @@ func (cm *connectionManager) Start(ctx context.Context) {
|
||||
return
|
||||
|
||||
case now := <-clockSource.C:
|
||||
cm.checkWake()
|
||||
cm.trafficTimer.Advance(now)
|
||||
for {
|
||||
localIndex, has := cm.trafficTimer.Purge()
|
||||
@@ -423,6 +323,12 @@ func (cm *connectionManager) makeTrafficDecision(localIndex uint32, now time.Tim
|
||||
return closeTunnel, hostinfo, nil
|
||||
}
|
||||
|
||||
if hostinfo.ConnectionState != nil && hostinfo.ConnectionState.messageCounter.Load() >= RejectAfterMessages {
|
||||
// Send path can't encrypt a CloseTunnel notify, so just delete locally; the peer recovers via recv_error.
|
||||
hostinfo.logger(cm.l).Error("Dropping tunnel, message counter is exhausted")
|
||||
return deleteTunnel, hostinfo, nil
|
||||
}
|
||||
|
||||
primary := cm.hostMap.Hosts[hostinfo.vpnAddrs[0]]
|
||||
mainHostInfo := true
|
||||
if primary != nil && primary != hostinfo {
|
||||
@@ -548,6 +454,11 @@ func (cm *connectionManager) shouldSwapPrimary(current *HostInfo) bool {
|
||||
return false
|
||||
}
|
||||
|
||||
if current.ConnectionState.messageCounter.Load() >= RehandshakeAfterMessages {
|
||||
// This tunnel is being rolled for counter exhaustion, never swap back onto its spent key.
|
||||
return false
|
||||
}
|
||||
|
||||
crt := cm.intf.pki.getCertState().getCertificate(current.ConnectionState.myCert.Version())
|
||||
if crt == nil {
|
||||
//my cert was reloaded away. We should definitely swap from this tunnel
|
||||
@@ -644,6 +555,15 @@ func (cm *connectionManager) tryRehandshake(hostinfo *HostInfo) {
|
||||
"reason", "current cert version < pki.initiatingVersion",
|
||||
)
|
||||
|
||||
cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
|
||||
return
|
||||
}
|
||||
if hostinfo.ConnectionState.messageCounter.Load() >= RehandshakeAfterMessages {
|
||||
cm.l.Info("Re-handshaking with remote",
|
||||
"vpnAddrs", hostinfo.vpnAddrs,
|
||||
"reason", "message counter rehandshake threshold reached",
|
||||
)
|
||||
|
||||
cm.intf.handshakeManager.StartHandshake(hostinfo.vpnAddrs[0], nil)
|
||||
return
|
||||
}
|
||||
|
||||
+74
-82
@@ -25,6 +25,7 @@ func newTestLighthouse() *LightHouse {
|
||||
lighthouses := []netip.Addr{}
|
||||
staticList := map[netip.Addr]struct{}{}
|
||||
|
||||
lh.localAddrsFn = func(*LocalAllowList) []netip.Addr { return nil }
|
||||
lh.lighthouses.Store(&lighthouses)
|
||||
lh.staticList.Store(&staticList)
|
||||
|
||||
@@ -198,6 +199,79 @@ func Test_NewConnectionManagerTest2(t *testing.T) {
|
||||
assert.Contains(t, nc.hostMap.Hosts, hostinfo.vpnAddrs[0])
|
||||
}
|
||||
|
||||
func Test_NewConnectionManager_CounterLimits(t *testing.T) {
|
||||
l := test.NewLogger()
|
||||
localrange := netip.MustParsePrefix("10.1.1.1/24")
|
||||
vpnIp := netip.MustParseAddr("172.1.1.2")
|
||||
preferredRanges := []netip.Prefix{localrange}
|
||||
|
||||
// Very incomplete mock objects
|
||||
hostMap := newHostMap(l)
|
||||
hostMap.preferredRanges.Store(&preferredRanges)
|
||||
|
||||
cs := &CertState{
|
||||
initiatingVersion: cert.Version1,
|
||||
privateKey: []byte{},
|
||||
v1Cert: &dummyCert{version: cert.Version1},
|
||||
v1Credential: nil,
|
||||
}
|
||||
|
||||
lh := newTestLighthouse()
|
||||
ifce := &Interface{
|
||||
hostMap: hostMap,
|
||||
inside: &overlaytest.NoopTun{},
|
||||
outside: &udp.NoopConn{},
|
||||
firewall: &Firewall{},
|
||||
lightHouse: lh,
|
||||
pki: &PKI{},
|
||||
myVpnAddrs: []netip.Addr{netip.MustParseAddr("172.1.1.1")}, // sorts below vpnIp so shouldSwapPrimary can proceed
|
||||
handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
|
||||
l: l,
|
||||
}
|
||||
ifce.pki.cs.Store(cs)
|
||||
|
||||
conf := config.NewC(test.NewLogger())
|
||||
punchy := NewPunchyFromConfig(test.NewLogger(), conf, nil)
|
||||
nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy)
|
||||
nc.intf = ifce
|
||||
|
||||
hostinfo := &HostInfo{
|
||||
vpnAddrs: []netip.Addr{vpnIp},
|
||||
localIndexId: 1099,
|
||||
remoteIndexId: 9901,
|
||||
}
|
||||
hostinfo.ConnectionState = &ConnectionState{
|
||||
myCert: &dummyCert{version: cert.Version1},
|
||||
}
|
||||
nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
|
||||
|
||||
// Below the rehandshake threshold, no handshake is started
|
||||
hostinfo.ConnectionState.messageCounter.Store(RehandshakeAfterMessages - 1)
|
||||
nc.tryRehandshake(hostinfo)
|
||||
assert.Nil(t, ifce.handshakeManager.QueryVpnAddr(vpnIp))
|
||||
|
||||
// A tunnel on its current cert would normally swap to primary
|
||||
assert.True(t, nc.shouldSwapPrimary(hostinfo))
|
||||
|
||||
// At the rehandshake threshold, a new handshake is started
|
||||
hostinfo.ConnectionState.messageCounter.Store(RehandshakeAfterMessages)
|
||||
nc.tryRehandshake(hostinfo)
|
||||
assert.NotNil(t, ifce.handshakeManager.QueryVpnAddr(vpnIp))
|
||||
|
||||
// An exhausted tunnel being rolled must never swap back to primary onto its spent key
|
||||
assert.False(t, nc.shouldSwapPrimary(hostinfo))
|
||||
|
||||
// Still below the reject limit, the tunnel stays up
|
||||
nc.In(hostinfo)
|
||||
decision, _, _ := nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
|
||||
assert.Equal(t, tryRehandshake, decision)
|
||||
|
||||
// At the reject limit, the tunnel is deleted locally without a doomed CloseTunnel notify
|
||||
hostinfo.ConnectionState.messageCounter.Store(RejectAfterMessages)
|
||||
decision, _, _ = nc.makeTrafficDecision(hostinfo.localIndexId, time.Now())
|
||||
assert.Equal(t, deleteTunnel, decision)
|
||||
}
|
||||
|
||||
func Test_NewConnectionManager_DisconnectInactive(t *testing.T) {
|
||||
l := test.NewLogger()
|
||||
localrange := netip.MustParsePrefix("10.1.1.1/24")
|
||||
@@ -501,85 +575,3 @@ func (d *dummyCert) MarshalJSON() ([]byte, error) {
|
||||
func (d *dummyCert) Copy() cert.Certificate {
|
||||
return d
|
||||
}
|
||||
|
||||
func TestConnectionManager_WakeClear(t *testing.T) {
|
||||
l := test.NewLogger()
|
||||
localrange := netip.MustParsePrefix("10.1.1.1/24")
|
||||
vpnIp := netip.MustParseAddr("172.1.1.2")
|
||||
preferredRanges := []netip.Prefix{localrange}
|
||||
|
||||
// Very incomplete mock objects
|
||||
hostMap := newHostMap(l)
|
||||
hostMap.preferredRanges.Store(&preferredRanges)
|
||||
|
||||
cs := &CertState{
|
||||
initiatingVersion: cert.Version1,
|
||||
privateKey: []byte{},
|
||||
v1Cert: &dummyCert{version: cert.Version1},
|
||||
v1Credential: nil,
|
||||
}
|
||||
|
||||
lh := newTestLighthouse()
|
||||
ifce := &Interface{
|
||||
hostMap: hostMap,
|
||||
inside: &overlaytest.NoopTun{},
|
||||
outside: &udp.NoopConn{},
|
||||
firewall: &Firewall{},
|
||||
lightHouse: lh,
|
||||
pki: &PKI{},
|
||||
handshakeManager: NewHandshakeManager(l, hostMap, lh, &udp.NoopConn{}, defaultHandshakeConfig),
|
||||
l: l,
|
||||
}
|
||||
ifce.pki.cs.Store(cs)
|
||||
|
||||
// Create manager
|
||||
conf := config.NewC(test.NewLogger())
|
||||
punchy := NewPunchyFromConfig(test.NewLogger(), conf, nil)
|
||||
nc := newConnectionManagerFromConfig(test.NewLogger(), conf, hostMap, punchy)
|
||||
nc.intf = ifce
|
||||
|
||||
// Drive the wake detector from a fake clock pair
|
||||
suspended := time.Duration(0)
|
||||
nc.wakeDetector = &wakeDetector{read: func() (time.Duration, bool) { return suspended, true }}
|
||||
nc.checkWake() // primes the baseline
|
||||
|
||||
addTunnel := func(localIndex uint32) *HostInfo {
|
||||
hostinfo := &HostInfo{
|
||||
vpnAddrs: []netip.Addr{vpnIp},
|
||||
localIndexId: localIndex,
|
||||
remoteIndexId: 9901,
|
||||
}
|
||||
hostinfo.ConnectionState = &ConnectionState{
|
||||
myCert: &dummyCert{version: cert.Version1},
|
||||
}
|
||||
nc.hostMap.unlockedAddHostInfo(hostinfo, ifce)
|
||||
return hostinfo
|
||||
}
|
||||
|
||||
addTunnel(1099)
|
||||
nc.RelayUsed(5000)
|
||||
|
||||
// No suspend, nothing happens
|
||||
nc.checkWake()
|
||||
assert.Contains(t, nc.hostMap.Indexes, uint32(1099))
|
||||
|
||||
// A suspend below the threshold leaves tunnels alone
|
||||
suspended += 5 * time.Second
|
||||
nc.checkWake()
|
||||
assert.Contains(t, nc.hostMap.Indexes, uint32(1099))
|
||||
|
||||
// A suspend past the threshold clears everything, including relay usage tracking
|
||||
suspended += time.Hour
|
||||
nc.checkWake()
|
||||
assert.Empty(t, nc.hostMap.Indexes)
|
||||
assert.Empty(t, nc.hostMap.Hosts)
|
||||
assert.Empty(t, nc.relayUsed)
|
||||
|
||||
// With clear_on_wake disabled the tunnels survive a long suspend
|
||||
addTunnel(1100)
|
||||
nc.clearOnWake.Store(false)
|
||||
suspended += time.Hour
|
||||
nc.checkWake()
|
||||
assert.Contains(t, nc.hostMap.Indexes, uint32(1100))
|
||||
assert.Contains(t, nc.hostMap.Hosts, vpnIp)
|
||||
}
|
||||
+30
-3
@@ -2,6 +2,7 @@ package nebula
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -12,7 +13,18 @@ import (
|
||||
"github.com/slackhq/nebula/noiseutil"
|
||||
)
|
||||
|
||||
const ReplayWindow = 1024
|
||||
const (
|
||||
ReplayWindow = 1024
|
||||
|
||||
// RehandshakeAfterMessages rolls keys inside the AES-GCM data-volume margin (~2^-36 advantage at 64KB frames).
|
||||
RehandshakeAfterMessages = uint64(1) << 34
|
||||
|
||||
// RejectAfterMessages is the nonce ceiling enforced by noiseutil; a tunnel here is deleted locally, not notified.
|
||||
RejectAfterMessages = noiseutil.RejectAfterMessages
|
||||
)
|
||||
|
||||
// RehandshakeAfterMessages must stay below RejectAfterMessages so tunnels roll before the hard send stop.
|
||||
const _ = RejectAfterMessages - RehandshakeAfterMessages
|
||||
|
||||
type ConnectionState struct {
|
||||
eKey noiseutil.CipherState
|
||||
@@ -30,7 +42,12 @@ type ConnectionState struct {
|
||||
// completed handshake.Result. It seeds messageCounter and the replay window so
|
||||
// that the post-handshake message indices already used on the wire don't count
|
||||
// as missed traffic in the data plane.
|
||||
func newConnectionStateFromResult(r *handshake.Result) *ConnectionState {
|
||||
func newConnectionStateFromResult(r *handshake.Result) (*ConnectionState, error) {
|
||||
// Refuse a MessageIndex too big for the replay window: it can only be a bug, and would spin the seed loop below.
|
||||
if r.MessageIndex >= ReplayWindow {
|
||||
return nil, fmt.Errorf("handshake message index %d exceeds replay window", r.MessageIndex)
|
||||
}
|
||||
|
||||
ci := &ConnectionState{
|
||||
myCert: r.MyCert,
|
||||
initiator: r.Initiator,
|
||||
@@ -43,7 +60,7 @@ func newConnectionStateFromResult(r *handshake.Result) *ConnectionState {
|
||||
for i := uint64(1); i <= r.MessageIndex; i++ {
|
||||
ci.window.Update(nil, i)
|
||||
}
|
||||
return ci
|
||||
return ci, nil
|
||||
}
|
||||
|
||||
func (cs *ConnectionState) MarshalJSON() ([]byte, error) {
|
||||
@@ -54,6 +71,16 @@ func (cs *ConnectionState) MarshalJSON() ([]byte, error) {
|
||||
})
|
||||
}
|
||||
|
||||
// NextMessageCounter reserves the next 1-based counter; RejectAfterMessages is the first we refuse, pinned to not wrap.
|
||||
func (cs *ConnectionState) NextMessageCounter() (uint64, bool) {
|
||||
c := cs.messageCounter.Add(1)
|
||||
if c >= RejectAfterMessages {
|
||||
cs.messageCounter.Store(RejectAfterMessages)
|
||||
return c, false
|
||||
}
|
||||
return c, true
|
||||
}
|
||||
|
||||
func (cs *ConnectionState) Curve() cert.Curve {
|
||||
return cs.myCert.Curve()
|
||||
}
|
||||
|
||||
@@ -6,10 +6,12 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/flynn/noise"
|
||||
"github.com/rcrowley/go-metrics"
|
||||
"github.com/slackhq/nebula/cert"
|
||||
ct "github.com/slackhq/nebula/cert_test"
|
||||
"github.com/slackhq/nebula/handshake"
|
||||
"github.com/slackhq/nebula/header"
|
||||
"github.com/slackhq/nebula/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -79,11 +81,51 @@ func runTestHandshake(t *testing.T) (initR, respR *handshake.Result) {
|
||||
return initR, respR
|
||||
}
|
||||
|
||||
func TestConnectionState_NextMessageCounter(t *testing.T) {
|
||||
cs := &ConnectionState{}
|
||||
cs.messageCounter.Store(RejectAfterMessages - 2)
|
||||
|
||||
c, ok := cs.NextMessageCounter()
|
||||
assert.True(t, ok)
|
||||
assert.Equal(t, RejectAfterMessages-1, c)
|
||||
|
||||
// Hitting the limit refuses and pins the counter there
|
||||
c, ok = cs.NextMessageCounter()
|
||||
assert.False(t, ok)
|
||||
assert.Equal(t, RejectAfterMessages, c)
|
||||
assert.Equal(t, RejectAfterMessages, cs.messageCounter.Load())
|
||||
|
||||
// Continued send attempts stay refused and the counter never wraps
|
||||
for i := 0; i < 10; i++ {
|
||||
_, ok = cs.NextMessageCounter()
|
||||
assert.False(t, ok)
|
||||
}
|
||||
assert.Equal(t, RejectAfterMessages, cs.messageCounter.Load())
|
||||
}
|
||||
|
||||
// TestSendNoMetricsDropsExhausted drives the send path to the exhausted drop; metric and out flag prove it.
|
||||
func TestSendNoMetricsDropsExhausted(t *testing.T) {
|
||||
initR, _ := runTestHandshake(t)
|
||||
ci, err := newConnectionStateFromResult(initR)
|
||||
require.NoError(t, err)
|
||||
ci.messageCounter.Store(RejectAfterMessages - 1)
|
||||
|
||||
f := &Interface{l: test.NewLogger(), messageMetrics: &MessageMetrics{txExhausted: metrics.NewCounter()}}
|
||||
hostinfo := &HostInfo{vpnAddrs: []netip.Addr{netip.MustParseAddr("10.0.0.1")}, ConnectionState: ci}
|
||||
|
||||
f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, []byte{}, make([]byte, 12), make([]byte, mtu), 0)
|
||||
|
||||
// The crossing send is refused: it records an exhaustion drop and never reaches connectionManager.Out.
|
||||
assert.Equal(t, int64(1), f.messageMetrics.txExhausted.Count())
|
||||
assert.False(t, hostinfo.out.Load())
|
||||
}
|
||||
|
||||
func TestNewConnectionStateFromResult(t *testing.T) {
|
||||
initR, respR := runTestHandshake(t)
|
||||
|
||||
t.Run("initiator", func(t *testing.T) {
|
||||
ci := newConnectionStateFromResult(initR)
|
||||
ci, err := newConnectionStateFromResult(initR)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, ci.initiator)
|
||||
assert.Equal(t, initR.MyCert, ci.myCert)
|
||||
assert.Equal(t, initR.RemoteCert, ci.peerCert)
|
||||
@@ -102,8 +144,17 @@ func TestNewConnectionStateFromResult(t *testing.T) {
|
||||
assert.True(t, ci.window.Check(nil, 3), "counter 3 must not be pre-seeded")
|
||||
})
|
||||
|
||||
t.Run("message index too large is refused", func(t *testing.T) {
|
||||
bad := *initR
|
||||
bad.MessageIndex = ReplayWindow
|
||||
ci, err := newConnectionStateFromResult(&bad)
|
||||
require.Error(t, err)
|
||||
assert.Nil(t, ci)
|
||||
})
|
||||
|
||||
t.Run("responder", func(t *testing.T) {
|
||||
ci := newConnectionStateFromResult(respR)
|
||||
ci, err := newConnectionStateFromResult(respR)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, ci.initiator)
|
||||
assert.Equal(t, respR.MyCert, ci.myCert)
|
||||
assert.Equal(t, respR.RemoteCert, ci.peerCert)
|
||||
|
||||
+9
-1
@@ -53,6 +53,7 @@ type Control struct {
|
||||
statsStart func()
|
||||
dnsStart func()
|
||||
lighthouseStart func()
|
||||
networkChangeStart func(rebind func())
|
||||
connectionManagerStart func(context.Context)
|
||||
}
|
||||
|
||||
@@ -104,6 +105,9 @@ func (c *Control) Start() error {
|
||||
if c.dnsStart != nil {
|
||||
go c.dnsStart()
|
||||
}
|
||||
if c.networkChangeStart != nil {
|
||||
go c.networkChangeStart(c.RebindUDPServer)
|
||||
}
|
||||
if c.connectionManagerStart != nil {
|
||||
go c.connectionManagerStart(c.ctx)
|
||||
}
|
||||
@@ -198,7 +202,11 @@ func (c *Control) RebindUDPServer() {
|
||||
return
|
||||
}
|
||||
|
||||
_ = c.f.outside.Rebind()
|
||||
// A failure here means we are likely still pinned to the interface we came up on, so the rest of this is
|
||||
// unlikely to help. Say so instead of silently carrying on as if we rebound.
|
||||
if err := c.f.outside.Rebind(); err != nil {
|
||||
c.l.Error("Failed to rebind udp socket", "error", err)
|
||||
}
|
||||
|
||||
// Trigger a lighthouse update, useful for mobile clients that should have an update interval of 0
|
||||
c.f.lightHouse.SendUpdate()
|
||||
|
||||
+13
-1
@@ -108,7 +108,19 @@ func (c *Control) GetVpnAddrs() []netip.Addr {
|
||||
}
|
||||
|
||||
func (c *Control) GetUDPAddr() netip.AddrPort {
|
||||
return c.f.outside.(*udp.TesterConn).Addr
|
||||
return c.f.outside.(*udp.TesterConn).GetAddr()
|
||||
}
|
||||
|
||||
// SetUDPAddr moves this node to a new underlay address, standing in for a laptop waking up on a different
|
||||
// network. Register the new address with the router as well or nothing will route back.
|
||||
func (c *Control) SetUDPAddr(addr netip.AddrPort) {
|
||||
c.f.outside.(*udp.TesterConn).SetAddr(addr)
|
||||
}
|
||||
|
||||
// SetLocalAddrsFn replaces underlay address discovery so a test can advertise its simulated address instead of
|
||||
// whatever this machine's NICs happen to be. Call it before Start, SendUpdate reads it from the update worker.
|
||||
func (c *Control) SetLocalAddrsFn(fn func(*LocalAllowList) []netip.Addr) {
|
||||
c.f.lightHouse.localAddrsFn = fn
|
||||
}
|
||||
|
||||
func (c *Control) KillPendingTunnel(vpnIp netip.Addr) bool {
|
||||
|
||||
@@ -0,0 +1,225 @@
|
||||
//go:build e2e_testing
|
||||
// +build e2e_testing
|
||||
|
||||
package e2e
|
||||
|
||||
import (
|
||||
"net/netip"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/slackhq/nebula"
|
||||
"github.com/slackhq/nebula/cert"
|
||||
"github.com/slackhq/nebula/cert_test"
|
||||
"github.com/slackhq/nebula/e2e/router"
|
||||
"github.com/slackhq/nebula/header"
|
||||
"github.com/slackhq/nebula/udp"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// reportedAddrs is what the lighthouse would hand a peer asking where vpnAddr is.
|
||||
func reportedAddrs(t *testing.T, lh *nebula.Control, vpnAddr netip.Addr) []netip.AddrPort {
|
||||
t.Helper()
|
||||
cm := lh.QueryLighthouse(vpnAddr)
|
||||
if cm == nil {
|
||||
return nil
|
||||
}
|
||||
var out []netip.AddrPort
|
||||
for _, c := range *cm {
|
||||
out = append(out, c.Reported...)
|
||||
out = append(out, c.Learned...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// waitForLighthouseMsg routes until a lighthouse message lands on lh, or gives up. Reports whether one arrived.
|
||||
func waitForLighthouseMsg(t *testing.T, r *router.R, lh *nebula.Control, wait time.Duration) bool {
|
||||
t.Helper()
|
||||
h := &header.H{}
|
||||
return r.RouteForAllExitFuncOrTimeout(wait, func(p *udp.Packet, c *nebula.Control) router.ExitType {
|
||||
if c != lh {
|
||||
return router.KeepRouting
|
||||
}
|
||||
// Punches are a single byte and never parse, they are just not what we are after
|
||||
if err := h.Parse(p.Data); err != nil {
|
||||
return router.KeepRouting
|
||||
}
|
||||
if h.Type == header.LightHouse {
|
||||
return router.RouteAndExit
|
||||
}
|
||||
return router.KeepRouting
|
||||
})
|
||||
}
|
||||
|
||||
// A laptop that changes networks has to tell the lighthouse promptly, otherwise the lighthouse keeps handing peers
|
||||
// the old address and their punches land nowhere. On a long lighthouse interval the only thing that closes that
|
||||
// window is the rebind, which on darwin the network change monitor drives. The e2e build compiles the monitor out,
|
||||
// so we call RebindUDPServer directly, which is the same thing the monitor does.
|
||||
func TestRebindSendsLighthouseUpdate(t *testing.T) {
|
||||
t.Parallel()
|
||||
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
|
||||
|
||||
lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
|
||||
"lighthouse": m{"am_lighthouse": true},
|
||||
})
|
||||
|
||||
// 600s interval, so nothing scheduled can send an update during this test. A rebind is the only thing that can.
|
||||
myControl, _, _, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{
|
||||
"lighthouse": m{
|
||||
"hosts": []any{lhVpnIpNet[0].Addr().String()},
|
||||
"interval": 600,
|
||||
},
|
||||
"static_host_map": m{
|
||||
lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
|
||||
},
|
||||
})
|
||||
|
||||
r := router.NewR(t, lhControl, myControl)
|
||||
defer r.RenderFlow()
|
||||
|
||||
lhControl.Start()
|
||||
myControl.Start()
|
||||
|
||||
// Let the startup registration finish, then clear everything it left behind
|
||||
require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an initial registration")
|
||||
r.RouteFor(time.Millisecond * 400)
|
||||
|
||||
// Nothing should be talking to the lighthouse on its own now
|
||||
require.False(t, waitForLighthouseMsg(t, r, lhControl, time.Millisecond*200),
|
||||
"nothing should reach the lighthouse before the rebind")
|
||||
|
||||
myControl.RebindUDPServer()
|
||||
|
||||
assert.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5),
|
||||
"a rebind should push an update to the lighthouse rather than waiting out the interval")
|
||||
|
||||
lhControl.Stop()
|
||||
myControl.Stop()
|
||||
}
|
||||
|
||||
// The other half of a rebind: every live tunnel requeries the lighthouse on its next send. That query is what makes
|
||||
// the lighthouse tell the peer to punch toward our new address, which is the part that actually revives a tunnel
|
||||
// whose remote NAT state died while we were on a different network.
|
||||
func TestRebindRequeriesPeersOnNextSend(t *testing.T) {
|
||||
t.Parallel()
|
||||
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
|
||||
|
||||
lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
|
||||
"lighthouse": m{"am_lighthouse": true},
|
||||
})
|
||||
|
||||
lhCfg := m{
|
||||
"lighthouse": m{
|
||||
"hosts": []any{lhVpnIpNet[0].Addr().String()},
|
||||
"interval": 600,
|
||||
// Without this the peers advertise this machine's real addresses and then try to punch at them,
|
||||
// which the router has no route for.
|
||||
"local_allow_list": m{
|
||||
"10.0.0.0/24": true,
|
||||
"::/0": false,
|
||||
},
|
||||
},
|
||||
"static_host_map": m{
|
||||
lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
|
||||
},
|
||||
}
|
||||
|
||||
myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", lhCfg)
|
||||
theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.3/24", lhCfg)
|
||||
|
||||
r := router.NewR(t, lhControl, myControl, theirControl)
|
||||
defer r.RenderFlow()
|
||||
|
||||
lhControl.Start()
|
||||
myControl.Start()
|
||||
theirControl.Start()
|
||||
r.RouteFor(time.Millisecond * 500)
|
||||
|
||||
// Point the peers at each other directly, this test is about the rebind and not about lighthouse discovery
|
||||
myControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
|
||||
theirControl.InjectLightHouseAddr(myVpnIpNet[0].Addr(), myUdpAddr)
|
||||
|
||||
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("initial")))
|
||||
r.RouteFor(time.Second)
|
||||
require.NotNil(t, myControl.GetHostInfoByVpnAddr(theirVpnIpNet[0].Addr(), false), "expected a tunnel to them")
|
||||
r.RouteFor(time.Millisecond * 300)
|
||||
|
||||
// Assert on what the peer sees rather than on lighthouse traffic. A query for them makes the lighthouse send
|
||||
// them a punch notification, which is the whole point. Our own update to the lighthouse sends them nothing,
|
||||
// so this cannot be satisfied by the update the rebind itself pushes.
|
||||
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("quiet")))
|
||||
require.False(t, waitForLighthouseMsg(t, r, theirControl, time.Millisecond*300),
|
||||
"an ordinary send should not requery the lighthouse")
|
||||
|
||||
myControl.RebindUDPServer()
|
||||
r.RouteFor(time.Millisecond * 300) // let the update the rebind itself sends pass by
|
||||
|
||||
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("after rebind")))
|
||||
assert.True(t, waitForLighthouseMsg(t, r, theirControl, time.Second*5),
|
||||
"the first send after a rebind should requery the lighthouse, which then tells the peer to punch at us")
|
||||
|
||||
lhControl.Stop()
|
||||
myControl.Stop()
|
||||
theirControl.Stop()
|
||||
}
|
||||
|
||||
// The scenario this whole thing exists for: a laptop sleeps at the office and wakes up at home on a new address.
|
||||
// Until it tells the lighthouse, the lighthouse keeps handing peers the office address, so their punches land
|
||||
// nowhere and the tunnel stays dead. On a long interval the rebind is the only thing that closes that window.
|
||||
func TestRebindAdvertisesNewAddressAfterMove(t *testing.T) {
|
||||
t.Parallel()
|
||||
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
|
||||
|
||||
lhControl, lhVpnIpNet, lhUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "lh", "10.128.0.1/24", m{
|
||||
"lighthouse": m{"am_lighthouse": true},
|
||||
})
|
||||
|
||||
myControl, myVpnIpNet, myUdpAddr, _ := newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.2/24", m{
|
||||
"lighthouse": m{
|
||||
"hosts": []any{lhVpnIpNet[0].Addr().String()},
|
||||
"interval": 600,
|
||||
},
|
||||
"static_host_map": m{
|
||||
lhVpnIpNet[0].Addr().String(): []any{lhUdpAddr.String()},
|
||||
},
|
||||
})
|
||||
|
||||
// Advertise wherever we currently are rather than this machine's real NICs, read fresh each time so a move
|
||||
// is picked up.
|
||||
myControl.SetLocalAddrsFn(func(*nebula.LocalAllowList) []netip.Addr {
|
||||
return []netip.Addr{myControl.GetUDPAddr().Addr()}
|
||||
})
|
||||
|
||||
r := router.NewR(t, lhControl, myControl)
|
||||
defer r.RenderFlow()
|
||||
|
||||
lhControl.Start()
|
||||
myControl.Start()
|
||||
|
||||
require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an initial registration")
|
||||
r.RouteFor(time.Millisecond * 400)
|
||||
|
||||
require.Contains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), myUdpAddr,
|
||||
"the lighthouse should know the address we started on")
|
||||
|
||||
// Wake up somewhere else
|
||||
newAddr := netip.MustParseAddrPort("10.0.0.99:4242")
|
||||
myControl.SetUDPAddr(newAddr)
|
||||
r.AddRoute(newAddr.Addr(), newAddr.Port(), myControl)
|
||||
|
||||
// Nothing has told the lighthouse, and with interval 600 nothing scheduled will
|
||||
r.RouteFor(time.Millisecond * 400)
|
||||
require.NotContains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), newAddr,
|
||||
"the lighthouse should still be handing out the old address before the rebind")
|
||||
|
||||
myControl.RebindUDPServer()
|
||||
require.True(t, waitForLighthouseMsg(t, r, lhControl, time.Second*5), "expected an update after the rebind")
|
||||
r.RouteFor(time.Millisecond * 400)
|
||||
|
||||
assert.Contains(t, reportedAddrs(t, lhControl, myVpnIpNet[0].Addr()), newAddr,
|
||||
"after the rebind the lighthouse should hand peers our new address")
|
||||
|
||||
lhControl.Stop()
|
||||
myControl.Stop()
|
||||
}
|
||||
+129
-27
@@ -114,6 +114,28 @@ type packet struct {
|
||||
packet *udp.Packet
|
||||
tun bool // a packet pulled off a tun device
|
||||
rx bool // the packet was received by a udp device
|
||||
|
||||
// h is the nebula header, parsed once when the packet is recorded. parseErr says why there isn't one, which
|
||||
// the flow log reports rather than hiding. Punchy sends a single byte, so an unparseable packet is normal.
|
||||
h header.H
|
||||
parseErr error
|
||||
}
|
||||
|
||||
// fromAddr and toAddr are the addresses this packet actually travelled between. Reading them off the control
|
||||
// instead would misreport the whole history once a test moves a node. Tun packets are synthesized without
|
||||
// addresses, so they fall back to the control.
|
||||
func (p *packet) fromAddr() netip.AddrPort {
|
||||
if p.tun || !p.packet.From.IsValid() {
|
||||
return p.from.GetUDPAddr()
|
||||
}
|
||||
return p.packet.From
|
||||
}
|
||||
|
||||
func (p *packet) toAddr() netip.AddrPort {
|
||||
if p.tun || !p.packet.To.IsValid() {
|
||||
return p.to.GetUDPAddr()
|
||||
}
|
||||
return p.packet.To
|
||||
}
|
||||
|
||||
func (p *packet) WasReceived() {
|
||||
@@ -249,7 +271,7 @@ func (r *R) renderFlow() {
|
||||
continue
|
||||
}
|
||||
|
||||
addr := e.packet.from.GetUDPAddr()
|
||||
addr := e.packet.fromAddr()
|
||||
if _, ok := participants[addr]; ok {
|
||||
continue
|
||||
}
|
||||
@@ -268,7 +290,6 @@ func (r *R) renderFlow() {
|
||||
}
|
||||
|
||||
// Print packets
|
||||
h := &header.H{}
|
||||
for _, e := range r.flow {
|
||||
if e.packet == nil {
|
||||
//fmt.Fprintf(f, " note over %s: %s\n", strings.Join(participantsVals, ", "), e.note)
|
||||
@@ -280,21 +301,22 @@ func (r *R) renderFlow() {
|
||||
fmt.Fprintln(f, r.formatUdpPacket(p))
|
||||
|
||||
} else {
|
||||
if err := h.Parse(p.packet.Data); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
line := "--x"
|
||||
if p.rx {
|
||||
line = "->>"
|
||||
}
|
||||
|
||||
fmt.Fprintf(f,
|
||||
" %s%s%s: %s(%s), index %v, counter: %v\n",
|
||||
normalizeName(p.from.GetUDPAddr().String()),
|
||||
detail := fmt.Sprintf("%s(%s), index %v, counter: %v",
|
||||
p.h.TypeName(), p.h.SubTypeName(), p.h.RemoteIndex, p.h.MessageCounter)
|
||||
if p.parseErr != nil {
|
||||
detail = fmt.Sprintf("unparsed, %v (%d bytes)", p.parseErr, len(p.packet.Data))
|
||||
}
|
||||
|
||||
fmt.Fprintf(f, " %s%s%s: %s\n",
|
||||
normalizeName(p.fromAddr().String()),
|
||||
line,
|
||||
normalizeName(p.to.GetUDPAddr().String()),
|
||||
h.TypeName(), h.SubTypeName(), h.RemoteIndex, h.MessageCounter,
|
||||
normalizeName(p.toAddr().String()),
|
||||
detail,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -408,29 +430,34 @@ func (r *R) unlockedInjectFlow(from, to *nebula.Control, p *udp.Packet, tun bool
|
||||
|
||||
r.renderHostmaps(fmt.Sprintf("Packet %v", len(r.flow)))
|
||||
|
||||
if len(r.ignoreFlows) > 0 {
|
||||
var h header.H
|
||||
err := h.Parse(p.Data)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
var h header.H
|
||||
var parseErr error
|
||||
if !tun {
|
||||
parseErr = h.Parse(p.Data)
|
||||
}
|
||||
|
||||
for _, i := range r.ignoreFlows {
|
||||
if !tun {
|
||||
if i.messageType == h.Type && i.subType == h.Subtype {
|
||||
return nil
|
||||
}
|
||||
} else if i.tun.HasValue && i.tun.IsTrue {
|
||||
// Decide before copying, the copy comes from a freelist and an ignored packet would never be released
|
||||
for _, i := range r.ignoreFlows {
|
||||
if tun {
|
||||
if i.tun.HasValue && i.tun.IsTrue {
|
||||
return nil
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// A packet we could not parse has no type to match against, so no rule can ignore it
|
||||
if parseErr == nil && i.messageType == h.Type && i.subType == h.Subtype {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
fp := &packet{
|
||||
from: from,
|
||||
to: to,
|
||||
packet: p.Copy(),
|
||||
tun: tun,
|
||||
from: from,
|
||||
to: to,
|
||||
packet: p.Copy(),
|
||||
tun: tun,
|
||||
h: h,
|
||||
parseErr: parseErr,
|
||||
}
|
||||
|
||||
r.flow = append(r.flow, flowEntry{packet: fp})
|
||||
@@ -690,6 +717,81 @@ func (r *R) RouteUntilAfterMsgType(sender *nebula.Control, msgType header.Messag
|
||||
})
|
||||
}
|
||||
|
||||
// RouteFor routes everything that shows up for the given duration and then returns. Use it to let a test settle
|
||||
// deterministically rather than sleeping and hoping: a single FlushAll races a completing handshake, which queues
|
||||
// more packets right behind it.
|
||||
func (r *R) RouteFor(d time.Duration) {
|
||||
r.RouteForAllExitFuncOrTimeout(d, func(*udp.Packet, *nebula.Control) ExitType {
|
||||
return KeepRouting
|
||||
})
|
||||
}
|
||||
|
||||
// RouteForAllExitFuncOrTimeout is RouteForAllExitFunc with a deadline, reporting whether whatDo asked to exit
|
||||
// before time ran out. The unbounded version blocks forever on a quiet network, so this is what a test needs to
|
||||
// assert that something does NOT happen, or to route for a fixed settling period.
|
||||
func (r *R) RouteForAllExitFuncOrTimeout(timeout time.Duration, whatDo ExitFunc) bool {
|
||||
sc := make([]reflect.SelectCase, 0, len(r.controls)+1)
|
||||
cm := make([]*nebula.Control, 0, len(r.controls))
|
||||
|
||||
for _, c := range r.controls {
|
||||
sc = append(sc, reflect.SelectCase{
|
||||
Dir: reflect.SelectRecv,
|
||||
Chan: reflect.ValueOf(c.GetUDPTxChan()),
|
||||
Send: reflect.Value{},
|
||||
})
|
||||
cm = append(cm, c)
|
||||
}
|
||||
|
||||
timer := time.NewTimer(timeout)
|
||||
defer timer.Stop()
|
||||
sc = append(sc, reflect.SelectCase{
|
||||
Dir: reflect.SelectRecv,
|
||||
Chan: reflect.ValueOf(timer.C),
|
||||
Send: reflect.Value{},
|
||||
})
|
||||
|
||||
for {
|
||||
x, rx, _ := reflect.Select(sc)
|
||||
if x == len(cm) {
|
||||
return false
|
||||
}
|
||||
|
||||
r.Lock()
|
||||
p := rx.Interface().(*udp.Packet)
|
||||
receiver := r.getControl(cm[x].GetUDPAddr(), p.To, p)
|
||||
if receiver == nil {
|
||||
r.Unlock()
|
||||
panic("Can't RouteForAllExitFuncOrTimeout for host: " + p.To.String())
|
||||
}
|
||||
|
||||
e := whatDo(p, receiver)
|
||||
switch e {
|
||||
case ExitNow:
|
||||
r.Unlock()
|
||||
p.Release()
|
||||
return true
|
||||
|
||||
case RouteAndExit:
|
||||
fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
|
||||
receiver.InjectUDPPacket(p)
|
||||
fp.WasReceived()
|
||||
r.Unlock()
|
||||
p.Release()
|
||||
return true
|
||||
|
||||
case KeepRouting:
|
||||
fp := r.unlockedInjectFlow(cm[x], receiver, p, false)
|
||||
receiver.InjectUDPPacket(p)
|
||||
fp.WasReceived()
|
||||
|
||||
default:
|
||||
panic(fmt.Sprintf("Unknown exitFunc return: %v", e))
|
||||
}
|
||||
r.Unlock()
|
||||
p.Release()
|
||||
}
|
||||
}
|
||||
|
||||
func (r *R) RouteForAllUntilAfterMsgTypeTo(receiver *nebula.Control, msgType header.MessageType, subType header.MessageSubType) {
|
||||
h := &header.H{}
|
||||
r.RouteForAllExitFunc(func(p *udp.Packet, r *nebula.Control) ExitType {
|
||||
|
||||
+8
-14
@@ -146,6 +146,14 @@ listen:
|
||||
# Default true; set to false to leave WDF in charge of inbound decisions on the listener port. Not reloadable.
|
||||
#windows_bypass_wdf: true
|
||||
|
||||
# On macOS only
|
||||
# macOS scopes the udp socket to the interface it was created on, so moving between networks (wifi to wired,
|
||||
# office to home) leaves Nebula sending out an interface that no longer has a route. When true, Nebula watches
|
||||
# the routing socket and rebinds the listener once the change settles.
|
||||
# iOS does not use this, the host app drives the same rebind itself.
|
||||
# Default true. Not reloadable.
|
||||
#rebind_on_network_change: true
|
||||
|
||||
# By default, Nebula replies to packets it has no tunnel for with a "recv_error" packet. This packet helps speed up reconnection
|
||||
# in the case that Nebula on either side did not shut down cleanly. This response can be abused as a way to discover if Nebula is running
|
||||
# on a host though. This option lets you configure if you want to send "recv_error" packets always, never, or only to private network remotes.
|
||||
@@ -390,20 +398,6 @@ logging:
|
||||
# This setting is reloadable
|
||||
#inactivity_timeout: 10m
|
||||
|
||||
# clear_on_wake controls whether all tunnels are immediately torn down (locally, without notifying the remotes)
|
||||
# when the machine detects it has just woken from system sleep. Tunnels rarely survive a suspend: NAT mappings
|
||||
# expire and the machine's address usually changes, so waiting for the normal liveness checks costs 15-20 seconds
|
||||
# of black-holed traffic per tunnel after wake. Clearing them means the first packet after wake starts a fresh
|
||||
# handshake right away.
|
||||
# This setting is reloadable
|
||||
#clear_on_wake: true
|
||||
|
||||
# wake_clear_threshold is the minimum time the machine must have been suspended for clear_on_wake to act.
|
||||
# Suspends shorter than this often come back before NAT state expires, so those tunnels may still be alive and
|
||||
# are left to the normal liveness checks. Values below 1s are treated as 1s.
|
||||
# This setting is reloadable
|
||||
#wake_clear_threshold: 30s
|
||||
|
||||
# Nebula security group configuration
|
||||
firewall:
|
||||
# Action to take when a packet is not allowed by the firewall rules.
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
module github.com/slackhq/nebula
|
||||
|
||||
go 1.25.0
|
||||
go 1.26.0
|
||||
|
||||
require (
|
||||
dario.cat/mergo v1.0.2
|
||||
filippo.io/bigmod v0.1.0
|
||||
github.com/DefinedNet/netlink v1.3.2-dn.1
|
||||
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
|
||||
github.com/armon/go-radix v1.0.0
|
||||
github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432
|
||||
github.com/flynn/noise v1.1.0
|
||||
github.com/gaissmai/bart v0.28.0
|
||||
github.com/gogo/protobuf v1.3.2
|
||||
@@ -21,7 +21,6 @@ require (
|
||||
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
|
||||
github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
|
||||
github.com/stretchr/testify v1.11.1
|
||||
github.com/vishvananda/netlink v1.3.1
|
||||
go.uber.org/goleak v1.3.0
|
||||
go.yaml.in/yaml/v3 v3.0.4
|
||||
golang.org/x/crypto v0.54.0
|
||||
|
||||
@@ -3,6 +3,8 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
|
||||
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
|
||||
filippo.io/bigmod v0.1.0 h1:UNzDk7y9ADKST+axd9skUpBQeW7fG2KrTZyOE4uGQy8=
|
||||
filippo.io/bigmod v0.1.0/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI=
|
||||
github.com/DefinedNet/netlink v1.3.2-dn.1 h1:Md2PsUE1Xuy1Zxs/Bm6356n1kt5bq2HspXlPJ6pEspg=
|
||||
github.com/DefinedNet/netlink v1.3.2-dn.1/go.mod h1:o4YSn8Jw7zoDcYXDmV77TRd4eS5sSLuE+vu2zAN67C8=
|
||||
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
|
||||
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
|
||||
@@ -19,8 +21,6 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432 h1:M5QgkYacWj0Xs8MhpIK/5uwU02icXpEoSo9sM2aRCps=
|
||||
github.com/cyberdelia/go-metrics-graphite v0.0.0-20161219230853-39f87cc3b432/go.mod h1:xwIwAxMvYnVrGJPe2FKx5prTrnAjGOD8zvDOnxnrrkM=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
@@ -145,8 +145,6 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
|
||||
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
|
||||
github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
|
||||
github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
|
||||
github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
|
||||
github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
@@ -206,8 +204,6 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
package nebula
|
||||
|
||||
// This file is a trimmed, inlined copy of the graphite exporter from
|
||||
// github.com/cyberdelia/go-metrics-graphite, retaining only the Config type and
|
||||
// the Once entrypoint that Nebula uses. The upstream package has been
|
||||
// unmaintained for 10+ years, so it was vendored here to drop the dependency.
|
||||
// See https://github.com/slackhq/nebula/issues/1831.
|
||||
//
|
||||
// Copyright 2015 Timothée Peignier. All rights reserved.
|
||||
//
|
||||
// Redistribution and use in source and binary forms, with or without
|
||||
// modification, are permitted provided that the following conditions are met:
|
||||
//
|
||||
// 1. Redistributions of source code must retain the above copyright notice,
|
||||
// this list of conditions and the following disclaimer.
|
||||
//
|
||||
// 2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
// this list of conditions and the following disclaimer in the documentation
|
||||
// and/or other materials provided with the distribution.
|
||||
//
|
||||
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/rcrowley/go-metrics"
|
||||
)
|
||||
|
||||
// graphiteConfigExport provides a container with configuration parameters for
|
||||
// the Graphite exporter.
|
||||
type graphiteConfigExport struct {
|
||||
Addr *net.TCPAddr // Network address to connect to
|
||||
Registry metrics.Registry // Registry to be exported
|
||||
FlushInterval time.Duration // Flush interval
|
||||
DurationUnit time.Duration // Time conversion unit for durations
|
||||
Prefix string // Prefix to be prepended to metric names
|
||||
Percentiles []float64 // Percentiles to export from timers and histograms
|
||||
}
|
||||
|
||||
// graphiteOnce performs a single submission to Graphite, returning a non-nil
|
||||
// error on failed connections.
|
||||
func graphiteOnce(c graphiteConfigExport) error {
|
||||
now := time.Now().Unix()
|
||||
du := float64(c.DurationUnit)
|
||||
flushSeconds := float64(c.FlushInterval) / float64(time.Second)
|
||||
conn, err := net.DialTCP("tcp", nil, c.Addr)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer conn.Close()
|
||||
w := bufio.NewWriter(conn)
|
||||
c.Registry.Each(func(name string, i any) {
|
||||
switch metric := i.(type) {
|
||||
case metrics.Counter:
|
||||
count := metric.Count()
|
||||
fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, count, now)
|
||||
fmt.Fprintf(w, "%s.%s.count_ps %.2f %d\n", c.Prefix, name, float64(count)/flushSeconds, now)
|
||||
case metrics.Gauge:
|
||||
fmt.Fprintf(w, "%s.%s.value %d %d\n", c.Prefix, name, metric.Value(), now)
|
||||
case metrics.GaugeFloat64:
|
||||
fmt.Fprintf(w, "%s.%s.value %f %d\n", c.Prefix, name, metric.Value(), now)
|
||||
case metrics.Histogram:
|
||||
h := metric.Snapshot()
|
||||
ps := h.Percentiles(c.Percentiles)
|
||||
fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, h.Count(), now)
|
||||
fmt.Fprintf(w, "%s.%s.min %d %d\n", c.Prefix, name, h.Min(), now)
|
||||
fmt.Fprintf(w, "%s.%s.max %d %d\n", c.Prefix, name, h.Max(), now)
|
||||
fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, h.Mean(), now)
|
||||
fmt.Fprintf(w, "%s.%s.std-dev %.2f %d\n", c.Prefix, name, h.StdDev(), now)
|
||||
for psIdx, psKey := range c.Percentiles {
|
||||
key := strings.Replace(strconv.FormatFloat(psKey*100.0, 'f', -1, 64), ".", "", 1)
|
||||
fmt.Fprintf(w, "%s.%s.%s-percentile %.2f %d\n", c.Prefix, name, key, ps[psIdx], now)
|
||||
}
|
||||
case metrics.Meter:
|
||||
m := metric.Snapshot()
|
||||
fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, m.Count(), now)
|
||||
fmt.Fprintf(w, "%s.%s.one-minute %.2f %d\n", c.Prefix, name, m.Rate1(), now)
|
||||
fmt.Fprintf(w, "%s.%s.five-minute %.2f %d\n", c.Prefix, name, m.Rate5(), now)
|
||||
fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, m.Rate15(), now)
|
||||
fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, m.RateMean(), now)
|
||||
case metrics.Timer:
|
||||
t := metric.Snapshot()
|
||||
ps := t.Percentiles(c.Percentiles)
|
||||
count := t.Count()
|
||||
fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, count, now)
|
||||
fmt.Fprintf(w, "%s.%s.count_ps %.2f %d\n", c.Prefix, name, float64(count)/flushSeconds, now)
|
||||
fmt.Fprintf(w, "%s.%s.min %d %d\n", c.Prefix, name, t.Min()/int64(du), now)
|
||||
fmt.Fprintf(w, "%s.%s.max %d %d\n", c.Prefix, name, t.Max()/int64(du), now)
|
||||
fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, t.Mean()/du, now)
|
||||
fmt.Fprintf(w, "%s.%s.std-dev %.2f %d\n", c.Prefix, name, t.StdDev()/du, now)
|
||||
for psIdx, psKey := range c.Percentiles {
|
||||
key := strings.Replace(strconv.FormatFloat(psKey*100.0, 'f', -1, 64), ".", "", 1)
|
||||
fmt.Fprintf(w, "%s.%s.%s-percentile %.2f %d\n", c.Prefix, name, key, ps[psIdx]/du, now)
|
||||
}
|
||||
fmt.Fprintf(w, "%s.%s.one-minute %.2f %d\n", c.Prefix, name, t.Rate1(), now)
|
||||
fmt.Fprintf(w, "%s.%s.five-minute %.2f %d\n", c.Prefix, name, t.Rate5(), now)
|
||||
fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, t.Rate15(), now)
|
||||
fmt.Fprintf(w, "%s.%s.mean-rate %.2f %d\n", c.Prefix, name, t.RateMean(), now)
|
||||
}
|
||||
w.Flush()
|
||||
})
|
||||
return nil
|
||||
}
|
||||
+21
-3
@@ -295,7 +295,13 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
|
||||
hm.messageMetrics.Tx(header.Handshake, hh.machine.Subtype(), 1)
|
||||
err := hm.outside.WriteTo(stage0, addr)
|
||||
if err != nil {
|
||||
hostinfo.logger(hm.l).Error("Failed to send handshake message",
|
||||
// These repeat every attempt, so match the success log below and only shout when the remotes changed
|
||||
level := slog.LevelDebug
|
||||
if remotesHaveChanged {
|
||||
level = slog.LevelError
|
||||
}
|
||||
|
||||
hostinfo.logger(hm.l).Log(context.Background(), level, "Failed to send handshake message",
|
||||
"udpAddr", addr,
|
||||
"initiatorIndex", hostinfo.localIndexId,
|
||||
"handshake", hsFields,
|
||||
@@ -743,8 +749,14 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
|
||||
return
|
||||
}
|
||||
|
||||
connState, err := newConnectionStateFromResult(result)
|
||||
if err != nil {
|
||||
f.l.Error("Discarding handshake with an invalid message index", "error", err, "vpnAddrs", vpnAddrs)
|
||||
return
|
||||
}
|
||||
|
||||
hostinfo := &HostInfo{
|
||||
ConnectionState: newConnectionStateFromResult(result),
|
||||
ConnectionState: connState,
|
||||
localIndexId: result.LocalIndex,
|
||||
remoteIndexId: result.RemoteIndex,
|
||||
vpnAddrs: vpnAddrs,
|
||||
@@ -862,7 +874,13 @@ func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostIn
|
||||
}
|
||||
|
||||
// Handshake complete; build the ConnectionState now that we have keys and a verified peer cert.
|
||||
hostinfo.ConnectionState = newConnectionStateFromResult(result)
|
||||
cs, err := newConnectionStateFromResult(result)
|
||||
if err != nil {
|
||||
f.l.Error("Discarding handshake with an invalid message index", "error", err, "vpnAddrs", hostinfo.vpnAddrs)
|
||||
hm.DeleteHostInfo(hostinfo)
|
||||
return
|
||||
}
|
||||
hostinfo.ConnectionState = cs
|
||||
|
||||
remoteCert := result.RemoteCert
|
||||
if remoteCert == nil {
|
||||
|
||||
-58
@@ -6,7 +6,6 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/netip"
|
||||
"slices"
|
||||
"sync"
|
||||
@@ -18,7 +17,6 @@ import (
|
||||
"github.com/slackhq/nebula/cert"
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/header"
|
||||
"github.com/slackhq/nebula/logging"
|
||||
)
|
||||
|
||||
const defaultPromoteEvery = 1000 // Count of packets sent before we try moving a tunnel to a preferred underlay ip address
|
||||
@@ -865,59 +863,3 @@ func (i *HostInfo) logger(l *slog.Logger) *slog.Logger {
|
||||
|
||||
return li
|
||||
}
|
||||
|
||||
// Utility functions
|
||||
|
||||
func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
|
||||
//FIXME: This function is pretty garbage
|
||||
var finalAddrs []netip.Addr
|
||||
ifaces, _ := net.Interfaces()
|
||||
for _, i := range ifaces {
|
||||
allow := allowList.AllowName(i.Name)
|
||||
if l.Enabled(context.Background(), logging.LevelTrace) {
|
||||
l.Log(context.Background(), logging.LevelTrace, "localAllowList.AllowName",
|
||||
"interfaceName", i.Name,
|
||||
"allow", allow,
|
||||
)
|
||||
}
|
||||
|
||||
if !allow {
|
||||
continue
|
||||
}
|
||||
addrs, _ := i.Addrs()
|
||||
for _, rawAddr := range addrs {
|
||||
var addr netip.Addr
|
||||
switch v := rawAddr.(type) {
|
||||
case *net.IPNet:
|
||||
//continue
|
||||
addr, _ = netip.AddrFromSlice(v.IP)
|
||||
case *net.IPAddr:
|
||||
addr, _ = netip.AddrFromSlice(v.IP)
|
||||
}
|
||||
|
||||
if !addr.IsValid() {
|
||||
if l.Enabled(context.Background(), slog.LevelDebug) {
|
||||
l.Debug("addr was invalid", "localAddr", rawAddr)
|
||||
}
|
||||
continue
|
||||
}
|
||||
addr = addr.Unmap()
|
||||
|
||||
if addr.IsLoopback() == false && addr.IsLinkLocalUnicast() == false {
|
||||
isAllowed := allowList.Allow(addr)
|
||||
if l.Enabled(context.Background(), logging.LevelTrace) {
|
||||
l.Log(context.Background(), logging.LevelTrace, "localAllowList.Allow",
|
||||
"localAddr", addr,
|
||||
"allowed", isAllowed,
|
||||
)
|
||||
}
|
||||
if !isAllowed {
|
||||
continue
|
||||
}
|
||||
|
||||
finalAddrs = append(finalAddrs, addr)
|
||||
}
|
||||
}
|
||||
}
|
||||
return finalAddrs
|
||||
}
|
||||
@@ -37,6 +37,7 @@ func (f *Interface) consumeInsidePacket(packet []byte, fwPacket *firewall.Packet
|
||||
// routes packets from the Nebula addr to the Nebula addr through the Nebula
|
||||
// TUN device.
|
||||
if immediatelyForwardToSelf {
|
||||
iputil.SetTransportChecksum(packet)
|
||||
_, err := f.readers[q].Write(packet)
|
||||
if err != nil {
|
||||
f.l.Error("Failed to forward to tun", "error", err)
|
||||
@@ -275,6 +276,14 @@ func (f *Interface) sendTo(t header.MessageType, st header.MessageSubType, ci *C
|
||||
f.sendNoMetrics(t, st, ci, hostinfo, remote, p, nb, out, 0)
|
||||
}
|
||||
|
||||
// dropExhausted records an exhaustion drop and logs once, on the crossing send, for a spent tunnel.
|
||||
func (f *Interface) dropExhausted(hostinfo *HostInfo, c uint64, msg string) {
|
||||
f.messageMetrics.TxExhausted(1)
|
||||
if c == RejectAfterMessages {
|
||||
hostinfo.logger(f.l).Error(msg)
|
||||
}
|
||||
}
|
||||
|
||||
// SendVia sends a payload through a Relay tunnel. No authentication or encryption is done
|
||||
// to the payload for the ultimate target host, making this a useful method for sending
|
||||
// handshake messages to peers through relay tunnels.
|
||||
@@ -294,7 +303,14 @@ func (f *Interface) SendVia(via *HostInfo,
|
||||
// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
|
||||
via.ConnectionState.writeLock.Lock()
|
||||
}
|
||||
c := via.ConnectionState.messageCounter.Add(1)
|
||||
c, ok := via.ConnectionState.NextMessageCounter()
|
||||
if !ok {
|
||||
if noiseutil.EncryptLockNeeded {
|
||||
via.ConnectionState.writeLock.Unlock()
|
||||
}
|
||||
f.dropExhausted(via, c, "Dropping outbound relay packets, tunnel message counter is exhausted")
|
||||
return
|
||||
}
|
||||
|
||||
out = header.Encode(out, header.Version, header.Message, header.MessageRelay, relay.RemoteIndex, c)
|
||||
f.connectionManager.Out(via)
|
||||
@@ -361,7 +377,14 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
|
||||
// NOTE: for goboring AESGCMTLS we need to lock because of the nonce check
|
||||
ci.writeLock.Lock()
|
||||
}
|
||||
c := ci.messageCounter.Add(1)
|
||||
c, ok := ci.NextMessageCounter()
|
||||
if !ok {
|
||||
if noiseutil.EncryptLockNeeded {
|
||||
ci.writeLock.Unlock()
|
||||
}
|
||||
f.dropExhausted(hostinfo, c, "Dropping outbound packets, tunnel message counter is exhausted")
|
||||
return
|
||||
}
|
||||
|
||||
//l.WithField("trace", string(debug.Stack())).Error("out Header ", &Header{Version, t, st, 0, hostinfo.remoteIndexId, c}, p)
|
||||
out = header.Encode(out, header.Version, t, st, hostinfo.remoteIndexId, c)
|
||||
@@ -408,7 +431,7 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
|
||||
if err != nil {
|
||||
hostinfo.logger(f.l).Error("Failed to write outgoing packet",
|
||||
"error", err,
|
||||
"udpAddr", remote,
|
||||
"udpAddr", hr,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
|
||||
+263
@@ -0,0 +1,263 @@
|
||||
package nebula
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"net/netip"
|
||||
"testing"
|
||||
|
||||
"github.com/gaissmai/bart"
|
||||
"github.com/slackhq/nebula/firewall"
|
||||
"github.com/slackhq/nebula/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
ipv4HeaderLen = 20
|
||||
ipv6HeaderLen = 40
|
||||
)
|
||||
|
||||
// capturingTun is a tio.Queue that records what is written to it. A queue that
|
||||
// discards writes is indistinguishable from a packet that was never forwarded.
|
||||
type capturingTun struct {
|
||||
writes [][]byte
|
||||
}
|
||||
|
||||
func (c *capturingTun) Read([]byte) (int, error) { return 0, io.EOF }
|
||||
func (c *capturingTun) Close() error { return nil }
|
||||
|
||||
func (c *capturingTun) Write(b []byte) (int, error) {
|
||||
c.writes = append(c.writes, append([]byte(nil), b...))
|
||||
return len(b), nil
|
||||
}
|
||||
|
||||
func newSelfForwardInterface(myAddrs ...netip.Addr) (*Interface, *capturingTun) {
|
||||
vpnAddrs := &bart.Lite{}
|
||||
for _, a := range myAddrs {
|
||||
vpnAddrs.Insert(netip.PrefixFrom(a, a.BitLen()))
|
||||
}
|
||||
|
||||
tun := &capturingTun{}
|
||||
return &Interface{
|
||||
l: test.NewLogger(),
|
||||
myVpnAddrsTable: vpnAddrs,
|
||||
myBroadcastAddrsTable: &bart.Lite{},
|
||||
readers: []io.ReadWriteCloser{tun},
|
||||
}, tun
|
||||
}
|
||||
|
||||
func consumeInside(f *Interface, packet []byte) {
|
||||
f.consumeInsidePacket(packet, &firewall.Packet{}, make([]byte, 12), make([]byte, mtu), 0, nil)
|
||||
}
|
||||
|
||||
// l4Proto describes one upper-layer header for these tests: its IP next-header
|
||||
// value, where its checksum field sits within the header, and how to build a
|
||||
// minimal instance of it.
|
||||
type l4Proto struct {
|
||||
name string
|
||||
nextHdr uint8
|
||||
cksumAt int
|
||||
build func() []byte
|
||||
}
|
||||
|
||||
var (
|
||||
tcpSyn = l4Proto{"tcp", firewall.ProtoTCP, 16, func() []byte {
|
||||
h := make([]byte, 20)
|
||||
binary.BigEndian.PutUint16(h[0:2], 49152)
|
||||
binary.BigEndian.PutUint16(h[2:4], 443)
|
||||
binary.BigEndian.PutUint32(h[4:8], 0x11223344) // sequence
|
||||
h[12] = 5 << 4 // data offset, no options
|
||||
h[13] = 0x02 // SYN
|
||||
binary.BigEndian.PutUint16(h[14:16], 65535) // window
|
||||
return h
|
||||
}}
|
||||
|
||||
udpDatagram = l4Proto{"udp", firewall.ProtoUDP, 6, func() []byte {
|
||||
h := make([]byte, 8+4)
|
||||
binary.BigEndian.PutUint16(h[0:2], 49152)
|
||||
binary.BigEndian.PutUint16(h[2:4], 53)
|
||||
binary.BigEndian.PutUint16(h[4:6], uint16(len(h)))
|
||||
copy(h[8:], "ping")
|
||||
return h
|
||||
}}
|
||||
|
||||
icmpEcho = l4Proto{"icmp", firewall.ProtoICMP, 2, func() []byte { return echoRequest(8) }}
|
||||
icmpv6Echo = l4Proto{"icmpv6", firewall.ProtoICMPv6, 2, func() []byte { return echoRequest(128) }}
|
||||
)
|
||||
|
||||
// echoRequest builds an echo request body. The type differs between ICMP and
|
||||
// ICMPv6, the rest of the header does not.
|
||||
func echoRequest(typ uint8) []byte {
|
||||
h := make([]byte, 8)
|
||||
h[0] = typ
|
||||
binary.BigEndian.PutUint16(h[4:6], 0xbeef) // identifier
|
||||
binary.BigEndian.PutUint16(h[6:8], 1) // sequence
|
||||
return h
|
||||
}
|
||||
|
||||
func buildIPv6(src, dst netip.Addr, p l4Proto) []byte {
|
||||
l4 := p.build()
|
||||
pkt := make([]byte, ipv6HeaderLen+len(l4))
|
||||
pkt[0] = 0x60
|
||||
binary.BigEndian.PutUint16(pkt[4:6], uint16(len(l4)))
|
||||
pkt[6] = p.nextHdr
|
||||
pkt[7] = 64
|
||||
copy(pkt[8:24], src.AsSlice())
|
||||
copy(pkt[24:40], dst.AsSlice())
|
||||
copy(pkt[ipv6HeaderLen:], l4)
|
||||
if l4 := pkt[ipv6HeaderLen:]; p.nextHdr == firewall.ProtoTCP || p.nextHdr == firewall.ProtoUDP {
|
||||
sum := ipv6PseudoheaderSum(src, dst, uint32(p.nextHdr), uint32(len(l4)))
|
||||
binary.BigEndian.PutUint16(l4[p.cksumAt:], ^fold(sumBytes(l4, sum)))
|
||||
}
|
||||
return pkt
|
||||
}
|
||||
|
||||
func buildIPv4(src, dst netip.Addr, p l4Proto) []byte {
|
||||
l4 := p.build()
|
||||
pkt := make([]byte, ipv4HeaderLen+len(l4))
|
||||
pkt[0] = 0x45
|
||||
binary.BigEndian.PutUint16(pkt[2:4], uint16(len(pkt)))
|
||||
pkt[8] = 64
|
||||
pkt[9] = p.nextHdr
|
||||
copy(pkt[12:16], src.AsSlice())
|
||||
copy(pkt[16:20], dst.AsSlice())
|
||||
copy(pkt[ipv4HeaderLen:], l4)
|
||||
if l4 := pkt[ipv4HeaderLen:]; p.nextHdr == firewall.ProtoTCP || p.nextHdr == firewall.ProtoUDP {
|
||||
sum := sumBytes(pkt[12:20], uint32(p.nextHdr)+uint32(len(l4)))
|
||||
binary.BigEndian.PutUint16(l4[p.cksumAt:], ^fold(sumBytes(l4, sum)))
|
||||
}
|
||||
return pkt
|
||||
}
|
||||
|
||||
// ipv6PseudoheaderSum is the RFC 2460 section 8.1 pseudo-header sum: source,
|
||||
// destination, a 32 bit upper-layer packet length and a 32 bit zero-padded next
|
||||
// header. Kept local to the test so these assertions do not check nebula's
|
||||
// checksum code against itself.
|
||||
func ipv6PseudoheaderSum(src, dst netip.Addr, nextHeader, length uint32) uint32 {
|
||||
var csum uint32
|
||||
s, d := src.AsSlice(), dst.AsSlice()
|
||||
for i := 0; i < 16; i += 2 {
|
||||
csum += uint32(s[i])<<8 | uint32(s[i+1])
|
||||
csum += uint32(d[i])<<8 | uint32(d[i+1])
|
||||
}
|
||||
return csum + length + nextHeader
|
||||
}
|
||||
|
||||
func sumBytes(b []byte, csum uint32) uint32 {
|
||||
for i := 0; i+1 < len(b); i += 2 {
|
||||
csum += uint32(b[i])<<8 | uint32(b[i+1])
|
||||
}
|
||||
if len(b)%2 == 1 {
|
||||
csum += uint32(b[len(b)-1]) << 8
|
||||
}
|
||||
return csum
|
||||
}
|
||||
|
||||
func fold(csum uint32) uint16 {
|
||||
for csum > 0xffff {
|
||||
csum = (csum >> 16) + (csum & 0xffff)
|
||||
}
|
||||
return uint16(csum)
|
||||
}
|
||||
|
||||
// l4ChecksumValid6 verifies an IPv6 upper-layer checksum the way a receiver
|
||||
// does: the pseudo-header plus the whole upper-layer segment, checksum field
|
||||
// included, folds to 0xffff. The next header field is the upper-layer protocol
|
||||
// only while there are no extension headers, which is all this file builds.
|
||||
func l4ChecksumValid6(pkt []byte) bool {
|
||||
src, _ := netip.AddrFromSlice(pkt[8:24])
|
||||
dst, _ := netip.AddrFromSlice(pkt[24:40])
|
||||
l4 := pkt[ipv6HeaderLen:]
|
||||
return fold(sumBytes(l4, ipv6PseudoheaderSum(src, dst, uint32(pkt[6]), uint32(len(l4))))) == 0xffff
|
||||
}
|
||||
|
||||
// l4ChecksumValid4 is the IPv4 counterpart: the RFC 793/768 pseudo-header is
|
||||
// source, destination, a zero byte, the protocol and the upper-layer length.
|
||||
func l4ChecksumValid4(pkt []byte) bool {
|
||||
ihl := int(pkt[0]&0x0f) << 2
|
||||
l4 := pkt[ihl:]
|
||||
return fold(sumBytes(l4, sumBytes(pkt[12:20], uint32(pkt[9])+uint32(len(l4))))) == 0xffff
|
||||
}
|
||||
|
||||
// TestConsumeInsidePacketSelfTraffic covers the self-addressed branch of
|
||||
// consumeInsidePacket, taken where immediatelyForwardToSelf is set (see
|
||||
// inside_bsd.go): the packet goes straight back to the tun, ahead of the
|
||||
// firewall and the handshake.
|
||||
func TestConsumeInsidePacketSelfTraffic(t *testing.T) {
|
||||
v4 := netip.MustParseAddr("100.100.1.42")
|
||||
v6 := netip.MustParseAddr("fd00::42")
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
addr netip.Addr
|
||||
pkt []byte
|
||||
}{
|
||||
{"ipv4/tcp", v4, buildIPv4(v4, v4, tcpSyn)},
|
||||
{"ipv4/udp", v4, buildIPv4(v4, v4, udpDatagram)},
|
||||
{"ipv4/icmp", v4, buildIPv4(v4, v4, icmpEcho)},
|
||||
{"ipv6/tcp", v6, buildIPv6(v6, v6, tcpSyn)},
|
||||
{"ipv6/udp", v6, buildIPv6(v6, v6, udpDatagram)},
|
||||
{"ipv6/icmpv6", v6, buildIPv6(v6, v6, icmpv6Echo)},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
f, tun := newSelfForwardInterface(tt.addr)
|
||||
// consumeInsidePacket writes through the slice it is handed, so a
|
||||
// packet that arrived with a valid checksum must come back out of
|
||||
// bytes taken before the call, unchanged.
|
||||
want := append([]byte(nil), tt.pkt...)
|
||||
consumeInside(f, tt.pkt)
|
||||
|
||||
if immediatelyForwardToSelf {
|
||||
require.Len(t, tun.writes, 1)
|
||||
assert.Equal(t, want, tun.writes[0])
|
||||
} else {
|
||||
assert.Empty(t, tun.writes, "self traffic reaches the tun over loopback here and must be dropped")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestConsumeInsidePacketSelfTrafficChecksum shows that the self-forward
|
||||
// returns the bytes it was handed, so a packet that arrived with a wrong
|
||||
// upper-layer checksum is written back with that same wrong checksum and the
|
||||
// kernel drops it on re-entry.
|
||||
//
|
||||
// This is how a macOS host loses TCP and UDP to its own IPv6 overlay address:
|
||||
// the kernel writes only the pseudo-header sum into the checksum field and
|
||||
// defers completion to hardware offload, state that does not survive the
|
||||
// crossing into userspace. Which kernels do this, for which protocols and IP
|
||||
// versions, is a property of the kernel and belongs to a test against a live
|
||||
// one; here the checksum is simply wrong, and the forward must make it right.
|
||||
func TestConsumeInsidePacketSelfTrafficChecksum(t *testing.T) {
|
||||
if !immediatelyForwardToSelf {
|
||||
t.Skip("self traffic never reaches the tun on this platform")
|
||||
}
|
||||
versions := []struct {
|
||||
name string
|
||||
addr netip.Addr
|
||||
build func(src, dst netip.Addr, p l4Proto) []byte
|
||||
l4At int
|
||||
valid func(pkt []byte) bool
|
||||
}{
|
||||
{"v4", netip.MustParseAddr("100.100.1.42"), buildIPv4, ipv4HeaderLen, l4ChecksumValid4},
|
||||
{"v6", netip.MustParseAddr("fd00::42"), buildIPv6, ipv6HeaderLen, l4ChecksumValid6},
|
||||
}
|
||||
for _, v := range versions {
|
||||
for _, p := range []l4Proto{tcpSyn, udpDatagram} {
|
||||
t.Run(v.name+"/"+p.name, func(t *testing.T) {
|
||||
pkt := v.build(v.addr, v.addr, p)
|
||||
binary.BigEndian.PutUint16(pkt[v.l4At+p.cksumAt:], 0x1234)
|
||||
require.False(t, v.valid(pkt), "the packet under test must start with a wrong checksum")
|
||||
f, tun := newSelfForwardInterface(v.addr)
|
||||
consumeInside(f, pkt)
|
||||
require.Len(t, tun.writes, 1)
|
||||
assert.True(t, v.valid(tun.writes[0]),
|
||||
"a forwarded %s packet must carry a valid checksum, got 0x%04x",
|
||||
p.name, binary.BigEndian.Uint16(tun.writes[0][v.l4At+p.cksumAt:]))
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,147 @@
|
||||
package iputil
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
|
||||
"github.com/google/gopacket/layers"
|
||||
"golang.org/x/net/ipv4"
|
||||
"golang.org/x/net/ipv6"
|
||||
"gvisor.dev/gvisor/pkg/tcpip/checksum"
|
||||
)
|
||||
|
||||
const udpHeaderLen = 8
|
||||
|
||||
// SetTransportChecksum recomputes the TCP or UDP checksum of an IPv4 or IPv6
|
||||
// packet in place.
|
||||
//
|
||||
// A kernel that offloads checksums to the NIC hands a packet to a tun with the
|
||||
// transport checksum unfinished: only the pseudo-header sum is in the field and
|
||||
// the rest is left for hardware that a tun does not have. A packet written
|
||||
// straight back to that tun is dropped on re-entry unless the checksum is
|
||||
// completed first. ICMP is left alone; it arrived complete on the kernels this
|
||||
// was measured against.
|
||||
//
|
||||
// So is any packet whose transport header cannot be located: fragments, unknown
|
||||
// extension headers and truncated packets. An IPv6 fragment header is declined
|
||||
// even when it carries the whole datagram (RFC 6946 atomic fragment), because
|
||||
// the walk reports only that a fragment header was present.
|
||||
func SetTransportChecksum(packet []byte) {
|
||||
if len(packet) < 1 {
|
||||
return
|
||||
}
|
||||
switch int(packet[0] >> 4) {
|
||||
case ipv4.Version:
|
||||
setTransportChecksum4(packet)
|
||||
case ipv6.Version:
|
||||
setTransportChecksum6(packet)
|
||||
}
|
||||
}
|
||||
|
||||
func setTransportChecksum4(packet []byte) {
|
||||
if len(packet) < ipv4.HeaderLen {
|
||||
return
|
||||
}
|
||||
ihl := int(packet[0]&0x0f) << 2
|
||||
end := int(binary.BigEndian.Uint16(packet[2:4]))
|
||||
if ihl < ipv4.HeaderLen || end < ihl || end > len(packet) {
|
||||
return
|
||||
}
|
||||
// The checksum covers the whole datagram, which a fragment (MF set or a
|
||||
// non-zero offset) does not carry.
|
||||
if binary.BigEndian.Uint16(packet[6:8])&0x3fff != 0 {
|
||||
return
|
||||
}
|
||||
|
||||
transport, ok := transportExtent(packet[ihl:end], packet[9])
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
csum := ipv4PseudoheaderChecksum(packet[12:16], packet[16:20], uint32(packet[9]), uint32(len(transport)))
|
||||
writeTransportChecksum(transport, layers.IPProtocol(packet[9]), csum)
|
||||
}
|
||||
|
||||
func setTransportChecksum6(packet []byte) {
|
||||
if len(packet) < ipv6.HeaderLen {
|
||||
return
|
||||
}
|
||||
end := ipv6.HeaderLen + int(binary.BigEndian.Uint16(packet[4:6]))
|
||||
if end > len(packet) {
|
||||
return
|
||||
}
|
||||
|
||||
// The checksum covers the whole datagram, which a fragment does not carry.
|
||||
// An unknown extension header hides where the transport header starts. A
|
||||
// chain longer than the walk's budget ends it early, at an offset that was
|
||||
// never checked against the packet.
|
||||
proto, offset, _, anyFragment, err := IPv6FindUpperProtocol(packet[:end])
|
||||
if err != nil || anyFragment || offset >= end {
|
||||
return
|
||||
}
|
||||
|
||||
transport, ok := transportExtent(packet[offset:end], proto)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
csum := ipv6PseudoheaderChecksum(packet[8:24], packet[24:40], uint32(proto), uint32(len(transport)))
|
||||
writeTransportChecksum(transport, layers.IPProtocol(proto), csum)
|
||||
}
|
||||
|
||||
// transportExtent narrows a segment to the length its own header declares. UDP
|
||||
// carries a Length field, and RFC 768 and RFC 8200 section 8.1 both make that
|
||||
// field, not the IP payload extent, the length the pseudo-header counts and the
|
||||
// checksum covers; a datagram padded out to a link's minimum frame is the usual
|
||||
// way the two differ. TCP has no such field, so its segment runs to the end of
|
||||
// the IP payload. A Length that overruns the bytes IP delivered describes a
|
||||
// datagram that is not there.
|
||||
func transportExtent(transport []byte, proto uint8) ([]byte, bool) {
|
||||
if layers.IPProtocol(proto) != layers.IPProtocolUDP {
|
||||
return transport, true
|
||||
}
|
||||
if len(transport) < udpHeaderLen {
|
||||
return nil, false
|
||||
}
|
||||
ulen := int(binary.BigEndian.Uint16(transport[4:6]))
|
||||
if ulen < udpHeaderLen || ulen > len(transport) {
|
||||
return nil, false
|
||||
}
|
||||
return transport[:ulen], true
|
||||
}
|
||||
|
||||
// writeTransportChecksum stores the checksum of transport, taken over the
|
||||
// pseudo-header sum csum, in the header's checksum field. A UDP checksum that
|
||||
// computes to zero goes on the wire as 0xffff: zero means no checksum was
|
||||
// computed (RFC 768), and over IPv6 the checksum is mandatory (RFC 8200
|
||||
// section 8.1).
|
||||
func writeTransportChecksum(transport []byte, proto layers.IPProtocol, csum uint32) {
|
||||
var at, minLen int
|
||||
switch proto {
|
||||
case layers.IPProtocolTCP:
|
||||
at, minLen = 16, 20
|
||||
case layers.IPProtocolUDP:
|
||||
at, minLen = 6, udpHeaderLen
|
||||
default:
|
||||
return
|
||||
}
|
||||
if len(transport) < minLen {
|
||||
return
|
||||
}
|
||||
|
||||
transport[at], transport[at+1] = 0, 0
|
||||
sum := ^checksum.Checksum(transport, fold(csum))
|
||||
if sum == 0 && proto == layers.IPProtocolUDP {
|
||||
sum = 0xffff
|
||||
}
|
||||
binary.BigEndian.PutUint16(transport[at:], sum)
|
||||
}
|
||||
|
||||
// fold reduces a pseudo-header sum to the 16 bit seed Checksum takes. Carrying
|
||||
// the high half back into the low half is what keeps the reduction lossless, so
|
||||
// the seed sums exactly as the wider value would; 0xffff is its fixed point.
|
||||
// Every term of that sum comes from a 16 bit field, so it stays far below the
|
||||
// width at which the accumulator would wrap.
|
||||
func fold(csum uint32) uint16 {
|
||||
for csum > 0xffff {
|
||||
csum = (csum >> 16) + (csum & 0xffff)
|
||||
}
|
||||
return uint16(csum)
|
||||
}
|
||||
@@ -0,0 +1,242 @@
|
||||
package iputil
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"net"
|
||||
"testing"
|
||||
|
||||
"github.com/google/gopacket"
|
||||
"github.com/google/gopacket/layers"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/net/ipv6"
|
||||
)
|
||||
|
||||
// serialize builds a packet with gopacket, whose checksums are computed
|
||||
// independently of this package.
|
||||
func serialize(t *testing.T, ls ...gopacket.SerializableLayer) []byte {
|
||||
buf := gopacket.NewSerializeBuffer()
|
||||
require.NoError(t, gopacket.SerializeLayers(buf, gopacket.SerializeOptions{FixLengths: true, ComputeChecksums: true}, ls...))
|
||||
return append([]byte(nil), buf.Bytes()...)
|
||||
}
|
||||
|
||||
// withExtensionHeader inserts an 8 byte IPv6 extension header of the given
|
||||
// type between the IPv6 header and its payload. The transport checksum does not
|
||||
// change: the pseudo-header counts only upper-layer bytes.
|
||||
func withExtensionHeader(pkt []byte, typ layers.IPProtocol, hdr [8]byte) []byte {
|
||||
hdr[0] = pkt[6]
|
||||
out := make([]byte, 0, len(pkt)+8)
|
||||
out = append(out, pkt[:40]...)
|
||||
out = append(out, hdr[:]...)
|
||||
out = append(out, pkt[40:]...)
|
||||
out[6] = byte(typ)
|
||||
binary.BigEndian.PutUint16(out[4:6], binary.BigEndian.Uint16(pkt[4:6])+8)
|
||||
return out
|
||||
}
|
||||
|
||||
// truncate copies the first n bytes into a buffer of exactly that capacity, so
|
||||
// a read past the length panics instead of quietly succeeding.
|
||||
func truncate(pkt []byte, n int) []byte {
|
||||
out := make([]byte, n)
|
||||
copy(out, pkt)
|
||||
return out
|
||||
}
|
||||
|
||||
// extChain builds an IPv6 packet fronted by n Destination Options headers. Each
|
||||
// points at another one, so the walk spends its whole budget without reaching a
|
||||
// transport header. lastExtLen inflates the final header's declared length,
|
||||
// which is how the walk ends up past the end of the packet.
|
||||
func extChain(n int, lastExtLen byte) []byte {
|
||||
pkt := make([]byte, ipv6.HeaderLen)
|
||||
pkt[0], pkt[6], pkt[7] = 0x60, 60, 64
|
||||
for i := range n {
|
||||
h := make([]byte, 8)
|
||||
h[0] = 60
|
||||
if i == n-1 {
|
||||
h[1] = lastExtLen
|
||||
}
|
||||
pkt = append(pkt, h...)
|
||||
}
|
||||
pkt = append(pkt, make([]byte, 20)...)
|
||||
binary.BigEndian.PutUint16(pkt[4:6], uint16(len(pkt)-ipv6.HeaderLen))
|
||||
return pkt
|
||||
}
|
||||
|
||||
func TestSetTransportChecksum(t *testing.T) {
|
||||
// Source and destination differ so that a pseudo-header built from the wrong
|
||||
// one, or from the two swapped, does not land on the same checksum anyway.
|
||||
v4 := func(proto layers.IPProtocol) *layers.IPv4 {
|
||||
return &layers.IPv4{Version: 4, TTL: 64, Id: 0x1234, Protocol: proto, SrcIP: net.IPv4(192, 0, 2, 1).To4(), DstIP: net.IPv4(198, 51, 100, 2).To4()}
|
||||
}
|
||||
v6 := func(proto layers.IPProtocol) *layers.IPv6 {
|
||||
return &layers.IPv6{Version: 6, HopLimit: 64, NextHeader: proto, SrcIP: net.ParseIP("2001:db8::1"), DstIP: net.ParseIP("2001:db8:1::2")}
|
||||
}
|
||||
tcp := func(ip gopacket.NetworkLayer) *layers.TCP {
|
||||
l := &layers.TCP{SrcPort: 49152, DstPort: 443, SYN: true, Window: 65535}
|
||||
require.NoError(t, l.SetNetworkLayerForChecksum(ip))
|
||||
return l
|
||||
}
|
||||
udp := func(ip gopacket.NetworkLayer) *layers.UDP {
|
||||
l := &layers.UDP{SrcPort: 49152, DstPort: 53}
|
||||
require.NoError(t, l.SetNetworkLayerForChecksum(ip))
|
||||
return l
|
||||
}
|
||||
payload := gopacket.Payload("self")
|
||||
nop := layers.IPv4Option{OptionType: 1, OptionLength: 1}
|
||||
|
||||
ip4tcp := v4(layers.IPProtocolTCP)
|
||||
ip4opts := v4(layers.IPProtocolTCP)
|
||||
ip4opts.Options = []layers.IPv4Option{nop, nop, nop, nop}
|
||||
ip4udp := v4(layers.IPProtocolUDP)
|
||||
ip6tcp := v6(layers.IPProtocolTCP)
|
||||
ip6udp := v6(layers.IPProtocolUDP)
|
||||
hopByHop := [8]byte{0, 0, 1, 4} // next header, length 0, PadN of 4
|
||||
|
||||
// Bytes past the length the IP header declares are not part of the
|
||||
// datagram and must not be summed.
|
||||
trailing4 := append(serialize(t, ip4tcp, tcp(ip4tcp), payload), []byte("trailing")...)
|
||||
trailing6 := append(serialize(t, ip6tcp, tcp(ip6tcp), payload), []byte("trailing")...)
|
||||
|
||||
// A datagram padded out past the length UDP declares: the pseudo-header
|
||||
// counts the UDP Length field, so the checksum is the unpadded one.
|
||||
padded4 := append(serialize(t, ip4udp, udp(ip4udp), payload), []byte("pad!")...)
|
||||
binary.BigEndian.PutUint16(padded4[2:4], uint16(len(padded4)))
|
||||
padded6 := append(serialize(t, ip6udp, udp(ip6udp), payload), []byte("pad!")...)
|
||||
binary.BigEndian.PutUint16(padded6[4:6], uint16(len(padded6)-ipv6.HeaderLen))
|
||||
|
||||
// Corrupting the checksum and asking for it back must yield gopacket's
|
||||
// packet, byte for byte.
|
||||
recomputed := []struct {
|
||||
name string
|
||||
pkt []byte
|
||||
cksum int
|
||||
}{
|
||||
{"v4 tcp", serialize(t, ip4tcp, tcp(ip4tcp), payload), 20 + 16},
|
||||
{"v4 tcp with ip options", serialize(t, ip4opts, tcp(ip4opts), payload), 24 + 16},
|
||||
{"v4 udp", serialize(t, ip4udp, udp(ip4udp), payload), 20 + 6},
|
||||
{"v4 tcp header only", serialize(t, ip4tcp, tcp(ip4tcp)), 20 + 16},
|
||||
{"v4 udp header only", serialize(t, ip4udp, udp(ip4udp)), 20 + 6},
|
||||
{"v6 tcp", serialize(t, ip6tcp, tcp(ip6tcp), payload), 40 + 16},
|
||||
{"v6 udp", serialize(t, ip6udp, udp(ip6udp), payload), 40 + 6},
|
||||
{"v6 udp header only", serialize(t, ip6udp, udp(ip6udp)), 40 + 6},
|
||||
{"v6 tcp behind hop-by-hop", withExtensionHeader(serialize(t, ip6tcp, tcp(ip6tcp), payload), layers.IPProtocolIPv6HopByHop, hopByHop), 48 + 16},
|
||||
{"v4 tcp with bytes past the total length", trailing4, 20 + 16},
|
||||
{"v6 tcp with bytes past the payload length", trailing6, 40 + 16},
|
||||
{"v4 udp padded past its declared length", padded4, 20 + 6},
|
||||
{"v6 udp padded past its declared length", padded6, 40 + 6},
|
||||
}
|
||||
for _, tt := range recomputed {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := append([]byte(nil), tt.pkt...)
|
||||
binary.BigEndian.PutUint16(got[tt.cksum:], 0x1234)
|
||||
require.NotEqual(t, tt.pkt, got)
|
||||
SetTransportChecksum(got)
|
||||
assert.Equal(t, tt.pkt, got)
|
||||
})
|
||||
}
|
||||
|
||||
ip4frag := v4(layers.IPProtocolTCP)
|
||||
ip4frag.Flags = layers.IPv4MoreFragments
|
||||
ip4later := v4(layers.IPProtocolTCP)
|
||||
ip4later.FragOffset = 1
|
||||
ip4icmp := v4(layers.IPProtocolICMPv4)
|
||||
|
||||
badIHL := serialize(t, ip4tcp, tcp(ip4tcp), payload)
|
||||
badIHL[0] = 0x44 // header length 16, shorter than an ipv4 header
|
||||
shortTotalLen := serialize(t, ip4tcp, tcp(ip4tcp), payload)
|
||||
binary.BigEndian.PutUint16(shortTotalLen[2:4], 10) // shorter than the header it introduces
|
||||
cutTCP := serialize(t, ip4tcp, tcp(ip4tcp), payload)
|
||||
binary.BigEndian.PutUint16(cutTCP[2:4], 20+19) // one byte short of a tcp header
|
||||
cutTCP = truncate(cutTCP, 20+19)
|
||||
cutUDP := serialize(t, ip4udp, udp(ip4udp), payload)
|
||||
binary.BigEndian.PutUint16(cutUDP[2:4], 20+7) // one byte short of a udp header
|
||||
cutUDP = truncate(cutUDP, 20+7)
|
||||
// Two bytes short, so a transport header survives whole and the minimum
|
||||
// length check cannot stand in for the bounds check.
|
||||
cutV6 := truncate(serialize(t, ip6tcp, tcp(ip6tcp), payload), 62)
|
||||
fragment := [8]byte{0, 0, 0, 1, 0, 0, 0, 1} // next header, reserved, offset 0 with M set, id
|
||||
overrun4 := serialize(t, ip4udp, udp(ip4udp), payload)
|
||||
binary.BigEndian.PutUint16(overrun4[24:26], uint16(len(overrun4)-20+1)) // one byte past what ip delivered
|
||||
overrun6 := serialize(t, ip6udp, udp(ip6udp), payload)
|
||||
binary.BigEndian.PutUint16(overrun6[44:46], uint16(len(overrun6)-ipv6.HeaderLen+1))
|
||||
shortUDPLen := serialize(t, ip4udp, udp(ip4udp), payload)
|
||||
binary.BigEndian.PutUint16(shortUDPLen[24:26], 7) // shorter than the header it counts
|
||||
|
||||
// Where the checksum cannot be completed the packet is left as it came.
|
||||
untouched := []struct {
|
||||
name string
|
||||
pkt []byte
|
||||
cksum int
|
||||
}{
|
||||
{"v4 first fragment", serialize(t, ip4frag, tcp(ip4frag), payload), 20 + 16},
|
||||
{"v4 later fragment", serialize(t, ip4later, tcp(ip4later), payload), 20 + 16},
|
||||
{"v4 icmp", serialize(t, ip4icmp, &layers.ICMPv4{TypeCode: layers.CreateICMPv4TypeCode(8, 0), Id: 1, Seq: 1}, payload), 20 + 2},
|
||||
{"v4 header length below the minimum", badIHL, 20 + 16},
|
||||
{"v4 total length below the header length", shortTotalLen, 20 + 16},
|
||||
{"v4 truncated below its total length", truncate(serialize(t, ip4tcp, tcp(ip4tcp), payload), 30), -1},
|
||||
{"v4 tcp header cut short", cutTCP, 20 + 16},
|
||||
{"v4 udp header cut short", cutUDP, -1},
|
||||
{"v6 fragment", withExtensionHeader(serialize(t, ip6tcp, tcp(ip6tcp), payload), layers.IPProtocolIPv6Fragment, fragment), 48 + 16},
|
||||
{"v6 truncated below its payload length", truncate(serialize(t, ip6tcp, tcp(ip6tcp), payload), 50), -1},
|
||||
{"v6 truncated with a whole transport header still present", cutV6, 40 + 16},
|
||||
{"v6 extension header chain longer than the walk", extChain(9, 0), 112 + 16},
|
||||
{"v6 extension header chain running past the packet", extChain(8, 255), 104 + 16},
|
||||
{"v4 udp length past the end of the datagram", overrun4, 20 + 6},
|
||||
{"v6 udp length past the end of the datagram", overrun6, 40 + 6},
|
||||
{"v4 udp length below a udp header", shortUDPLen, 20 + 6},
|
||||
}
|
||||
for _, tt := range untouched {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.cksum >= 0 {
|
||||
binary.BigEndian.PutUint16(tt.pkt[tt.cksum:], 0x1234)
|
||||
}
|
||||
want := append([]byte(nil), tt.pkt...)
|
||||
SetTransportChecksum(tt.pkt)
|
||||
assert.Equal(t, want, tt.pkt)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("too short to carry a header", func(t *testing.T) {
|
||||
for _, pkt := range [][]byte{nil, {}, {0x45}, {0x60}} {
|
||||
assert.NotPanics(t, func() { SetTransportChecksum(pkt) })
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("tcp checksum of zero goes out as zero", func(t *testing.T) {
|
||||
pkt := serialize(t, ip4tcp, tcp(ip4tcp), gopacket.Payload{0, 0})
|
||||
c := binary.BigEndian.Uint16(pkt[36:38])
|
||||
require.NotZero(t, c)
|
||||
// Only udp reserves zero to mean "not computed", so tcp keeps it.
|
||||
binary.BigEndian.PutUint16(pkt[40:42], c)
|
||||
SetTransportChecksum(pkt)
|
||||
assert.Zero(t, binary.BigEndian.Uint16(pkt[36:38]))
|
||||
})
|
||||
|
||||
t.Run("udp checksum of zero goes out as 0xffff", func(t *testing.T) {
|
||||
pkt := serialize(t, ip4udp, udp(ip4udp), gopacket.Payload{0, 0})
|
||||
c := binary.BigEndian.Uint16(pkt[26:28])
|
||||
require.NotZero(t, c)
|
||||
// The one's complement sum is now 0xffff - c; adding c to the payload
|
||||
// makes it 0xffff, whose complement is zero.
|
||||
binary.BigEndian.PutUint16(pkt[28:30], c)
|
||||
SetTransportChecksum(pkt)
|
||||
assert.Equal(t, uint16(0xffff), binary.BigEndian.Uint16(pkt[26:28]))
|
||||
})
|
||||
}
|
||||
|
||||
func TestFold(t *testing.T) {
|
||||
// 0xffff is the fold's fixed point, so a loop bound one notch tight never
|
||||
// terminates on it.
|
||||
for _, tt := range []struct {
|
||||
in uint32
|
||||
want uint16
|
||||
}{
|
||||
{0, 0},
|
||||
{0xffff, 0xffff},
|
||||
{0x10000, 1},
|
||||
{0x1fffe, 0xffff},
|
||||
{0xffffffff, 0xffff},
|
||||
} {
|
||||
assert.Equal(t, tt.want, fold(tt.in))
|
||||
}
|
||||
}
|
||||
+31
-8
@@ -2,11 +2,16 @@ package iputil
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
|
||||
"golang.org/x/net/ipv4"
|
||||
"golang.org/x/net/ipv6"
|
||||
)
|
||||
|
||||
// ErrIPv6CouldNotFindPayload is returned when the ipv6 extension header chain is truncated before a terminal
|
||||
// upper layer protocol is reached.
|
||||
var ErrIPv6CouldNotFindPayload = errors.New("could not find payload in ipv6 packet")
|
||||
|
||||
const (
|
||||
// MaxIPv4RejectPacketSize is the largest IPv4 reject packet:
|
||||
// - 20 byte ipv4 header
|
||||
@@ -199,8 +204,8 @@ func ipv4CreateRejectTCPPacket(packet []byte, out []byte) []byte {
|
||||
}
|
||||
|
||||
func ipv6CreateRejectPacket(packet []byte, out []byte) []byte {
|
||||
proto, offset, isFragment := ipv6FindUpperProtocol(packet)
|
||||
if isFragment {
|
||||
proto, offset, isFragment, _, err := IPv6FindUpperProtocol(packet)
|
||||
if err != nil || isFragment {
|
||||
return nil
|
||||
}
|
||||
switch proto {
|
||||
@@ -333,7 +338,18 @@ func ipv6CreateRejectTCPPacket(packet []byte, out []byte, offset int) []byte {
|
||||
return out
|
||||
}
|
||||
|
||||
func ipv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragment bool) {
|
||||
// IPv6FindUpperProtocol walks the ipv6 extension header chain and returns the upper layer protocol, the
|
||||
// offset it begins at, and whether the packet is a non-first fragment. Only the RFC 8200 and IANA extension
|
||||
// headers below are walked. Everything else, including Mobility (135), HIP (139), Shim6 (140), experimental
|
||||
// 253/254, and real upper layer protocols like SCTP or GRE, is terminal. Walking those as extension headers
|
||||
// is a firewall bypass, so they fail closed. For a non-first fragment the returned protocol is the fragmented
|
||||
// protocol and offset points at the fragment header, there is no transport header to locate. Returns
|
||||
// ErrIPv6CouldNotFindPayload if packet is smaller than an ipv6 header or the chain is truncated before a
|
||||
// terminal protocol is reached.
|
||||
func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragment, anyFragment bool, err error) {
|
||||
if len(packet) < ipv6.HeaderLen {
|
||||
return 0, 0, false, false, ErrIPv6CouldNotFindPayload
|
||||
}
|
||||
nextHeader = packet[6]
|
||||
offset = ipv6.HeaderLen
|
||||
|
||||
@@ -341,30 +357,37 @@ func ipv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragm
|
||||
switch nextHeader {
|
||||
case 0, 43, 60: // Hop-by-Hop, Routing, Destination
|
||||
if len(packet) < offset+2 {
|
||||
return nextHeader, offset, isFragment
|
||||
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload
|
||||
}
|
||||
nextHeader = packet[offset]
|
||||
offset += (int(packet[offset+1]) + 1) << 3
|
||||
|
||||
case 44: // Fragment
|
||||
if len(packet) < offset+8 {
|
||||
return nextHeader, offset, isFragment
|
||||
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload
|
||||
}
|
||||
anyFragment = true
|
||||
// Non-first fragments carry no transport header, report the fragmented protocol and stop
|
||||
if packet[offset+2] != 0 || packet[offset+3]&0xf8 != 0 {
|
||||
isFragment = true
|
||||
return packet[offset], offset, true, anyFragment, nil
|
||||
}
|
||||
nextHeader = packet[offset]
|
||||
offset += 8
|
||||
|
||||
case 51: // AH
|
||||
if len(packet) < offset+2 {
|
||||
return nextHeader, offset, isFragment
|
||||
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload
|
||||
}
|
||||
nextHeader = packet[offset]
|
||||
offset += (int(packet[offset+1]) + 2) << 2
|
||||
|
||||
default:
|
||||
return nextHeader, offset, isFragment
|
||||
// A prior extension header can declare a length that advances offset past the packet. The terminal
|
||||
// protocol's header isn't actually here, so treat the chain as truncated rather than classifying it.
|
||||
if offset > len(packet) {
|
||||
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload
|
||||
}
|
||||
return nextHeader, offset, isFragment, anyFragment, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/net/ipv4"
|
||||
"golang.org/x/net/ipv6"
|
||||
)
|
||||
@@ -474,3 +475,63 @@ func TestCreateICMPEchoResponse_IPv6_NotICMPv6(t *testing.T) {
|
||||
result := CreateICMPEchoResponse(packet, out)
|
||||
assert.Nil(t, result)
|
||||
}
|
||||
|
||||
func Test_IPv6FindUpperProtocol(t *testing.T) {
|
||||
src := net.ParseIP("fd00::1")
|
||||
dst := net.ParseIP("fd00::2")
|
||||
|
||||
// 8 byte extension/transport stand-ins, first byte is the next header, second is the length field
|
||||
extToTCP := []byte{6, 0, 0, 0, 0, 0, 0, 0} // len 0 -> 8 bytes, next = TCP
|
||||
extToUDP := []byte{17, 0, 0, 0, 0, 0, 0, 0} // len 0 -> 8 bytes, next = UDP
|
||||
extToRouting := []byte{43, 0, 0, 0, 0, 0, 0, 0} // len 0 -> 8 bytes, next = Routing
|
||||
ahToUDP := []byte{17, 0, 0, 0, 0, 0, 0, 0} // AH len 0 -> (0+2)<<2 = 8 bytes, next = UDP
|
||||
firstFragToUDP := []byte{17, 0, 0, 1, 0, 0, 0, 1} // frag offset 0, M=1, next = UDP
|
||||
nonFirstFrag := []byte{17, 0, 0, 9, 0, 0, 0, 1} // frag offset non-zero, next = UDP
|
||||
transport := []byte{0, 80, 1, 187, 0, 0, 0, 0} // stand-in bytes, IPv6FindUpperProtocol never reads ports
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
nextHeader uint8
|
||||
payload []byte
|
||||
wantProto uint8
|
||||
wantOffset int
|
||||
wantFragment bool
|
||||
wantAnyFrag bool
|
||||
wantErr error
|
||||
}{
|
||||
{"plain udp", 17, transport, 17, ipv6.HeaderLen, false, false, nil},
|
||||
{"hop-by-hop then tcp", 0, append(extToTCP, transport...), 6, ipv6.HeaderLen + 8, false, false, nil},
|
||||
{"routing then tcp", 43, append(extToTCP, transport...), 6, ipv6.HeaderLen + 8, false, false, nil},
|
||||
{"destination then udp", 60, append(extToUDP, transport...), 17, ipv6.HeaderLen + 8, false, false, nil},
|
||||
{"hop-by-hop, routing, then tcp", 0, append(append(extToRouting, extToTCP...), transport...), 6, ipv6.HeaderLen + 16, false, false, nil},
|
||||
{"ah then udp", 51, append(ahToUDP, transport...), 17, ipv6.HeaderLen + 8, false, false, nil},
|
||||
{"first fragment walks to transport", 44, append(firstFragToUDP, transport...), 17, ipv6.HeaderLen + 8, false, true, nil},
|
||||
{"non-first fragment stops", 44, append(nonFirstFrag, transport...), 17, ipv6.HeaderLen, true, true, nil},
|
||||
{"unknown protocol is terminal", 132, transport, 132, ipv6.HeaderLen, false, false, nil}, // SCTP
|
||||
{"truncated extension header", 0, nil, 0, ipv6.HeaderLen, false, false, ErrIPv6CouldNotFindPayload},
|
||||
// Destination Options with a declared length (255+1)*8 = 2048 that runs past the 48 byte buffer, next = SCTP
|
||||
{"extension length past buffer", 60, []byte{132, 255, 0, 0, 0, 0, 0, 0}, 132, ipv6.HeaderLen + 2048, false, false, ErrIPv6CouldNotFindPayload},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
packet := makeIPv6Packet(src, dst, tt.nextHeader, tt.payload)
|
||||
proto, offset, isFragment, anyFragment, err := IPv6FindUpperProtocol(packet)
|
||||
if tt.wantErr != nil {
|
||||
assert.ErrorIs(t, err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tt.wantProto, proto)
|
||||
assert.Equal(t, tt.wantOffset, offset)
|
||||
assert.Equal(t, tt.wantFragment, isFragment)
|
||||
assert.Equal(t, tt.wantAnyFrag, anyFragment)
|
||||
})
|
||||
}
|
||||
|
||||
// A packet smaller than an ipv6 header must error rather than panic reading byte 6
|
||||
t.Run("shorter than ipv6 header", func(t *testing.T) {
|
||||
_, _, _, _, err := IPv6FindUpperProtocol(make([]byte, 6))
|
||||
assert.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
|
||||
})
|
||||
}
|
||||
+36
-1
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/header"
|
||||
"github.com/slackhq/nebula/logging"
|
||||
"github.com/slackhq/nebula/nicfinder"
|
||||
"github.com/slackhq/nebula/udp"
|
||||
"github.com/slackhq/nebula/util"
|
||||
)
|
||||
@@ -36,6 +37,14 @@ type LightHouse struct {
|
||||
myVpnNetworksTable *bart.Lite
|
||||
punchy *Punchy
|
||||
|
||||
// localAddrsFn enumerates the underlay addresses we advertise. It is a field so tests can supply simulated
|
||||
// addresses rather than whatever this machine's NICs happen to be. Set it before Start.
|
||||
localAddrsFn func(*LocalAllowList) []netip.Addr
|
||||
|
||||
// lastLocalAddrsErr is the previous localAddrsFn failure. Enumeration runs on every update, so an
|
||||
// unchanged failure is demoted to Debug rather than warning every lighthouse.interval forever.
|
||||
lastLocalAddrsErr atomic.Pointer[string]
|
||||
|
||||
// Local cache of answers from light houses
|
||||
// map of vpn addr to answers
|
||||
addrMap map[netip.Addr]*RemoteList
|
||||
@@ -107,6 +116,12 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
|
||||
queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)),
|
||||
l: l,
|
||||
}
|
||||
h.localAddrsFn = func(al *LocalAllowList) []netip.Addr {
|
||||
addrs, err := nicfinder.Find(ctx, h.l, al)
|
||||
h.logLocalAddrsErr(err)
|
||||
return addrs
|
||||
}
|
||||
|
||||
lighthouses := make([]netip.Addr, 0)
|
||||
h.lighthouses.Store(&lighthouses)
|
||||
staticList := make(map[netip.Addr]struct{})
|
||||
@@ -905,6 +920,26 @@ func (lh *LightHouse) TriggerUpdate() {
|
||||
}
|
||||
}
|
||||
|
||||
// logLocalAddrsErr reports a localAddrs failure at Warn the first time it is seen and at Debug while
|
||||
// it persists unchanged, so a permanent failure does not warn on every update forever.
|
||||
func (lh *LightHouse) logLocalAddrsErr(err error) {
|
||||
if err == nil {
|
||||
lh.lastLocalAddrsErr.Store(nil)
|
||||
return
|
||||
}
|
||||
|
||||
msg := err.Error()
|
||||
prev := lh.lastLocalAddrsErr.Swap(&msg)
|
||||
if prev != nil && *prev == msg {
|
||||
if lh.l.Enabled(context.Background(), slog.LevelDebug) {
|
||||
lh.l.Debug("Failed to collect local addresses to advertise", "error", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
lh.l.Warn("Failed to collect local addresses to advertise", "error", err)
|
||||
}
|
||||
|
||||
func (lh *LightHouse) SendUpdate() {
|
||||
var v4 []*V4AddrPort
|
||||
var v6 []*V6AddrPort
|
||||
@@ -918,7 +953,7 @@ func (lh *LightHouse) SendUpdate() {
|
||||
}
|
||||
|
||||
lal := lh.GetLocalAllowList()
|
||||
for _, e := range localAddrs(lh.l, lal) {
|
||||
for _, e := range lh.localAddrsFn(lal) {
|
||||
if lh.myVpnNetworksTable.Contains(e) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package nebula
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/netip"
|
||||
"testing"
|
||||
@@ -738,3 +740,32 @@ func TestLighthouse_DeletesWork(t *testing.T) {
|
||||
out = lh.Query(testHost)
|
||||
assert.Nil(t, out)
|
||||
}
|
||||
|
||||
func TestLightHouse_logLocalAddrsErr(t *testing.T) {
|
||||
out := &bytes.Buffer{}
|
||||
lh := &LightHouse{l: test.NewLoggerWithOutput(out)}
|
||||
|
||||
// The first sighting of a failure warns.
|
||||
lh.logLocalAddrsErr(errors.New("permission denied"))
|
||||
assert.Contains(t, out.String(), "level=WARN")
|
||||
assert.Contains(t, out.String(), "permission denied")
|
||||
|
||||
// Repeating unchanged does not warn again, which is what keeps a permanent failure from warning
|
||||
// on every lighthouse.interval for the life of the process.
|
||||
out.Reset()
|
||||
lh.logLocalAddrsErr(errors.New("permission denied"))
|
||||
assert.NotContains(t, out.String(), "level=WARN")
|
||||
|
||||
// A different failure is a new event and warns.
|
||||
out.Reset()
|
||||
lh.logLocalAddrsErr(errors.New("something else"))
|
||||
assert.Contains(t, out.String(), "level=WARN")
|
||||
assert.Contains(t, out.String(), "something else")
|
||||
|
||||
// Recovering resets, so the same failure returning later warns again.
|
||||
out.Reset()
|
||||
lh.logLocalAddrsErr(nil)
|
||||
assert.Empty(t, out.String())
|
||||
lh.logLocalAddrsErr(errors.New("something else"))
|
||||
assert.Contains(t, out.String(), "level=WARN")
|
||||
}
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/noiseutil"
|
||||
"github.com/slackhq/nebula/overlay"
|
||||
"github.com/slackhq/nebula/sshd"
|
||||
"github.com/slackhq/nebula/udp"
|
||||
@@ -20,6 +21,12 @@ import (
|
||||
|
||||
type m = map[string]any
|
||||
|
||||
// maxRoutines caps routines below the RejectHeadroom nonce gap so concurrent senders can't race the counter past wrap.
|
||||
const maxRoutines = 1 << 16
|
||||
|
||||
// The reject headroom must exceed every sender that can be mid-reservation at once, about two per routine.
|
||||
const _ = noiseutil.RejectHeadroom - 4*maxRoutines
|
||||
|
||||
func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
// Automatically cancel the context if Main returns an error, to signal all created goroutines to quit.
|
||||
@@ -81,9 +88,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
|
||||
if routines < 1 {
|
||||
routines = 1
|
||||
}
|
||||
if routines > 1 {
|
||||
l.Info("Using multiple routines", "routines", routines)
|
||||
}
|
||||
} else {
|
||||
// deprecated and undocumented
|
||||
tunQueues := c.GetInt("tun.routines", 1)
|
||||
@@ -93,6 +97,12 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
|
||||
l.Warn("Setting tun.routines and listen.routines is deprecated. Use `routines` instead", "routines", routines)
|
||||
}
|
||||
}
|
||||
if routines > maxRoutines {
|
||||
l.Warn("Using multiple routines", "routines", maxRoutines, "clamped", true, "requestedRoutines", routines)
|
||||
routines = maxRoutines
|
||||
} else if routines > 1 {
|
||||
l.Info("Using multiple routines", "routines", routines)
|
||||
}
|
||||
|
||||
// EXPERIMENTAL
|
||||
// Intentionally not documented yet while we do more testing and determine
|
||||
@@ -268,6 +278,8 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
|
||||
|
||||
attachCommands(l, c, ssh, ifce)
|
||||
|
||||
networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c)
|
||||
|
||||
return &Control{
|
||||
state: StateReady,
|
||||
f: ifce,
|
||||
@@ -278,6 +290,7 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
|
||||
statsStart: stats.Start,
|
||||
dnsStart: ds.Start,
|
||||
lighthouseStart: lightHouse.StartUpdateWorker,
|
||||
networkChangeStart: networkChanges.Start,
|
||||
connectionManagerStart: connManager.Start,
|
||||
}, nil
|
||||
}
|
||||
|
||||
+13
-4
@@ -14,7 +14,8 @@ type MessageMetrics struct {
|
||||
rxUnknown metrics.Counter
|
||||
txUnknown metrics.Counter
|
||||
|
||||
rxInvalid metrics.Counter
|
||||
rxInvalid metrics.Counter
|
||||
txExhausted metrics.Counter
|
||||
}
|
||||
|
||||
func (m *MessageMetrics) Rx(t header.MessageType, s header.MessageSubType, i int64) {
|
||||
@@ -41,6 +42,13 @@ func (m *MessageMetrics) RxInvalid(i int64) {
|
||||
}
|
||||
}
|
||||
|
||||
// TxExhausted counts outbound packets dropped because the tunnel's message counter is spent.
|
||||
func (m *MessageMetrics) TxExhausted(i int64) {
|
||||
if m != nil && m.txExhausted != nil {
|
||||
m.txExhausted.Inc(i)
|
||||
}
|
||||
}
|
||||
|
||||
func newMessageMetrics() *MessageMetrics {
|
||||
gen := func(t string) [][]metrics.Counter {
|
||||
return [][]metrics.Counter{
|
||||
@@ -61,9 +69,10 @@ func newMessageMetrics() *MessageMetrics {
|
||||
rx: gen("rx"),
|
||||
tx: gen("tx"),
|
||||
|
||||
rxUnknown: metrics.GetOrRegisterCounter("messages.rx.other", nil),
|
||||
txUnknown: metrics.GetOrRegisterCounter("messages.tx.other", nil),
|
||||
rxInvalid: metrics.GetOrRegisterCounter("messages.rx.invalid", nil),
|
||||
rxUnknown: metrics.GetOrRegisterCounter("messages.rx.other", nil),
|
||||
txUnknown: metrics.GetOrRegisterCounter("messages.tx.other", nil),
|
||||
rxInvalid: metrics.GetOrRegisterCounter("messages.rx.invalid", nil),
|
||||
txExhausted: metrics.GetOrRegisterCounter("messages.tx.exhausted", nil),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
//go:build !android
|
||||
|
||||
package nicfinder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/netip"
|
||||
)
|
||||
|
||||
func localAddrs(ctx context.Context, l *slog.Logger, filter Filter) ([]netip.Addr, error) {
|
||||
ifaces, err := net.Interfaces()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err)
|
||||
}
|
||||
|
||||
var out []netip.Addr
|
||||
var errs []error
|
||||
|
||||
for _, i := range ifaces {
|
||||
if !allowName(ctx, l, filter, i.Name) {
|
||||
continue
|
||||
}
|
||||
addrs, err := i.Addrs()
|
||||
if err != nil {
|
||||
errs = append(errs, fmt.Errorf("failed to get addresses for %s: %w", i.Name, err))
|
||||
continue
|
||||
}
|
||||
for _, rawAddr := range addrs {
|
||||
if addr, ok := allowedAddr(ctx, l, filter, rawAddr); ok {
|
||||
out = append(out, addr)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, errors.Join(errs...)
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
//go:build android
|
||||
|
||||
package nicfinder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/netip"
|
||||
"os"
|
||||
|
||||
"github.com/DefinedNet/netlink"
|
||||
"github.com/DefinedNet/netlink/nl"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// Android denies bind() on netlink route sockets and RTM_GETLINK, so net.Interfaces fails with EACCES.
|
||||
// Enumerate with an RTM_GETADDR dump on an unbound socket instead.
|
||||
|
||||
// localInterfaces returns every interface that has an address, with its addresses. It lists
|
||||
// addresses over a netlink socket that is never bound and names each index with SIOCGIFNAME, the
|
||||
// same operations Bionic's getifaddrs uses for app UIDs, which the sandbox permits.
|
||||
func localAddrs(ctx context.Context, l *slog.Logger, filter Filter) ([]netip.Addr, error) {
|
||||
var errs []error
|
||||
addrs, err := netlinkAddrList()
|
||||
if err != nil {
|
||||
if !errors.Is(err, netlink.ErrDumpInterrupted) {
|
||||
return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err)
|
||||
}
|
||||
// The address table kept changing under the dump and netlink gave up retrying. What it
|
||||
// returned is still worth advertising.
|
||||
errs = append(errs, fmt.Errorf("local addresses may be incomplete: %w", err))
|
||||
}
|
||||
|
||||
// Any socket will do for SIOCGIFNAME.
|
||||
sfd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0)
|
||||
if err != nil {
|
||||
return nil, os.NewSyscallError("socket", err)
|
||||
}
|
||||
defer unix.Close(sfd)
|
||||
|
||||
var out []netip.Addr
|
||||
nameAllowed := make(map[int]bool) // by interface index, once named
|
||||
for _, a := range addrs {
|
||||
allow, named := nameAllowed[a.LinkIndex]
|
||||
if (named && !allow) || a.IPNet == nil {
|
||||
continue
|
||||
}
|
||||
addr, ok := allowedAddr(ctx, l, filter, a.IPNet)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if !named {
|
||||
name, err := interfaceNameByIndex(sfd, a.LinkIndex)
|
||||
// ENODEV means the interface went away after the dump. Any other failure is reported
|
||||
// and the interface skipped, so the rest are still returned.
|
||||
if err != nil && !errors.Is(err, unix.ENODEV) {
|
||||
errs = append(errs, fmt.Errorf("failed to resolve name of interface %d: %w", a.LinkIndex, err))
|
||||
}
|
||||
allow = err == nil && allowName(ctx, l, filter, name)
|
||||
nameAllowed[a.LinkIndex] = allow
|
||||
}
|
||||
if allow {
|
||||
out = append(out, addr)
|
||||
}
|
||||
}
|
||||
return out, errors.Join(errs...)
|
||||
}
|
||||
|
||||
// netlinkAddrList is netlink.AddrList over a socket that is never bound. The kernel autobinds it
|
||||
// on the first send, which the sandbox permits.
|
||||
func netlinkAddrList() ([]netlink.Addr, error) {
|
||||
fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE)
|
||||
if err != nil {
|
||||
return nil, os.NewSyscallError("socket", err)
|
||||
}
|
||||
s, err := nl.NewNetlinkSocketFromFd(fd)
|
||||
if err != nil {
|
||||
unix.Close(fd)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
h := netlink.NewHandleFromSockets(
|
||||
map[int]*nl.SocketHandle{unix.NETLINK_ROUTE: {Socket: s}},
|
||||
netlink.HandleOptions{RetryInterrupted: true},
|
||||
)
|
||||
defer h.Close()
|
||||
if err := h.SetSocketTimeout(netlink.GetSocketTimeout()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return h.AddrList(nil, netlink.FAMILY_ALL)
|
||||
}
|
||||
|
||||
// interfaceNameByIndex resolves an interface index with SIOCGIFNAME.
|
||||
func interfaceNameByIndex(fd int, index int) (string, error) {
|
||||
var ifr unix.Ifreq
|
||||
ifr.SetUint32(uint32(index)) // ifr_ifindex is in the ifr_ifru union
|
||||
if err := unix.IoctlIfreq(fd, unix.SIOCGIFNAME, &ifr); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return ifr.Name(), nil
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
package nicfinder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/netip"
|
||||
|
||||
"github.com/slackhq/nebula/logging"
|
||||
)
|
||||
|
||||
// Filter decides which interfaces, by name, and which of their addresses Find returns.
|
||||
type Filter interface {
|
||||
AllowName(string) bool
|
||||
Allow(netip.Addr) bool
|
||||
}
|
||||
|
||||
// Find returns the addresses of this host's network interfaces that allowList accepts, with
|
||||
// loopback and link-local addresses left out. When only some interfaces could be read it returns
|
||||
// their addresses alongside an error for the rest. allowList must not be nil; it is called as
|
||||
// given, so a nil pointer of a type whose methods accept a nil receiver is fine.
|
||||
func Find(ctx context.Context, l *slog.Logger, allowList Filter) ([]netip.Addr, error) {
|
||||
return localAddrs(ctx, l, allowList)
|
||||
}
|
||||
|
||||
// allowName asks filter about an interface name and traces the answer.
|
||||
func allowName(ctx context.Context, l *slog.Logger, filter Filter, name string) bool {
|
||||
allow := filter.AllowName(name)
|
||||
if l.Enabled(ctx, logging.LevelTrace) {
|
||||
l.Log(ctx, logging.LevelTrace, "localAllowList.AllowName", "interfaceName", name, "allow", allow)
|
||||
}
|
||||
return allow
|
||||
}
|
||||
|
||||
// allowedAddr converts one address a platform reports and says whether to advertise it: it must
|
||||
// be a *net.IPNet or *net.IPAddr carrying an IP, not be loopback or link-local, and be accepted by
|
||||
// filter, whose answer is traced. IPv4 comes back unmapped.
|
||||
func allowedAddr(ctx context.Context, l *slog.Logger, filter Filter, rawAddr net.Addr) (netip.Addr, bool) {
|
||||
var ip net.IP
|
||||
switch v := rawAddr.(type) {
|
||||
case *net.IPNet:
|
||||
ip = v.IP
|
||||
case *net.IPAddr:
|
||||
ip = v.IP
|
||||
}
|
||||
addr, ok := netip.AddrFromSlice(ip)
|
||||
if !ok {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
addr = addr.Unmap()
|
||||
if addr.IsLoopback() || addr.IsLinkLocalUnicast() {
|
||||
return netip.Addr{}, false
|
||||
}
|
||||
isAllowed := filter.Allow(addr)
|
||||
if l.Enabled(ctx, logging.LevelTrace) {
|
||||
l.Log(ctx, logging.LevelTrace, "localAllowList.Allow", "localAddr", addr, "allowed", isAllowed)
|
||||
}
|
||||
return addr, isAllowed
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
package nicfinder
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/netip"
|
||||
"regexp"
|
||||
"testing"
|
||||
|
||||
"github.com/slackhq/nebula/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
type allowAll struct{}
|
||||
|
||||
func (allowAll) Allow(netip.Addr) bool { return true }
|
||||
func (allowAll) AllowName(string) bool { return true }
|
||||
|
||||
type testAllowList struct {
|
||||
bannedName *regexp.Regexp
|
||||
bannedAddr netip.Addr
|
||||
}
|
||||
|
||||
func (l *testAllowList) Allow(addr netip.Addr) bool {
|
||||
return addr != l.bannedAddr
|
||||
}
|
||||
|
||||
func (l *testAllowList) AllowName(n string) bool {
|
||||
return !l.bannedName.MatchString(n)
|
||||
}
|
||||
|
||||
func newTestAllowList(bannedName string) *testAllowList {
|
||||
return &testAllowList{
|
||||
bannedName: regexp.MustCompile(bannedName),
|
||||
}
|
||||
}
|
||||
|
||||
func TestAllowedAddr(t *testing.T) {
|
||||
ctx, l := context.Background(), test.NewLogger()
|
||||
allowed := func(filter Filter, addrs []net.Addr) []netip.Addr {
|
||||
var out []netip.Addr
|
||||
for _, rawAddr := range addrs {
|
||||
if addr, ok := allowedAddr(ctx, l, filter, rawAddr); ok {
|
||||
out = append(out, addr)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
addrs := []net.Addr{
|
||||
&net.IPNet{IP: net.ParseIP("127.0.0.1"), Mask: net.CIDRMask(8, 32)},
|
||||
&net.IPNet{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)},
|
||||
&net.IPNet{IP: net.ParseIP("10.0.0.5"), Mask: net.CIDRMask(24, 32)},
|
||||
&net.IPNet{IP: net.ParseIP("169.254.10.10"), Mask: net.CIDRMask(16, 32)},
|
||||
&net.IPNet{IP: net.ParseIP("fe80::1"), Mask: net.CIDRMask(64, 128)},
|
||||
&net.IPAddr{IP: net.ParseIP("fd00::5")},
|
||||
&net.IPNet{},
|
||||
otherAddr{},
|
||||
nil,
|
||||
}
|
||||
|
||||
// Loopback, link local and anything without an IP are dropped, and IPv4 comes back unmapped
|
||||
// even though net.ParseIP hands over a 16 byte slice.
|
||||
assert.Equal(t, []netip.Addr{
|
||||
netip.MustParseAddr("10.0.0.5"),
|
||||
netip.MustParseAddr("fd00::5"),
|
||||
}, allowed(allowAll{}, addrs))
|
||||
|
||||
// The filter sees the unmapped form and can drop a single address.
|
||||
al := newTestAllowList("none")
|
||||
al.bannedAddr = netip.MustParseAddr("10.0.0.5")
|
||||
assert.Equal(t, []netip.Addr{netip.MustParseAddr("fd00::5")}, allowed(al, addrs))
|
||||
}
|
||||
|
||||
// otherAddr is a net.Addr that is neither *net.IPNet nor *net.IPAddr. netlink.Addr is one such
|
||||
// type: it satisfies net.Addr through an embedded *net.IPNet, and passing it directly once
|
||||
// silently produced no addresses.
|
||||
type otherAddr struct{}
|
||||
|
||||
func (otherAddr) Network() string { return "other" }
|
||||
func (otherAddr) String() string { return "other" }
|
||||
|
||||
// hostFilter counts AllowName calls and can reject every name or every address.
|
||||
type hostFilter struct {
|
||||
rejectNames bool
|
||||
rejectAddrs bool
|
||||
asked map[string]int
|
||||
}
|
||||
|
||||
func (f *hostFilter) Allow(netip.Addr) bool { return !f.rejectAddrs }
|
||||
func (f *hostFilter) AllowName(name string) bool {
|
||||
f.asked[name]++
|
||||
return !f.rejectNames
|
||||
}
|
||||
|
||||
// Runs this platform's localAddrs against the host and checks it honors the whole filter.
|
||||
func TestLocalAddrsAppliesFilter(t *testing.T) {
|
||||
ctx, l := context.Background(), test.NewLogger()
|
||||
|
||||
f := &hostFilter{asked: map[string]int{}}
|
||||
all, err := localAddrs(ctx, l, f)
|
||||
if len(f.asked) == 0 {
|
||||
t.Skipf("no interfaces to enumerate here: %v", err)
|
||||
}
|
||||
for name, n := range f.asked {
|
||||
assert.Equal(t, 1, n, "AllowName asked %d times for %s", n, name)
|
||||
}
|
||||
// Loopback and link-local never come back, whatever the filter says.
|
||||
for _, addr := range all {
|
||||
assert.False(t, addr.IsLoopback() || addr.IsLinkLocalUnicast(), "%s", addr)
|
||||
}
|
||||
|
||||
none, _ := localAddrs(ctx, l, &hostFilter{rejectNames: true, asked: map[string]int{}})
|
||||
assert.Empty(t, none, "addresses got past a filter that rejects every interface name")
|
||||
|
||||
none, _ = localAddrs(ctx, l, &hostFilter{rejectAddrs: true, asked: map[string]int{}})
|
||||
assert.Empty(t, none, "addresses got past a filter that rejects every address")
|
||||
}
|
||||
@@ -25,6 +25,9 @@ func (s *CipherStateAESGCM) EncryptDanger(out, ad, plaintext []byte, n uint64, n
|
||||
if s == nil {
|
||||
return nil, errors.New("no cipher state available to encrypt")
|
||||
}
|
||||
if n >= RejectAfterMessages {
|
||||
return nil, ErrMessageCounterExhausted
|
||||
}
|
||||
nb[0] = 0
|
||||
nb[1] = 0
|
||||
nb[2] = 0
|
||||
|
||||
@@ -24,6 +24,9 @@ func (s *CipherStateChaChaPoly) EncryptDanger(out, ad, plaintext []byte, n uint6
|
||||
if s == nil {
|
||||
return nil, errors.New("no cipher state available to encrypt")
|
||||
}
|
||||
if n >= RejectAfterMessages {
|
||||
return nil, ErrMessageCounterExhausted
|
||||
}
|
||||
nb[0] = 0
|
||||
nb[1] = 0
|
||||
nb[2] = 0
|
||||
|
||||
@@ -1,11 +1,22 @@
|
||||
package noiseutil
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
|
||||
"github.com/flynn/noise"
|
||||
)
|
||||
|
||||
// RejectHeadroom is the wrap gap for senders racing the counter, sized large enough for any routine count.
|
||||
const RejectHeadroom = uint64(1) << 40
|
||||
|
||||
// RejectAfterMessages is the nonce ceiling: encrypting stops RejectHeadroom short of the wrap.
|
||||
const RejectAfterMessages = math.MaxUint64 - RejectHeadroom
|
||||
|
||||
// ErrMessageCounterExhausted is returned by EncryptDanger once the nonce reaches RejectAfterMessages.
|
||||
var ErrMessageCounterExhausted = errors.New("message counter exhausted")
|
||||
|
||||
// CipherState is the post-handshake AEAD cipher used for the data plane.
|
||||
// Each supported cipher has its own concrete implementation in this package with the nonce endianness hardcoded,
|
||||
// so the encrypt/decrypt fast path avoids interface dispatch on the byte order.
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package noiseutil
|
||||
|
||||
import (
|
||||
"math"
|
||||
"testing"
|
||||
|
||||
"github.com/flynn/noise"
|
||||
@@ -89,6 +90,24 @@ func roundtrip(t *testing.T, enc, dec CipherState) {
|
||||
assert.Equal(t, 16, enc.Overhead())
|
||||
}
|
||||
|
||||
func TestEncryptRejectsExhaustedCounter(t *testing.T) {
|
||||
// Pin the headroom below the uint64 wrap so a typo can't silently move the ceiling.
|
||||
require.Equal(t, uint64(1)<<40, RejectHeadroom)
|
||||
require.Equal(t, math.MaxUint64-RejectHeadroom, RejectAfterMessages)
|
||||
|
||||
encA, _ := buildCipherStates(t, CipherAESGCM)
|
||||
encC, _ := buildCipherStates(t, noise.CipherChaChaPoly)
|
||||
nb := make([]byte, 12)
|
||||
|
||||
for _, cs := range []CipherState{NewCipherStateAESGCM(encA), NewCipherStateChaChaPoly(encC)} {
|
||||
_, err := cs.EncryptDanger(nil, nil, []byte("x"), RejectAfterMessages-1, nb)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = cs.EncryptDanger(nil, nil, []byte("x"), RejectAfterMessages, nb)
|
||||
require.ErrorIs(t, err, ErrMessageCounterExhausted)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkCipherStateEncryptAESGCM(b *testing.B) {
|
||||
enc, _ := buildCipherStatesB(b, CipherAESGCM)
|
||||
benchEncryptCipherState(b, NewCipherState(enc, CipherAESGCM))
|
||||
|
||||
+43
-85
@@ -13,6 +13,7 @@ import (
|
||||
|
||||
"github.com/slackhq/nebula/firewall"
|
||||
"github.com/slackhq/nebula/header"
|
||||
"github.com/slackhq/nebula/iputil"
|
||||
"golang.org/x/net/ipv4"
|
||||
)
|
||||
|
||||
@@ -299,7 +300,6 @@ var (
|
||||
ErrIPv4InvalidHeaderLength = errors.New("invalid ipv4 header length")
|
||||
ErrIPv4PacketTooShort = errors.New("ipv4 packet is too short")
|
||||
ErrIPv6PacketTooShort = errors.New("ipv6 packet is too short")
|
||||
ErrIPv6CouldNotFindPayload = errors.New("could not find payload in ipv6 packet")
|
||||
)
|
||||
|
||||
// newPacket validates and parses the interesting bits for the firewall out of the ip and sub protocol headers
|
||||
@@ -332,101 +332,59 @@ func parseV6(data []byte, incoming bool, fp *firewall.Packet) error {
|
||||
fp.RemoteAddr, _ = netip.AddrFromSlice(data[24:40])
|
||||
}
|
||||
|
||||
protoAt := 6 // NextHeader is at 6 bytes into the ipv6 header
|
||||
offset := ipv6.HeaderLen // Start at the end of the ipv6 header
|
||||
next := 0
|
||||
for {
|
||||
if protoAt >= dataLen {
|
||||
break
|
||||
// Walk the extension header chain to the upper layer protocol. iputil.IPv6FindUpperProtocol is the single
|
||||
// source of truth for which headers are extension headers, so this stays in lockstep with the reject path
|
||||
// and cannot drift into misreading an unknown protocol (SCTP, GRE, etc.) as a forged transport.
|
||||
proto, offset, isFragment, _, err := iputil.IPv6FindUpperProtocol(data)
|
||||
if err != nil {
|
||||
return ErrIPv6PacketTooShort
|
||||
}
|
||||
|
||||
fp.Protocol = proto
|
||||
fp.Fragment = isFragment
|
||||
if isFragment {
|
||||
// Non-first fragments carry no transport header, so we have no ports to read
|
||||
fp.RemotePort = 0
|
||||
fp.LocalPort = 0
|
||||
return nil
|
||||
}
|
||||
|
||||
switch layers.IPProtocol(proto) {
|
||||
case layers.IPProtocolICMPv6:
|
||||
// An ICMPv6 message is at least type, code and checksum, 4 bytes. Only echo carries more than we read.
|
||||
if dataLen < offset+4 {
|
||||
return ErrIPv6PacketTooShort
|
||||
}
|
||||
proto := layers.IPProtocol(data[protoAt])
|
||||
|
||||
switch proto {
|
||||
case layers.IPProtocolESP, layers.IPProtocolNoNextHeader:
|
||||
fp.Protocol = uint8(proto)
|
||||
fp.RemotePort = 0
|
||||
fp.LocalPort = 0
|
||||
fp.Fragment = false
|
||||
return nil
|
||||
|
||||
case layers.IPProtocolICMPv6:
|
||||
fp.LocalPort = 0 //incoming vs outgoing doesn't matter for icmpv6
|
||||
switch data[offset] { //icmp type
|
||||
case layers.ICMPv6TypeEchoRequest, layers.ICMPv6TypeEchoReply:
|
||||
if dataLen < offset+6 {
|
||||
return ErrIPv6PacketTooShort
|
||||
}
|
||||
fp.Protocol = uint8(proto)
|
||||
fp.LocalPort = 0 //incoming vs outgoing doesn't matter for icmpv6
|
||||
icmptype := data[offset+1]
|
||||
switch icmptype {
|
||||
case layers.ICMPv6TypeEchoRequest, layers.ICMPv6TypeEchoReply:
|
||||
fp.RemotePort = binary.BigEndian.Uint16(data[offset+4 : offset+6]) //identifier
|
||||
default:
|
||||
fp.RemotePort = 0
|
||||
}
|
||||
fp.Fragment = false
|
||||
return nil
|
||||
|
||||
case layers.IPProtocolTCP, layers.IPProtocolUDP:
|
||||
if dataLen < offset+4 {
|
||||
return ErrIPv6PacketTooShort
|
||||
}
|
||||
|
||||
fp.Protocol = uint8(proto)
|
||||
if incoming {
|
||||
fp.RemotePort = binary.BigEndian.Uint16(data[offset : offset+2])
|
||||
fp.LocalPort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
|
||||
} else {
|
||||
fp.LocalPort = binary.BigEndian.Uint16(data[offset : offset+2])
|
||||
fp.RemotePort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
|
||||
}
|
||||
|
||||
fp.Fragment = false
|
||||
return nil
|
||||
|
||||
case layers.IPProtocolIPv6Fragment:
|
||||
// Fragment header is 8 bytes, need at least offset+4 to read the offset field
|
||||
if dataLen < offset+8 {
|
||||
return ErrIPv6PacketTooShort
|
||||
}
|
||||
|
||||
// Check if this is the first fragment
|
||||
fragmentOffset := binary.BigEndian.Uint16(data[offset+2:offset+4]) &^ uint16(0x7) // Remove the reserved and M flag bits
|
||||
if fragmentOffset != 0 {
|
||||
// Non-first fragment, use what we have now and stop processing
|
||||
fp.Protocol = data[offset]
|
||||
fp.Fragment = true
|
||||
fp.RemotePort = 0
|
||||
fp.LocalPort = 0
|
||||
return nil
|
||||
}
|
||||
|
||||
// The next loop should be the transport layer since we are the first fragment
|
||||
next = 8 // Fragment headers are always 8 bytes
|
||||
|
||||
case layers.IPProtocolAH:
|
||||
// Auth headers, used by IPSec, have a different meaning for header length
|
||||
if dataLen <= offset+1 {
|
||||
break
|
||||
}
|
||||
next = (int(data[offset+1]) + 2) << 2
|
||||
|
||||
fp.RemotePort = binary.BigEndian.Uint16(data[offset+4 : offset+6]) //identifier
|
||||
default:
|
||||
// Normal ipv6 header length processing
|
||||
if dataLen <= offset+1 {
|
||||
break
|
||||
}
|
||||
next = (int(data[offset+1]) + 1) << 3
|
||||
fp.RemotePort = 0
|
||||
}
|
||||
|
||||
if next <= 0 {
|
||||
// Safety check, each ipv6 header has to be at least 8 bytes
|
||||
next = 8
|
||||
case layers.IPProtocolTCP, layers.IPProtocolUDP:
|
||||
if dataLen < offset+4 {
|
||||
return ErrIPv6PacketTooShort
|
||||
}
|
||||
if incoming {
|
||||
fp.RemotePort = binary.BigEndian.Uint16(data[offset : offset+2])
|
||||
fp.LocalPort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
|
||||
} else {
|
||||
fp.LocalPort = binary.BigEndian.Uint16(data[offset : offset+2])
|
||||
fp.RemotePort = binary.BigEndian.Uint16(data[offset+2 : offset+4])
|
||||
}
|
||||
|
||||
protoAt = offset
|
||||
offset = offset + next
|
||||
default:
|
||||
// don't set ports for protocols Nebula doesn't inspect
|
||||
fp.RemotePort = 0
|
||||
fp.LocalPort = 0
|
||||
}
|
||||
|
||||
return ErrIPv6CouldNotFindPayload
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseV4(data []byte, incoming bool, fp *firewall.Packet) error {
|
||||
|
||||
+89
-9
@@ -14,6 +14,7 @@ import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/net/ipv4"
|
||||
"golang.org/x/net/ipv6"
|
||||
)
|
||||
|
||||
func Test_newPacket(t *testing.T) {
|
||||
@@ -115,12 +116,12 @@ func Test_newPacket_v6(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
|
||||
err = newPacket(buffer.Bytes(), true, p)
|
||||
require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
|
||||
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
|
||||
|
||||
// A v6 packet with a hop-by-hop extension
|
||||
// ICMPv6 Payload (Echo Request)
|
||||
icmpLayer := layers.ICMPv6{
|
||||
TypeCode: layers.ICMPv6TypeEchoRequest,
|
||||
TypeCode: layers.CreateICMPv6TypeCode(layers.ICMPv6TypeEchoRequest, 0),
|
||||
}
|
||||
// Hop-by-Hop Extension Header
|
||||
hopOption := layers.IPv6HopByHopOption{}
|
||||
@@ -149,12 +150,12 @@ func Test_newPacket_v6(t *testing.T) {
|
||||
// A full IPv6 header and 1 byte in the first extension, but missing
|
||||
// the length byte.
|
||||
err = newPacket(buffer.Bytes()[:41], true, p)
|
||||
require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
|
||||
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
|
||||
|
||||
// A full IPv6 header plus 1 full extension, but only 1 byte of the
|
||||
// next layer, missing length byte
|
||||
err = newPacket(buffer.Bytes()[:49], true, p)
|
||||
require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
|
||||
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
|
||||
err = nil
|
||||
|
||||
// A good ICMP packet
|
||||
@@ -167,7 +168,7 @@ func Test_newPacket_v6(t *testing.T) {
|
||||
}
|
||||
|
||||
icmp := layers.ICMPv6{
|
||||
TypeCode: layers.ICMPv6TypeEchoRequest,
|
||||
TypeCode: layers.CreateICMPv6TypeCode(layers.ICMPv6TypeEchoRequest, 0),
|
||||
Checksum: 0x1234,
|
||||
}
|
||||
|
||||
@@ -189,6 +190,18 @@ func Test_newPacket_v6(t *testing.T) {
|
||||
assert.Equal(t, uint16(0), p.LocalPort)
|
||||
assert.False(t, p.Fragment)
|
||||
|
||||
// A minimal 4 byte non-echo ICMPv6 message (type, code, checksum), no identifier to read
|
||||
icmpMin := make([]byte, ipv6.HeaderLen+4)
|
||||
copy(icmpMin, buffer.Bytes()[:ipv6.HeaderLen])
|
||||
icmpMin[6] = byte(layers.IPProtocolICMPv6)
|
||||
icmpMin[ipv6.HeaderLen] = 1 // type 1, destination unreachable, not echo
|
||||
err = newPacket(icmpMin, true, p)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint8(layers.IPProtocolICMPv6), p.Protocol)
|
||||
assert.Equal(t, uint16(0), p.RemotePort)
|
||||
assert.Equal(t, uint16(0), p.LocalPort)
|
||||
assert.False(t, p.Fragment)
|
||||
|
||||
// A good ESP packet
|
||||
b := buffer.Bytes()
|
||||
b[6] = byte(layers.IPProtocolESP)
|
||||
@@ -213,11 +226,15 @@ func Test_newPacket_v6(t *testing.T) {
|
||||
assert.Equal(t, uint16(0), p.LocalPort)
|
||||
assert.False(t, p.Fragment)
|
||||
|
||||
// An unknown protocol packet
|
||||
// An unknown protocol packet, we don't dissect it so we fail closed on its true protocol with no ports
|
||||
b = buffer.Bytes()
|
||||
b[6] = 255 // 255 is a reserved protocol number
|
||||
err = newPacket(b, true, p)
|
||||
require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint8(255), p.Protocol)
|
||||
assert.Equal(t, uint16(0), p.RemotePort)
|
||||
assert.Equal(t, uint16(0), p.LocalPort)
|
||||
assert.False(t, p.Fragment)
|
||||
|
||||
// A good UDP packet
|
||||
ip = layers.IPv6{
|
||||
@@ -334,14 +351,14 @@ func Test_newPacket_v6(t *testing.T) {
|
||||
assert.Equal(t, uint16(22), p.LocalPort)
|
||||
assert.False(t, p.Fragment)
|
||||
|
||||
// Ensure buffer bounds checking during processing
|
||||
// Ensure buffer bounds checking during processing, a truncated AH header can't reach the payload
|
||||
err = newPacket(b[:41], true, p)
|
||||
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
|
||||
|
||||
// Invalid AH header
|
||||
b = buffer.Bytes()
|
||||
err = newPacket(b, true, p)
|
||||
require.ErrorIs(t, err, ErrIPv6CouldNotFindPayload)
|
||||
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
|
||||
}
|
||||
|
||||
func Test_newPacket_ipv6Fragment(t *testing.T) {
|
||||
@@ -675,3 +692,66 @@ func Test_newPacket_v6ExtHeaderOverflow(t *testing.T) {
|
||||
// the host delivers to, not the forged 443 at the overflowed offset.
|
||||
assert.Equal(t, uint16(22), p.LocalPort, "firewall must parse the real transport header, not the overflowed offset")
|
||||
}
|
||||
|
||||
// Test_newPacket_v6ExtHeaderPastBuffer is a regression test for an extension header whose declared length
|
||||
// advances the walk past the end of the packet. The upper layer protocol's header isn't actually present,
|
||||
// so parseV6 must drop the packet rather than classify it as the terminal protocol with no ports.
|
||||
func Test_newPacket_v6ExtHeaderPastBuffer(t *testing.T) {
|
||||
p := &firewall.Packet{}
|
||||
|
||||
pkt := make([]byte, 48)
|
||||
pkt[0] = 0x60
|
||||
pkt[6] = byte(layers.IPProtocolIPv6Destination) // Destination Options
|
||||
pkt[7] = 64 // hop limit
|
||||
pkt[40] = byte(layers.IPProtocolSCTP) // Dest Options next header = SCTP
|
||||
pkt[41] = 255 // declared length (255+1)*8 = 2048, past the 48 byte buffer
|
||||
|
||||
require.ErrorIs(t, newPacket(pkt, true, p), ErrIPv6PacketTooShort)
|
||||
}
|
||||
|
||||
// Test_newPacket_v6ExtHeaderConfusion is a regression test for parseV6 walking any unrecognized
|
||||
// Next Header as if it were an ipv6 extension header. A real upper layer protocol Nebula doesn't
|
||||
// dissect (SCTP here) is not walkable, so applying the (len+1)*8 formula marched into the SCTP
|
||||
// payload and landed on a byte that looked like UDP, forging a protocol/port pair the firewall
|
||||
// would trust while the host delivered the real SCTP datagram. The fix fails closed: the packet
|
||||
// is classified as its true protocol with no ports, so it only matches an `any` rule.
|
||||
func Test_newPacket_v6ExtHeaderConfusion(t *testing.T) {
|
||||
p := &firewall.Packet{}
|
||||
|
||||
pkt := make([]byte, 52)
|
||||
pkt[0] = 0x60 // version 6
|
||||
pkt[6] = byte(layers.IPProtocolSCTP) // NextHeader = SCTP, a real protocol, not an extension header
|
||||
pkt[7] = 64 // hop limit
|
||||
|
||||
// Real SCTP header at offset 40. Pre-fix parseV6 walked SCTP as an extension header: byte 41 (0x00, the
|
||||
// low byte of the src port below) was read as the header length, giving next=(0+1)*8=8, which landed the
|
||||
// walk on byte 40 (0x11), misread as NextHeader=UDP, then bytes 48-51 as ports.
|
||||
binary.BigEndian.PutUint16(pkt[40:42], 0x1100) // SCTP src port; byte 40=0x11, byte 41=0x00
|
||||
binary.BigEndian.PutUint16(pkt[42:44], 445) // SCTP dst port, never read by parseV6
|
||||
binary.BigEndian.PutUint16(pkt[48:50], 53) // SCTP checksum bytes, pre-fix forged RemotePort
|
||||
binary.BigEndian.PutUint16(pkt[50:52], 53) // pre-fix forged LocalPort
|
||||
|
||||
require.NoError(t, newPacket(pkt, true, p))
|
||||
assert.Equal(t, uint8(layers.IPProtocolSCTP), p.Protocol, "must classify as the true protocol, not the forged UDP")
|
||||
assert.Equal(t, uint16(0), p.RemotePort)
|
||||
assert.Equal(t, uint16(0), p.LocalPort)
|
||||
assert.False(t, p.Fragment)
|
||||
|
||||
// Same confusion, but the unknown protocol sits after a real extension header. The HopByHop is walked
|
||||
// correctly, then SCTP must still fail closed instead of being walked into its own payload. Protocol is
|
||||
// the only assertion that discriminates the fix here, a regression that walked SCTP would misclassify it.
|
||||
chained := make([]byte, 60)
|
||||
chained[0] = 0x60 // version 6
|
||||
chained[6] = byte(layers.IPProtocolIPv6HopByHop) // NextHeader = HopByHop extension
|
||||
chained[7] = 64 // hop limit
|
||||
chained[40] = byte(layers.IPProtocolSCTP) // HopByHop NextHeader = SCTP
|
||||
chained[41] = 0 // HopByHop length 0 -> 8 bytes, SCTP begins at offset 48
|
||||
binary.BigEndian.PutUint16(chained[48:50], 0x1100) // SCTP src port, pre-fix forged NextHeader/length bait
|
||||
binary.BigEndian.PutUint16(chained[50:52], 445) // SCTP dst port, never read by parseV6
|
||||
|
||||
require.NoError(t, newPacket(chained, true, p))
|
||||
assert.Equal(t, uint8(layers.IPProtocolSCTP), p.Protocol, "must fail closed on the unknown protocol after the extension header")
|
||||
assert.Equal(t, uint16(0), p.RemotePort)
|
||||
assert.Equal(t, uint16(0), p.LocalPort)
|
||||
assert.False(t, p.Fragment)
|
||||
}
|
||||
+12
-2
@@ -5,6 +5,7 @@ package overlay
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
@@ -17,11 +18,11 @@ import (
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"github.com/DefinedNet/netlink"
|
||||
"github.com/gaissmai/bart"
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/routing"
|
||||
"github.com/slackhq/nebula/util"
|
||||
"github.com/vishvananda/netlink"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
@@ -483,7 +484,16 @@ func (t *tun) addIPs(link netlink.Link) error {
|
||||
//iterate over remainder, remove whoever shouldn't be there
|
||||
al, err := netlink.AddrList(link, netlink.FAMILY_ALL)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get tun address list: %s", err)
|
||||
//RTM_GETADDR dumps the whole system, so any concurrent address change
|
||||
//interrupts it - including the kernel's async tentative->preferred
|
||||
//flip of an IPv6 address the AddrReplace calls above just added,
|
||||
//which makes this a race against our own setup. Partial results are
|
||||
//still returned; the worst case is a stale address surviving until
|
||||
//the next config reload, which beats failing startup over it.
|
||||
if !errors.Is(err, netlink.ErrDumpInterrupted) {
|
||||
return fmt.Errorf("failed to get tun address list: %s", err)
|
||||
}
|
||||
t.l.Warn("tun address list dump was interrupted, stale addresses may remain")
|
||||
}
|
||||
|
||||
for i := range al {
|
||||
|
||||
@@ -12,6 +12,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
@@ -178,6 +179,7 @@ func (t *winTun) addRoutes(logErrors bool) error {
|
||||
luid := winipcfg.LUID(t.tun.LUID())
|
||||
routes := *t.Routes.Load()
|
||||
foundDefault4 := false
|
||||
carriesV6 := slices.ContainsFunc(t.vpnNetworks, func(p netip.Prefix) bool { return p.Addr().Is6() })
|
||||
|
||||
for _, r := range routes {
|
||||
if len(r.Via) == 0 || !r.Install {
|
||||
@@ -185,6 +187,9 @@ func (t *winTun) addRoutes(logErrors bool) error {
|
||||
continue
|
||||
}
|
||||
|
||||
// A v6 unsafe_route is legal under a v4-only cert; uninstalled ones put nothing on the adapter.
|
||||
carriesV6 = carriesV6 || r.Cidr.Addr().Is6()
|
||||
|
||||
// Add our unsafe route as an on-link route to the nebula tun device.
|
||||
err := luid.AddRoute(r.Cidr, unspecifiedNextHop(r.Cidr), uint32(r.Metric))
|
||||
if err != nil {
|
||||
@@ -206,6 +211,11 @@ func (t *winTun) addRoutes(logErrors bool) error {
|
||||
}
|
||||
}
|
||||
|
||||
return t.setMTU(luid, foundDefault4, carriesV6)
|
||||
}
|
||||
|
||||
// setMTU applies tun.mtu per address family. The default route metric rides along on the v4 handle.
|
||||
func (t *winTun) setMTU(luid winipcfg.LUID, foundDefault4, carriesV6 bool) error {
|
||||
ipif, err := luid.IPInterface(windows.AF_INET)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get ip interface: %w", err)
|
||||
@@ -220,6 +230,25 @@ func (t *winTun) addRoutes(logErrors bool) error {
|
||||
if err := ipif.Set(); err != nil {
|
||||
return fmt.Errorf("failed to set ip interface: %w", err)
|
||||
}
|
||||
|
||||
// Windows tracks NLMTU per family and wintun sets neither, so v6 keeps the adapter default of 65535.
|
||||
// Gated so a v4-only overlay under 1280 boots; a v6 one deliberately does not, as linux also refuses.
|
||||
if !carriesV6 {
|
||||
return nil
|
||||
}
|
||||
|
||||
ipif6, err := luid.IPInterface(windows.AF_INET6)
|
||||
if err != nil {
|
||||
// No v6 on the adapter means there is no NLMTU to get wrong. A failed Set below is not the same thing.
|
||||
t.l.Info("Skipping ipv6 MTU, no ipv6 interface on this adapter", "error", err)
|
||||
return nil
|
||||
}
|
||||
|
||||
ipif6.NLMTU = uint32(t.MTU)
|
||||
if err := ipif6.Set(); err != nil {
|
||||
return fmt.Errorf("failed to set ipv6 interface: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
graphite "github.com/cyberdelia/go-metrics-graphite"
|
||||
mp "github.com/nbrownus/go-metrics-prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
@@ -253,7 +252,7 @@ func (s *statsServer) buildRuntime(cfg statsConfig) ([]func(), *http.Server) {
|
||||
// loadStatsConfig already resolved and validated the address; re-parse
|
||||
// the resolved form (no DNS lookup) to get a *net.TCPAddr.
|
||||
addr, _ := net.ResolveTCPAddr(cfg.graphite.protocol, cfg.graphite.resolvedAddr)
|
||||
gcfg := graphite.Config{
|
||||
gcfg := graphiteConfigExport{
|
||||
Addr: addr,
|
||||
Registry: metrics.DefaultRegistry,
|
||||
FlushInterval: cfg.interval,
|
||||
@@ -262,7 +261,7 @@ func (s *statsServer) buildRuntime(cfg statsConfig) ([]func(), *http.Server) {
|
||||
Percentiles: []float64{0.5, 0.75, 0.95, 0.99, 0.999},
|
||||
}
|
||||
captureFns = append(captureFns, func() {
|
||||
if err := graphite.Once(gcfg); err != nil {
|
||||
if err := graphiteOnce(gcfg); err != nil {
|
||||
s.l.Error("Graphite export failed", "error", err)
|
||||
}
|
||||
})
|
||||
|
||||
+1
-1
@@ -371,7 +371,7 @@ func waitForListening(t *testing.T, addr string) {
|
||||
})
|
||||
}
|
||||
|
||||
// graphiteSink is a minimal TCP accept-and-discard server so graphite.Once
|
||||
// graphiteSink is a minimal TCP accept-and-discard server so graphiteOnce
|
||||
// calls in tests don't spam error logs or wedge on connection refused.
|
||||
type graphiteSink struct {
|
||||
ln net.Listener
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
package udp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
|
||||
"github.com/slackhq/nebula/config"
|
||||
)
|
||||
|
||||
// NetworkChangeMonitor rebinds the udp listener when the local network moves out from under it.
|
||||
//
|
||||
// Detection lives here in the udp package, next to the socket it concerns and the platform matrix that already knows
|
||||
// which sockets go stale. What to do about a change — updating the lighthouse, requerying tunnels — is not the udp
|
||||
// package's business, so Start takes the reaction as a plain function. Passing it at Start rather than holding it
|
||||
// keeps this package from referencing whatever owns the rebind.
|
||||
//
|
||||
// On platforms whose sockets do not go stale, watchNetworkChanges hands back a nil channel and Start returns.
|
||||
type NetworkChangeMonitor struct {
|
||||
l *slog.Logger
|
||||
ctx context.Context
|
||||
enabled bool
|
||||
}
|
||||
|
||||
// NewNetworkChangeMonitor builds a monitor for local network changes. The returned monitor is always usable: Start
|
||||
// is safe to call unconditionally, it no-ops when disabled or on a platform that does not need it.
|
||||
func NewNetworkChangeMonitor(ctx context.Context, l *slog.Logger, c *config.C) *NetworkChangeMonitor {
|
||||
return &NetworkChangeMonitor{
|
||||
l: l,
|
||||
ctx: ctx,
|
||||
enabled: c.GetBool("listen.rebind_on_network_change", true),
|
||||
}
|
||||
}
|
||||
|
||||
// Start watches for network changes until the context is cancelled, calling rebind once per settled change. It
|
||||
// blocks, so callers run it in a goroutine, and it no-ops when disabled, unsupported, or with nothing to rebind.
|
||||
func (m *NetworkChangeMonitor) Start(rebind func()) {
|
||||
if !m.enabled || rebind == nil || m.ctx.Err() != nil {
|
||||
return
|
||||
}
|
||||
|
||||
changes, err := watchNetworkChanges(m.ctx, m.l)
|
||||
if err != nil {
|
||||
// Not fatal. Everything else still works, we just won't notice a network change on our own.
|
||||
m.l.Error("Failed to watch for network changes, will not rebind the udp listener when the network moves",
|
||||
"error", err,
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
if changes == nil {
|
||||
// This platform's sockets don't go stale, so there is nothing to watch for.
|
||||
return
|
||||
}
|
||||
|
||||
m.l.Info("Watching for network changes to rebind the udp listener")
|
||||
|
||||
for range changes {
|
||||
m.l.Info("Local network changed, rebinding the udp listener")
|
||||
rebind()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
//go:build darwin && !ios && !e2e_testing
|
||||
// +build darwin,!ios,!e2e_testing
|
||||
|
||||
package udp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
const (
|
||||
// netChangeSettleWindow is how long we keep swallowing routing messages after the first interesting one. A
|
||||
// single network change is never a single message, it is a burst: the link drops, addresses go away, new ones
|
||||
// arrive, routes get rewritten. Reporting part way through that just means reporting again.
|
||||
netChangeSettleWindow = time.Second
|
||||
|
||||
// netChangeReadBuffer is sized well past any rt_msghdr plus its addresses. A short read would be discarded by
|
||||
// the kernel, so being generous here is how we avoid missing a message.
|
||||
netChangeReadBuffer = 4096
|
||||
)
|
||||
|
||||
// watchNetworkChanges reports when the local network moves out from under us, so the listener can be rebound.
|
||||
//
|
||||
// Darwin scopes a udp socket to whatever interface it came up on. Move between networks and we keep sending out an
|
||||
// interface that no longer has a route, which surfaces as an instant "no route to host" with no packet ever leaving
|
||||
// the box. Rebind clears that, but only if something notices the change and calls it. iOS has always been told by
|
||||
// the host app off NWPathMonitor. This is the equivalent for everything else that runs on darwin.
|
||||
//
|
||||
// The returned channel is buffered and coalescing: a send is dropped if one is already pending, since both mean the
|
||||
// same thing to a reader. It is closed when ctx is cancelled or the routing socket fails, so a caller can simply
|
||||
// range over it. Platforms whose sockets do not need rebinding return a nil channel and no error.
|
||||
func watchNetworkChanges(ctx context.Context, l *slog.Logger) (<-chan struct{}, error) {
|
||||
sock, err := openRouteSocket()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
changes := make(chan struct{}, 1)
|
||||
|
||||
go func() {
|
||||
defer close(changes)
|
||||
defer func() { _ = sock.Close() }()
|
||||
|
||||
// Closing the socket is what unblocks the read in watchRouteSocket, so this turns cancellation into a
|
||||
// close. It is scoped to this call so it cannot outlive the watch it belongs to.
|
||||
done := make(chan struct{})
|
||||
defer close(done)
|
||||
go func() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
_ = sock.Close()
|
||||
case <-done:
|
||||
}
|
||||
}()
|
||||
|
||||
watchRouteSocket(l, sock, changes)
|
||||
}()
|
||||
|
||||
return changes, nil
|
||||
}
|
||||
|
||||
// watchRouteSocket blocks reading the routing socket, reporting once per settled burst of changes. It returns when
|
||||
// the socket is closed, which is how cancellation gets us out of here.
|
||||
func watchRouteSocket(l *slog.Logger, sock *os.File, changes chan<- struct{}) {
|
||||
buf := make([]byte, netChangeReadBuffer)
|
||||
|
||||
for {
|
||||
n, err := sock.Read(buf)
|
||||
if err != nil {
|
||||
logRouteSocketError(l, err)
|
||||
return
|
||||
}
|
||||
|
||||
if !isNetworkChange(buf[:n]) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Swallow the rest of the burst. The deadline is absolute and not extended by what arrives, so this always
|
||||
// ends after the settle window no matter how chatty the socket is. Changes that land after the window
|
||||
// simply produce another report, which is the correct outcome anyway.
|
||||
deadline := time.Now().Add(netChangeSettleWindow)
|
||||
for {
|
||||
if err = sock.SetReadDeadline(deadline); err != nil {
|
||||
logRouteSocketError(l, err)
|
||||
return
|
||||
}
|
||||
|
||||
if _, err = sock.Read(buf); err != nil {
|
||||
if os.IsTimeout(err) {
|
||||
break
|
||||
}
|
||||
logRouteSocketError(l, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err = sock.SetReadDeadline(time.Time{}); err != nil {
|
||||
logRouteSocketError(l, err)
|
||||
return
|
||||
}
|
||||
|
||||
select {
|
||||
case changes <- struct{}{}:
|
||||
default:
|
||||
// One already pending, and a second "the network moved" tells the reader nothing new.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// logRouteSocketError reports a routing socket failure unless it is just us shutting the socket down.
|
||||
func logRouteSocketError(l *slog.Logger, err error) {
|
||||
if errors.Is(err, os.ErrClosed) {
|
||||
return
|
||||
}
|
||||
|
||||
l.Error("Error reading the routing socket, will no longer notice local network changes", "error", err)
|
||||
}
|
||||
|
||||
// openRouteSocket returns the routing socket as a non blocking os.File. Going through os.File puts reads on the go
|
||||
// poller, which buys us both a working read deadline and a Close that unblocks a read in progress.
|
||||
func openRouteSocket() (*os.File, error) {
|
||||
fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err = unix.SetNonblock(fd, true); err != nil {
|
||||
_ = unix.Close(fd)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return os.NewFile(uintptr(fd), "route"), nil
|
||||
}
|
||||
|
||||
// isNetworkChange reports whether a routing message means our local addressing may have moved out from under us.
|
||||
//
|
||||
// We read the header instead of parsing the message because the type is the only part we need, and a full parse can
|
||||
// fail on shapes we don't care about, which would turn "a message I can't parse" into "a change I missed".
|
||||
// rt_msghdr, if_msghdr and ifa_msghdr all begin with the same three fields, so this is the same for every type.
|
||||
func isNetworkChange(msg []byte) bool {
|
||||
if len(msg) < 4 {
|
||||
return false
|
||||
}
|
||||
|
||||
// u_short msglen, u_char version, u_char type
|
||||
if int(binary.NativeEndian.Uint16(msg[0:2])) > len(msg) || msg[2] != unix.RTM_VERSION {
|
||||
return false
|
||||
}
|
||||
|
||||
switch msg[3] {
|
||||
case unix.RTM_NEWADDR, unix.RTM_DELADDR, unix.RTM_IFINFO:
|
||||
// An address arrived or left, or a link changed state. Anything else on this socket is either a route
|
||||
// churning underneath us, which a rebind doesn't help with, or unrelated traffic.
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
//go:build darwin && !ios && !e2e_testing
|
||||
// +build darwin,!ios,!e2e_testing
|
||||
|
||||
package udp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/goleak"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// routeMsg builds the first four bytes of a routing message, which is all isNetworkChange reads.
|
||||
func routeMsg(msgType uint8, extra int) []byte {
|
||||
msg := make([]byte, 4+extra)
|
||||
binary.NativeEndian.PutUint16(msg[0:2], uint16(len(msg)))
|
||||
msg[2] = unix.RTM_VERSION
|
||||
msg[3] = msgType
|
||||
return msg
|
||||
}
|
||||
|
||||
func TestIsNetworkChange(t *testing.T) {
|
||||
// The three that mean our addressing may have moved
|
||||
assert.True(t, isNetworkChange(routeMsg(unix.RTM_NEWADDR, 0)))
|
||||
assert.True(t, isNetworkChange(routeMsg(unix.RTM_DELADDR, 0)))
|
||||
assert.True(t, isNetworkChange(routeMsg(unix.RTM_IFINFO, 0)))
|
||||
|
||||
// Route churn is not something a rebind helps with
|
||||
assert.False(t, isNetworkChange(routeMsg(unix.RTM_ADD, 0)))
|
||||
assert.False(t, isNetworkChange(routeMsg(unix.RTM_DELETE, 0)))
|
||||
assert.False(t, isNetworkChange(routeMsg(unix.RTM_GET, 0)))
|
||||
|
||||
// Garbage must not be mistaken for a change
|
||||
assert.False(t, isNetworkChange(nil), "empty")
|
||||
assert.False(t, isNetworkChange([]byte{0, 0, 0}), "short header")
|
||||
|
||||
wrongVersion := routeMsg(unix.RTM_NEWADDR, 0)
|
||||
wrongVersion[2] = unix.RTM_VERSION + 1
|
||||
assert.False(t, isNetworkChange(wrongVersion), "wrong rtm_version")
|
||||
|
||||
lying := routeMsg(unix.RTM_NEWADDR, 0)
|
||||
binary.NativeEndian.PutUint16(lying[0:2], 512)
|
||||
assert.False(t, isNetworkChange(lying), "msglen longer than what we read")
|
||||
}
|
||||
|
||||
// socketPair returns a connected pair of datagram sockets, the first wrapped the same way the routing socket is. It
|
||||
// stands in for the kernel so the watch loop can be driven with synthetic messages.
|
||||
func socketPair(t *testing.T) (*os.File, int) {
|
||||
t.Helper()
|
||||
|
||||
fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_DGRAM, 0)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, unix.SetNonblock(fds[0], true))
|
||||
|
||||
f := os.NewFile(uintptr(fds[0]), "route")
|
||||
t.Cleanup(func() {
|
||||
_ = f.Close()
|
||||
_ = unix.Close(fds[1])
|
||||
})
|
||||
|
||||
return f, fds[1]
|
||||
}
|
||||
|
||||
func TestWatchRouteSocketCoalescesABurst(t *testing.T) {
|
||||
sock, kernel := socketPair(t)
|
||||
changes := make(chan struct{}, 1)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
watchRouteSocket(test.NewLogger(), sock, changes)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
// One network change is a burst of messages. All of these land inside the settle window, so they must produce
|
||||
// exactly one report rather than one apiece.
|
||||
for range 5 {
|
||||
_, err := unix.Write(kernel, routeMsg(unix.RTM_NEWADDR, 8))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
// Uninteresting messages in the middle of a burst must not add a report of their own either.
|
||||
_, err := unix.Write(kernel, routeMsg(unix.RTM_ADD, 8))
|
||||
require.NoError(t, err)
|
||||
|
||||
select {
|
||||
case <-changes:
|
||||
case <-time.After(netChangeSettleWindow * 4):
|
||||
t.Fatal("a burst should have reported a change")
|
||||
}
|
||||
|
||||
// Nothing more from that burst
|
||||
select {
|
||||
case <-changes:
|
||||
t.Fatal("a burst should report exactly once")
|
||||
case <-time.After(netChangeSettleWindow):
|
||||
}
|
||||
|
||||
// A change after the window has closed is a separate event and gets its own report.
|
||||
_, err = unix.Write(kernel, routeMsg(unix.RTM_IFINFO, 8))
|
||||
require.NoError(t, err)
|
||||
select {
|
||||
case <-changes:
|
||||
case <-time.After(netChangeSettleWindow * 4):
|
||||
t.Fatal("a later change should report again")
|
||||
}
|
||||
|
||||
// Closing the socket is how the real thing shuts down
|
||||
require.NoError(t, sock.Close())
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second * 5):
|
||||
t.Fatal("watchRouteSocket did not return after the socket was closed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWatchRouteSocketIgnoresUninterestingMessages(t *testing.T) {
|
||||
sock, kernel := socketPair(t)
|
||||
changes := make(chan struct{}, 1)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
watchRouteSocket(test.NewLogger(), sock, changes)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
for _, msgType := range []uint8{unix.RTM_ADD, unix.RTM_DELETE, unix.RTM_GET, unix.RTM_MISS} {
|
||||
_, err := unix.Write(kernel, routeMsg(msgType, 8))
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
select {
|
||||
case <-changes:
|
||||
t.Fatal("route churn alone must not report a change")
|
||||
case <-time.After(netChangeSettleWindow * 2):
|
||||
}
|
||||
|
||||
require.NoError(t, sock.Close())
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second * 5):
|
||||
t.Fatal("watchRouteSocket did not return after the socket was closed")
|
||||
}
|
||||
}
|
||||
|
||||
// TestWatchRouteSocketDropsRatherThanBlocks covers the coalescing send. A reader that is busy rebinding must not
|
||||
// wedge the watcher, and a second pending "the network moved" tells it nothing new anyway.
|
||||
func TestWatchRouteSocketDropsRatherThanBlocks(t *testing.T) {
|
||||
sock, kernel := socketPair(t)
|
||||
changes := make(chan struct{}, 1)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
watchRouteSocket(test.NewLogger(), sock, changes)
|
||||
close(done)
|
||||
}()
|
||||
|
||||
// Nobody is reading changes, so after the first report the buffer is full for the rest of this test
|
||||
for range 3 {
|
||||
_, err := unix.Write(kernel, routeMsg(unix.RTM_NEWADDR, 8))
|
||||
require.NoError(t, err)
|
||||
time.Sleep(netChangeSettleWindow + time.Millisecond*250)
|
||||
}
|
||||
|
||||
// The watcher must still be alive and responsive to a close
|
||||
require.NoError(t, sock.Close())
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second * 5):
|
||||
t.Fatal("watchRouteSocket wedged on a full channel")
|
||||
}
|
||||
|
||||
assert.Len(t, changes, 1, "the pending report should have coalesced, not queued")
|
||||
}
|
||||
|
||||
// TestWatchNetworkChangesStopsWithContext covers the detection path against a real routing socket, including that
|
||||
// cancelling the context closes the channel so a ranging caller falls out of its loop.
|
||||
func TestWatchNetworkChangesStopsWithContext(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
changes, err := watchNetworkChanges(ctx, test.NewLogger())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, changes, "darwin should support watching")
|
||||
|
||||
drained := make(chan struct{})
|
||||
go func() {
|
||||
for range changes {
|
||||
}
|
||||
close(drained)
|
||||
}()
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case <-drained:
|
||||
case <-time.After(time.Second * 5):
|
||||
t.Fatal("cancelling the context should close the changes channel")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNetworkChangeMonitorStopsWithContext drives the whole monitor against a real routing socket: Start must block
|
||||
// watching, and cancelling the context (which is all Control does on shutdown, it never stops the monitor directly)
|
||||
// must return it and clean up the watch goroutines.
|
||||
func TestNetworkChangeMonitorStopsWithContext(t *testing.T) {
|
||||
// IgnoreCurrent because other tests in this package leave readers running; we only care about what this test
|
||||
// leaks itself.
|
||||
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
|
||||
l := test.NewLogger()
|
||||
c := config.NewC(l)
|
||||
require.NoError(t, c.LoadString("listen:\n rebind_on_network_change: true\n"))
|
||||
m := NewNetworkChangeMonitor(ctx, l, c)
|
||||
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
m.Start(func() {})
|
||||
close(done)
|
||||
}()
|
||||
|
||||
// Start should be sitting on the routing socket, not have fallen out. If it returned early it either failed to
|
||||
// watch or no-op'd, both of which we want to catch.
|
||||
select {
|
||||
case <-done:
|
||||
t.Fatal("Start returned instead of watching")
|
||||
case <-time.After(time.Millisecond * 250):
|
||||
}
|
||||
|
||||
cancel()
|
||||
select {
|
||||
case <-done:
|
||||
case <-time.After(time.Second * 5):
|
||||
t.Fatal("Start did not return after the context was cancelled")
|
||||
}
|
||||
|
||||
// Starting again after the context is dead must not open anything.
|
||||
m.Start(func() {})
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
//go:build !darwin || ios || e2e_testing
|
||||
// +build !darwin ios e2e_testing
|
||||
|
||||
package udp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
// watchNetworkChanges is a no-op outside of darwin.
|
||||
//
|
||||
// Darwin is the platform that scopes a udp socket to the interface it came up on, so it is the platform whose socket
|
||||
// goes stale when the local network changes. Everywhere else Rebind has nothing to do, so there is nothing to watch
|
||||
// for. iOS is excluded on purpose even though it is darwin: the host app already drives the rebind off NWPathMonitor,
|
||||
// and two things racing to rebind the same socket is worse than one.
|
||||
//
|
||||
// A nil channel means "not supported here", which callers must treat as "do not start a watcher" rather than
|
||||
// selecting on it, since a receive from a nil channel blocks forever.
|
||||
func watchNetworkChanges(_ context.Context, _ *slog.Logger) (<-chan struct{}, error) {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package udp
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/test"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newMonitor(t *testing.T, ctx context.Context, cfg string) *NetworkChangeMonitor {
|
||||
t.Helper()
|
||||
l := test.NewLogger()
|
||||
c := config.NewC(l)
|
||||
require.NoError(t, c.LoadString(cfg))
|
||||
return NewNetworkChangeMonitor(ctx, l, c)
|
||||
}
|
||||
|
||||
func TestNetworkChangeMonitorDefaultsOn(t *testing.T) {
|
||||
// Says nothing about rebinding, so this covers the default.
|
||||
m := newMonitor(t, context.Background(), "listen:\n host: 0.0.0.0\n")
|
||||
assert.True(t, m.enabled, "should default to on")
|
||||
}
|
||||
|
||||
func TestNetworkChangeMonitorDisabledIsANoOp(t *testing.T) {
|
||||
m := newMonitor(t, context.Background(), "listen:\n rebind_on_network_change: false\n")
|
||||
require.False(t, m.enabled)
|
||||
|
||||
// Must return without opening a socket. If it watched anything this would block.
|
||||
m.Start(func() {})
|
||||
}
|
||||
|
||||
func TestNetworkChangeMonitorNilRebindIsANoOp(t *testing.T) {
|
||||
// Nothing to rebind, so there is no point watching, on any platform.
|
||||
m := newMonitor(t, context.Background(), "listen:\n rebind_on_network_change: true\n")
|
||||
m.Start(nil)
|
||||
}
|
||||
+4
-5
@@ -187,6 +187,9 @@ func (u *StdConn) SupportsMultipleReaders() bool {
|
||||
return false
|
||||
}
|
||||
|
||||
// Rebind clears the interface the kernel scoped this socket to, so that sends are routed against the current
|
||||
// routing table instead of the interface we happened to be on when the socket was created. Darwin pins sockets
|
||||
// this way on its own, which is what strands us after the underlying network changes.
|
||||
func (u *StdConn) Rebind() error {
|
||||
var err error
|
||||
if u.isV4 {
|
||||
@@ -195,9 +198,5 @@ func (u *StdConn) Rebind() error {
|
||||
err = syscall.SetsockoptInt(int(u.sysFd), syscall.IPPROTO_IPV6, syscall.IPV6_BOUND_IF, 0)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
u.l.Error("Failed to rebind udp socket", "error", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
return err
|
||||
}
|
||||
+18
-5
@@ -31,7 +31,8 @@ func procyield(cycles uint32)
|
||||
|
||||
const (
|
||||
packetsPerRing = 1024
|
||||
bytesPerPacket = 2048 - 32
|
||||
// Caps tun.mtu at MTU-32 direct, MTU-64 relayed, unenforced anywhere else. 17.6MB page locked per socket.
|
||||
bytesPerPacket = MTU
|
||||
receiveSpins = 15
|
||||
)
|
||||
|
||||
@@ -69,12 +70,14 @@ func NewRIOListener(l *slog.Logger, addr netip.Addr, port int) (*RIOConn, error)
|
||||
|
||||
err := u.bind(l, &windows.SockaddrInet6{Addr: addr.As16(), Port: port})
|
||||
if err != nil {
|
||||
u.close()
|
||||
return nil, fmt.Errorf("bind: %w", err)
|
||||
}
|
||||
|
||||
for i := 0; i < packetsPerRing; i++ {
|
||||
err = u.insertReceiveRequest()
|
||||
if err != nil {
|
||||
u.close()
|
||||
return nil, fmt.Errorf("init rx ring: %w", err)
|
||||
}
|
||||
}
|
||||
@@ -342,15 +345,25 @@ func (u *RIOConn) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
u.close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Also unwinds a partial build from NewRIOListener, where isOpen is false and Close would no-op.
|
||||
// Socket first, unlike wireguard-go: receive() re-arms every slot, so freeing the rings under a live socket
|
||||
// hands the kernel freed pages for all packetsPerRing outstanding receives.
|
||||
func (u *RIOConn) close() {
|
||||
// WSASocket reports failure as InvalidHandle, not zero.
|
||||
if u.sock != 0 && u.sock != windows.InvalidHandle {
|
||||
windows.CloseHandle(u.sock)
|
||||
}
|
||||
u.sock = 0
|
||||
|
||||
windows.PostQueuedCompletionStatus(u.rx.iocp, 0, 0, nil)
|
||||
windows.PostQueuedCompletionStatus(u.tx.iocp, 0, 0, nil)
|
||||
|
||||
u.rx.CloseAndZero()
|
||||
u.tx.CloseAndZero()
|
||||
if u.sock != 0 {
|
||||
windows.CloseHandle(u.sock)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (ring *ringBuffer) Push() *ringPacket {
|
||||
|
||||
+20
-6
@@ -10,6 +10,7 @@ import (
|
||||
"net/netip"
|
||||
"os"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/header"
|
||||
@@ -64,7 +65,9 @@ func acquirePacket() *Packet {
|
||||
}
|
||||
|
||||
type TesterConn struct {
|
||||
Addr netip.AddrPort
|
||||
// addr is read by nebula's own goroutines on every send and by the router's flow renderer, and a test can
|
||||
// move it mid-run to simulate roaming, so it is atomic rather than a plain field.
|
||||
addr atomic.Pointer[netip.AddrPort]
|
||||
|
||||
RxPackets chan *Packet // Packets to receive into nebula
|
||||
TxPackets chan *Packet // Packets transmitted outside by nebula
|
||||
@@ -82,13 +85,24 @@ type TesterConn struct {
|
||||
}
|
||||
|
||||
func NewListener(l *slog.Logger, ip netip.Addr, port int, _ bool, _ int) (Conn, error) {
|
||||
return &TesterConn{
|
||||
Addr: netip.AddrPortFrom(ip, uint16(port)),
|
||||
c := &TesterConn{
|
||||
RxPackets: make(chan *Packet, 10),
|
||||
TxPackets: make(chan *Packet, 10),
|
||||
done: make(chan struct{}),
|
||||
l: l,
|
||||
}, nil
|
||||
}
|
||||
c.SetAddr(netip.AddrPortFrom(ip, uint16(port)))
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// GetAddr returns the underlay address this conn currently sends from.
|
||||
func (u *TesterConn) GetAddr() netip.AddrPort {
|
||||
return *u.addr.Load()
|
||||
}
|
||||
|
||||
// SetAddr moves this conn to a new underlay address, standing in for a host waking up on a different network.
|
||||
func (u *TesterConn) SetAddr(addr netip.AddrPort) {
|
||||
u.addr.Store(&addr)
|
||||
}
|
||||
|
||||
// Send will place a UdpPacket onto the receive queue for nebula to consume
|
||||
@@ -147,7 +161,7 @@ func (u *TesterConn) WriteTo(b []byte, addr netip.AddrPort) error {
|
||||
p.Data = p.Data[:len(b)]
|
||||
}
|
||||
copy(p.Data, b)
|
||||
p.From = u.Addr
|
||||
p.From = u.GetAddr()
|
||||
p.To = addr
|
||||
select {
|
||||
case <-u.done:
|
||||
@@ -178,7 +192,7 @@ func NewUDPStatsEmitter(_ []Conn) func() {
|
||||
}
|
||||
|
||||
func (u *TesterConn) LocalAddr() (netip.AddrPort, error) {
|
||||
return u.Addr, nil
|
||||
return u.GetAddr(), nil
|
||||
}
|
||||
|
||||
func (u *TesterConn) SupportsMultipleReaders() bool {
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
//go:build darwin
|
||||
|
||||
package nebula
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// suspendClockDelta returns CLOCK_MONOTONIC - CLOCK_UPTIME_RAW. On macOS CLOCK_MONOTONIC keeps counting across
|
||||
// system sleep while CLOCK_UPTIME_RAW (mach_absolute_time) pauses, so the difference grows by time spent asleep.
|
||||
//
|
||||
// The pausing clock is read first so scheduling jitter between the two reads biases the delta positive; the
|
||||
// wakeDetector clamps out the noise.
|
||||
//
|
||||
// Caveat: on Apple Silicon the hardware timebase keeps ticking through sleep, which can make both clocks advance
|
||||
// and the spread stay flat, leaving this detector blind. That fails safe (no clears, behavior as before); IOKit
|
||||
// power notifications are the follow-up for full coverage on those machines.
|
||||
func suspendClockDelta() (time.Duration, bool) {
|
||||
var uptime, mono unix.Timespec
|
||||
if err := unix.ClockGettime(unix.CLOCK_UPTIME_RAW, &uptime); err != nil {
|
||||
return 0, false
|
||||
}
|
||||
if err := unix.ClockGettime(unix.CLOCK_MONOTONIC, &mono); err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return time.Duration(mono.Nano() - uptime.Nano()), true
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
//go:build !linux && !darwin && !windows
|
||||
|
||||
package nebula
|
||||
|
||||
import "time"
|
||||
|
||||
// suspendClockDelta reports that this platform has no usable clock pair for detecting system sleep; the wake
|
||||
// detector stays dormant and dead tunnels are left to the normal traffic checks.
|
||||
func suspendClockDelta() (time.Duration, bool) {
|
||||
return 0, false
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
//go:build linux
|
||||
|
||||
package nebula
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// suspendClockDelta returns CLOCK_BOOTTIME - CLOCK_MONOTONIC. CLOCK_MONOTONIC pauses while the system is suspended
|
||||
// and CLOCK_BOOTTIME does not, so the difference only ever grows, and only by time spent suspended. Both reads are
|
||||
// vDSO calls, cheap enough for a hot ticker.
|
||||
//
|
||||
// The pausing clock is read first so scheduling jitter between the two reads biases the delta positive; the
|
||||
// wakeDetector clamps out the noise.
|
||||
func suspendClockDelta() (time.Duration, bool) {
|
||||
var mono, boot unix.Timespec
|
||||
if err := unix.ClockGettime(unix.CLOCK_MONOTONIC, &mono); err != nil {
|
||||
return 0, false
|
||||
}
|
||||
if err := unix.ClockGettime(unix.CLOCK_BOOTTIME, &boot); err != nil {
|
||||
return 0, false
|
||||
}
|
||||
return time.Duration(boot.Nano() - mono.Nano()), true
|
||||
}
|
||||
@@ -1,41 +0,0 @@
|
||||
//go:build windows
|
||||
|
||||
package nebula
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
"unsafe"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
var (
|
||||
procQueryInterruptTime = windows.NewLazySystemDLL("kernelbase.dll").NewProc("QueryInterruptTime")
|
||||
procQueryUnbiasedInterruptTime = windows.NewLazySystemDLL("kernel32.dll").NewProc("QueryUnbiasedInterruptTime")
|
||||
|
||||
// QueryInterruptTime needs Windows 10; probe once and stay dormant on anything older.
|
||||
wakeClockAvailable = sync.OnceValue(func() bool {
|
||||
return procQueryInterruptTime.Find() == nil && procQueryUnbiasedInterruptTime.Find() == nil
|
||||
})
|
||||
)
|
||||
|
||||
// suspendClockDelta returns interrupt time minus unbiased interrupt time, both in 100ns units. The unbiased count
|
||||
// excludes time the system spends suspended while the biased one includes it, so the difference grows by exactly
|
||||
// the time spent asleep.
|
||||
//
|
||||
// The pausing (unbiased) clock is read first so scheduling jitter between the two reads biases the delta positive;
|
||||
// the wakeDetector clamps out the noise.
|
||||
func suspendClockDelta() (time.Duration, bool) {
|
||||
if !wakeClockAvailable() {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
var unbiased, biased uint64
|
||||
if r1, _, _ := procQueryUnbiasedInterruptTime.Call(uintptr(unsafe.Pointer(&unbiased))); r1 == 0 {
|
||||
return 0, false
|
||||
}
|
||||
// Returns void, cannot fail once resolved.
|
||||
_, _, _ = procQueryInterruptTime.Call(uintptr(unsafe.Pointer(&biased)))
|
||||
return time.Duration(int64(biased-unbiased)) * 100, true
|
||||
}
|
||||
@@ -1,47 +0,0 @@
|
||||
package nebula
|
||||
|
||||
import "time"
|
||||
|
||||
// wakeDetector notices when the machine has returned from system sleep and measures how long it was suspended.
|
||||
//
|
||||
// It samples the spread between two kernel clocks: one that pauses across a suspend and one that keeps counting
|
||||
// (suspendClockDelta, per platform). While the machine is awake the spread is constant no matter how starved,
|
||||
// stopped, or stepped this process is — SIGSTOP, debugger pauses, scheduler starvation, and NTP adjustments move
|
||||
// both clocks together or neither, so none of them can fake a wake. A true suspend is the only thing that grows
|
||||
// the spread, and it grows by exactly the time spent suspended.
|
||||
//
|
||||
// Sample is intended to piggyback on a ticker the caller already runs; it costs two clock reads. It is not safe
|
||||
// for concurrent use.
|
||||
type wakeDetector struct {
|
||||
// read returns the current spread between the two clocks, false if this platform can't provide one.
|
||||
read func() (time.Duration, bool)
|
||||
last time.Duration
|
||||
primed bool
|
||||
}
|
||||
|
||||
func newWakeDetector() *wakeDetector {
|
||||
return &wakeDetector{read: suspendClockDelta}
|
||||
}
|
||||
|
||||
// Sample returns how long the machine was suspended since the previous call, 0 if it wasn't, and false if the
|
||||
// platform has no way to tell. The first call primes the baseline and always reports 0.
|
||||
func (w *wakeDetector) Sample() (time.Duration, bool) {
|
||||
delta, ok := w.read()
|
||||
if !ok {
|
||||
return 0, false
|
||||
}
|
||||
|
||||
if !w.primed {
|
||||
w.primed = true
|
||||
w.last = delta
|
||||
return 0, true
|
||||
}
|
||||
|
||||
slept := delta - w.last
|
||||
w.last = delta
|
||||
if slept < 0 {
|
||||
// The clock pair is read non-atomically so tiny negative jitter is possible; it is never a wake.
|
||||
slept = 0
|
||||
}
|
||||
return slept, true
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
package nebula
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestWakeDetector(t *testing.T) {
|
||||
delta := time.Duration(0)
|
||||
ok := true
|
||||
w := &wakeDetector{read: func() (time.Duration, bool) { return delta, ok }}
|
||||
|
||||
// The first sample primes the baseline and never reports a wake, even with a pre-existing spread
|
||||
delta = 3 * time.Hour
|
||||
slept, sok := w.Sample()
|
||||
assert.True(t, sok)
|
||||
assert.Equal(t, time.Duration(0), slept)
|
||||
|
||||
// A stable spread means the machine never slept
|
||||
slept, sok = w.Sample()
|
||||
assert.True(t, sok)
|
||||
assert.Equal(t, time.Duration(0), slept)
|
||||
|
||||
// The spread grows by exactly the time spent suspended
|
||||
delta += 42 * time.Second
|
||||
slept, sok = w.Sample()
|
||||
assert.True(t, sok)
|
||||
assert.Equal(t, 42*time.Second, slept)
|
||||
|
||||
// A wake is reported once, then the baseline moves with it
|
||||
slept, sok = w.Sample()
|
||||
assert.True(t, sok)
|
||||
assert.Equal(t, time.Duration(0), slept)
|
||||
|
||||
// Negative jitter from the non-atomic clock pair reads clamps to zero
|
||||
delta -= time.Microsecond
|
||||
slept, sok = w.Sample()
|
||||
assert.True(t, sok)
|
||||
assert.Equal(t, time.Duration(0), slept)
|
||||
|
||||
// Consecutive suspends both report; the clamped jitter moved the baseline so it is not double-counted
|
||||
delta += time.Minute
|
||||
slept, _ = w.Sample()
|
||||
assert.Equal(t, time.Minute, slept)
|
||||
delta += time.Hour
|
||||
slept, _ = w.Sample()
|
||||
assert.Equal(t, time.Hour, slept)
|
||||
|
||||
// An unsupported platform read reports not-ok
|
||||
ok = false
|
||||
_, sok = w.Sample()
|
||||
assert.False(t, sok)
|
||||
}
|
||||
|
||||
// TestWakeDetectorPlatformClock smoke tests the real clock pair: two samples close together must not report a
|
||||
// wake on a machine that isn't suspending mid-test.
|
||||
func TestWakeDetectorPlatformClock(t *testing.T) {
|
||||
w := newWakeDetector()
|
||||
if _, ok := w.Sample(); !ok {
|
||||
t.Skip("platform has no suspend clock pair")
|
||||
}
|
||||
slept, ok := w.Sample()
|
||||
assert.True(t, ok)
|
||||
assert.Less(t, slept, time.Second)
|
||||
}
|
||||
Reference in new issue
Block a user