Compare commits

..
Author SHA1 Message Date
JackDoan 71c5297134 improve lighthouse addrMap correctness for multi-address certs 2026-09-02 14:37:27 -05:00
144 changed files with 771 additions and 7595 deletions

No files matched your search

-142
View File
@@ -1,142 +0,0 @@
#!/usr/bin/env bash
#
# Backport a merged PR to a release branch.
#
# Cherry-picks the merge commit of a merged PR onto release-1.11 and opens a
# backport PR, mirroring the shape of #1842.
#
# Usage: ./backport.sh <pr-number> [target-version] [--continue]
# pr-number The merged PR to backport
# target-version Release version to target (default: 1.11)
# --continue Resume after resolving cherry-pick conflicts: assumes the
# fixes are staged, runs `git cherry-pick --continue`, and
# proceeds to push and open the backport PR.
#
# Requires: gh (authenticated), git, jq.
set -euo pipefail
PR_NUMBER=${1:?usage: backport.sh <pr-number> [target-version] [--continue]}
shift
CONTINUE=0
TARGET_VERSION=""
for arg in "$@"; do
case "$arg" in
--continue) CONTINUE=1 ;;
*) TARGET_VERSION=$arg ;;
esac
done
for cmd in gh git jq; do
command -v "$cmd" >/dev/null || { echo "error: $cmd is required" >&2; exit 1; }
done
# Pull the source PR's metadata. Refuse to backport a PR that never merged.
pr_json=$(gh pr view "$PR_NUMBER" --json title,body,mergedAt,mergeCommit,milestone)
PR_TITLE=$(jq -r '.title' <<<"$pr_json")
PR_BODY=$(jq -r '.body' <<<"$pr_json")
MERGE_SHA=$(jq -r '.mergeCommit.oid // empty' <<<"$pr_json")
PR_MILESTONE=$(jq -r '.milestone.title // empty' <<<"$pr_json")
if [ "$(jq -r '.mergedAt // empty' <<<"$pr_json")" = "" ] || [ -z "$MERGE_SHA" ]; then
echo "error: PR #${PR_NUMBER} is not merged (no merge commit to cherry-pick)" >&2
exit 1
fi
# Default the target to one minor below the PR's milestone: a PR landing in
# v1.12.0 backports to release-1.11. An explicit target-version arg overrides.
if [ -z "$TARGET_VERSION" ]; then
if [[ $PR_MILESTONE =~ ^v?([0-9]+)\.([0-9]+) ]]; then
TARGET_VERSION="${BASH_REMATCH[1]}.$(( BASH_REMATCH[2] - 1 ))"
else
echo "error: PR #${PR_NUMBER} has no v<major>.<minor>.* milestone to infer the target from." >&2
echo "Pass the target version explicitly, e.g. $0 ${PR_NUMBER} 1.11" >&2
exit 1
fi
fi
TARGET_BRANCH="release-${TARGET_VERSION}"
# Slug from the PR title: lower-case, non-alphanumerics to dashes, trimmed.
slug=$(printf '%s' "$PR_TITLE" \
| tr '[:upper:]' '[:lower:]' \
| sed -E 's/[^a-z0-9]+/-/g; s/^-+//; s/-+$//' \
| cut -c1-50 \
| sed -E 's/-+$//')
BRANCH="backport-${TARGET_VERSION//./-}-${slug}"
if [ "$CONTINUE" -eq 1 ]; then
# Resuming: the branch already exists and a cherry-pick is mid-conflict.
current=$(git rev-parse --abbrev-ref HEAD)
if [ "$current" != "$BRANCH" ]; then
echo "error: --continue expects to be on ${BRANCH}, but HEAD is ${current}" >&2
exit 1
fi
echo "Resuming backport of #${PR_NUMBER} on ${BRANCH}"
# Conflicts assumed resolved and staged; core.editor=true keeps the commit message.
git -c core.editor=true cherry-pick --continue
else
# A fresh run switches branches and cherry-picks, so tracked changes must be
# clean. Untracked files are fine, and --continue is exempt (its resolved
# conflicts are meant to be staged).
if [ -n "$(git status --porcelain --untracked-files=no)" ]; then
echo "error: working tree has uncommitted changes to tracked files; commit or stash them first." >&2
exit 1
fi
echo "Backporting #${PR_NUMBER} (${MERGE_SHA}) onto ${TARGET_BRANCH} as ${BRANCH}"
git fetch origin
git checkout -b "$BRANCH" "origin/${TARGET_BRANCH}"
# -m 1 handles a real merge commit; plain cherry-pick handles a squash merge.
if ! { git cherry-pick -x -m 1 "$MERGE_SHA" 2>/dev/null || git cherry-pick -x "$MERGE_SHA"; }; then
echo "error: ${MERGE_SHA} did not cherry-pick cleanly onto ${TARGET_BRANCH}." >&2
echo "Resolve the conflicts, 'git add' them, then re-run:" >&2
echo " $0 ${PR_NUMBER} ${TARGET_VERSION} --continue" >&2
exit 1
fi
fi
title="backport v${TARGET_VERSION}: ${PR_TITLE}"
body=$(printf 'Backport from #%s to release v%s\n\n---\n\n%s' \
"$PR_NUMBER" "$TARGET_VERSION" "$PR_BODY")
# Highest open milestone matching v<version>.* (e.g. v1.11.1), if any.
MILESTONE=$(gh api "repos/{owner}/{repo}/milestones?state=open" \
--jq ".[].title | select(startswith(\"v${TARGET_VERSION}.\"))" \
| sort -V | tail -n1)
milestone_args=()
if [ -n "$MILESTONE" ]; then
milestone_args=(--milestone "$MILESTONE")
else
echo "warning: no open milestone matching v${TARGET_VERSION}.* found; PR will have no milestone" >&2
fi
echo
echo "About to run:"
printf ' git push -u --force-with-lease origin %q\n' "$BRANCH"
printf ' gh pr create --base %q --head %q --title %q --body %q' \
"$TARGET_BRANCH" "$BRANCH" "$title" "$body"
[ -n "$MILESTONE" ] && printf ' --milestone %q' "$MILESTONE"
printf '\n'
echo
read -r -p "Open this pull request? [y/N] " reply
case "$reply" in
[yY] | [yY][eE][sS]) ;;
*) echo "Aborted. The cherry-pick is on local branch ${BRANCH}; push it manually if desired."; exit 0 ;;
esac
git push -u --force-with-lease origin "$BRANCH"
gh pr create \
--base "$TARGET_BRANCH" \
--head "$BRANCH" \
--title "$title" \
--body "$body" \
${milestone_args[@]+"${milestone_args[@]}"}
# The backport now exists, so drop the label that flagged this PR for one.
gh pr edit "$PR_NUMBER" --remove-label needs-backport
# Switch back to the original branch we were on before doing the backport
git checkout -
+1 -1
View File
@@ -98,7 +98,7 @@ jobs:
# WSL2 + Ubuntu so the smoke can run a real linux peer with its own
# netns. iputils-ping is needed for the in-WSL ping check. WSL1 has no
# real kernel and would lack /dev/net/tun, so we have to force WSL2.
- uses: Vampire/setup-wsl@v7
- uses: Vampire/setup-wsl@v3
with:
distribution: Ubuntu-24.04
additional-packages: iputils-ping iproute2
+5 -48
View File
@@ -51,19 +51,15 @@ wsl -d $Distro -- bash -c "rm -rf $WslDir && mkdir -p $WslDir" | Out-Null
$DevName = 'nebula-smoke'
$Ip1 = '192.168.241.1'
$Ip2 = '192.168.241.2'
# Dual stack on purpose: a v4-only overlay never exercises the v6 side of tun.mtu.
$Ip6_1 = 'fd42:4242:241::1'
$Ip6_2 = 'fd42:4242:241::2'
$Mtu = 1300
$Port = 4242
& $NebulaCert ca -name 'smoke-ca' -out-crt "$WorkDir\ca.crt" -out-key "$WorkDir\ca.key"
if ($LASTEXITCODE -ne 0) { throw "nebula-cert ca failed (exit $LASTEXITCODE)" }
& $NebulaCert sign -name 'lighthouse' -networks "$Ip1/24,$Ip6_1/64" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\lighthouse.crt" -out-key "$WorkDir\lighthouse.key"
& $NebulaCert sign -name 'lighthouse' -networks "$Ip1/24" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\lighthouse.crt" -out-key "$WorkDir\lighthouse.key"
if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign lighthouse failed (exit $LASTEXITCODE)" }
& $NebulaCert sign -name 'peer' -networks "$Ip2/24,$Ip6_2/64" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\peer.crt" -out-key "$WorkDir\peer.key"
& $NebulaCert sign -name 'peer' -networks "$Ip2/24" -ca-crt "$WorkDir\ca.crt" -ca-key "$WorkDir\ca.key" -out-crt "$WorkDir\peer.crt" -out-key "$WorkDir\peer.key"
if ($LASTEXITCODE -ne 0) { throw "nebula-cert sign peer failed (exit $LASTEXITCODE)" }
# Windows lighthouse config.
@@ -86,7 +82,7 @@ tun:
drop_local_broadcast: false
drop_multicast: false
tx_queue: 500
mtu: $Mtu
mtu: 1300
network_category: private
logging:
level: info
@@ -130,7 +126,7 @@ tun:
drop_local_broadcast: false
drop_multicast: false
tx_queue: 500
mtu: $Mtu
mtu: 1300
logging:
level: info
format: text
@@ -173,7 +169,7 @@ Write-Host '=== WSL diagnostic ==='
wsl --version 2>&1 | Out-Host
wsl --list --verbose 2>&1 | Out-Host
wsl -d $Distro -u root -- uname -a | Out-Host
wsl -d $Distro -u root -- bash -c "modprobe tun 2>&1 || true; mkdir -p /dev/net; [ -c /dev/net/tun ] || mknod /dev/net/tun c 10 200; chmod 600 /dev/net/tun; { echo 0 > /proc/sys/net/ipv6/conf/all/disable_ipv6; echo 0 > /proc/sys/net/ipv6/conf/default/disable_ipv6; } 2>/dev/null || true; ls -l /dev/net/tun"
wsl -d $Distro -u root -- bash -c "modprobe tun 2>&1 || true; mkdir -p /dev/net; [ -c /dev/net/tun ] || mknod /dev/net/tun c 10 200; chmod 600 /dev/net/tun; ls -l /dev/net/tun"
if ($LASTEXITCODE -ne 0) { throw "failed to prepare /dev/net/tun in WSL (TUN support missing?)" }
# Deliberately no New-NetFirewallRule calls here -- nebula's windows_bypass_wdf
@@ -218,16 +214,6 @@ try {
}
Write-Host "OK: $DevName NetworkCategory=Private"
# v6 silently kept the adapter default of 65535 while v4 was correct.
foreach ($family in @('IPv4', 'IPv6')) {
Wait-Until -TimeoutSec 30 -What "$DevName $family NlMtu=$Mtu" -Predicate {
if ($lhProc.HasExited) { throw "lighthouse exited (code $($lhProc.ExitCode)) before $family mtu was set" }
$rows = @(Get-NetIPInterface -InterfaceAlias $DevName -AddressFamily $family -ErrorAction SilentlyContinue)
$rows.Count -gt 0 -and -not ($rows | Where-Object { $_.NlMtu -ne $Mtu })
}
Write-Host "OK: $DevName $family NlMtu=$Mtu"
}
Wait-Until -TimeoutSec 30 -What "WSL nebula1 with $Ip2" -Predicate {
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before tun was ready" }
$r = wsl -d $Distro -u root -- bash -c "ip -o addr show nebula1 2>/dev/null | grep -q 'inet $Ip2' && echo yes"
@@ -235,13 +221,6 @@ try {
}
Write-Host "OK: WSL nebula1 has $Ip2"
Wait-Until -TimeoutSec 30 -What "WSL nebula1 with $Ip6_2" -Predicate {
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the v6 address was up" }
$r = wsl -d $Distro -u root -- bash -c "ip -o addr show nebula1 2>/dev/null | grep -q 'inet6 $Ip6_2' && echo yes"
("$r").Trim() -eq 'yes'
}
Write-Host "OK: WSL nebula1 has $Ip6_2"
Wait-Until -TimeoutSec 30 -What "ping from WSL peer to windows lighthouse ($Ip1)" -Predicate {
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before ping succeeded" }
$r = wsl -d $Distro -u root -- bash -c "ping -c1 -W1 $Ip1 >/dev/null 2>&1 && echo OK"
@@ -255,28 +234,6 @@ try {
}
Write-Host "OK: windows lighthouse -> WSL peer"
# Otherwise the v6 networks only prove the interface exists, not that it forwards.
Wait-Until -TimeoutSec 30 -What "v6 ping from WSL peer to windows lighthouse ($Ip6_1)" -Predicate {
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the v6 ping succeeded" }
$r = wsl -d $Distro -u root -- bash -c "ping -6 -c1 -W1 $Ip6_1 >/dev/null 2>&1 && echo OK"
("$r").Trim() -eq 'OK'
}
Write-Host "OK: WSL peer -> windows lighthouse over v6"
# The NlMtu check above proves the value was written, not that windows honors it. A payload this size only
# crosses the tunnel as fragments cut at NlMtu on the way out of the adapter. Left at the adapter default it
# reaches nebula whole, overflows the udp.MTU (9001) write buffer and is dropped, so the ping never answers.
# Anything smaller would ride out as one oversized datagram and survive on IP fragmentation of the underlay.
$BigPayload = 9000
foreach ($target in @(@{ Family = 'v4'; Ip = $Ip2 }, @{ Family = 'v6'; Ip = $Ip6_2 })) {
Wait-Until -TimeoutSec 30 -What "$($target.Family) ping with a $BigPayload byte payload from windows lighthouse to WSL peer ($($target.Ip))" -Predicate {
if ($peerProc.HasExited) { throw "peer exited (code $($peerProc.ExitCode)) before the large $($target.Family) ping succeeded" }
$null = & ping.exe -n 1 -w 1000 -l $BigPayload $target.Ip
$LASTEXITCODE -eq 0
}
Write-Host "OK: windows lighthouse -> WSL peer, $BigPayload byte $($target.Family) payload"
}
Write-Host ''
Write-Host 'All smoke checks passed.'
}
+1 -21
View File
@@ -7,17 +7,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [Unreleased]
### Fixed
- `create-tunnel -address <ip:port>` panicked on a nil RemoteList and, over the ssh debug server, took nebula
down with it. A hostinfo returned by StartHandshake had no RemoteList until the handshake loop's first
pass attached one, and `create-tunnel -address` set the remote before that. The list is now attached when
the handshake starts, so a pending hostinfo never exists without one.
## [1.11.1] - 2026-08-21
See the [v1.11.1](https://github.com/slackhq/nebula/milestone/30?closed=1) milestone for a complete list of changes.
### Changed
- IPv6 packets whose next header is a protocol Nebula does not parse (SCTP, GRE, IP-in-IP, etc.) are now
@@ -26,18 +15,11 @@ See the [v1.11.1](https://github.com/slackhq/nebula/milestone/30?closed=1) miles
their true protocol, so only a `proto: any` rule allows them. If you carry one of these protocols over the
overlay, confirm a `proto: any` rule covers it before upgrading, it may have been passing only through this
bypass. (#1840)
- Drop the dependency on `github.com/cyberdelia/go-metrics-graphite`, which has been unmaintained for over ten
years, by inlining the small amount of code Nebula used. (#1832)
### Fixed
- The ICMPv6 type was read from the wrong byte when classifying IPv6 packets, so the echo identifier used
for conntrack was never picked up. (#1840)
- Enforce outbound message counter limits so a tunnel is rehandshaked before the counter can wrap, preventing
nonce reuse. This is unreachable in practice, but is enforced as a defense-in-depth measure. (#1841)
- Prevent `nebula-cert ca` from running out of memory on 32bit systems when generating encrypted private keys. (#1834)
- Tolerate `ErrDumpInterrupted` when listing tun addresses on Linux, so a transient interrupted netlink dump
no longer aborts startup. (#1835)
## [1.11.0] - 2026-07-23
@@ -902,9 +884,7 @@ created.)
- Initial public release.
[Unreleased]: https://github.com/slackhq/nebula/compare/v1.11.1...HEAD
[1.11.1]: https://github.com/slackhq/nebula/releases/tag/v1.11.1
[1.11.0]: https://github.com/slackhq/nebula/releases/tag/v1.11.0
[Unreleased]: https://github.com/slackhq/nebula/compare/v1.10.3...HEAD
[1.10.3]: https://github.com/slackhq/nebula/releases/tag/v1.10.3
[1.10.2]: https://github.com/slackhq/nebula/releases/tag/v1.10.2
[1.10.1]: https://github.com/slackhq/nebula/releases/tag/v1.10.1
-3
View File
@@ -242,14 +242,11 @@ test-cov-html:
go test -coverprofile=coverage.out
go tool cover -html=coverage.out
# The package builds only compile. The final line links an android binary so a linker-only failure
# cannot pass CI.
build-test-mobile:
GOARCH=amd64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=amd64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
GOARCH=arm64 GOOS=android go build -o /dev/null ${NEBULA_CMD_PATH}
bench:
go test -bench=.
+11 -2
View File
@@ -79,7 +79,13 @@ func (b *Bits) clearRange(startPos, count uint64) uint64 {
// handle the potential partial word before pos becomes u64 aligned
word := pos >> 6
bit := pos & 63
take := min(min(uint64(64)-bit, remaining), b.length-pos)
take := uint64(64) - bit
if take > remaining {
take = remaining
}
if take > b.length-pos {
take = b.length - pos
}
var mask uint64
if take == 64 {
mask = math.MaxUint64
@@ -183,7 +189,10 @@ func (b *Bits) Update(l *slog.Logger, i uint64) bool {
func (b *Bits) updateSlow(l *slog.Logger, i uint64) bool {
// If i is a jump, adjust the window, record lost, update current, and return true
if i > b.current {
end := min(i, b.current+b.length)
end := i
if end > b.current+b.length {
end = b.current + b.length
}
count := end - b.current
startPos := (b.current + 1) & b.lengthMask
+1 -1
View File
@@ -120,7 +120,7 @@ func TestCertificate_SignP256_AlwaysNormalized(t *testing.T) {
pub := elliptic.Marshal(elliptic.P256(), priv.PublicKey.X, priv.PublicKey.Y)
rawPriv := priv.D.FillBytes(make([]byte, 32))
for i := range 1000 {
for i := 0; i < 1000; i++ {
if i&1 == 1 {
tbs.Version = Version1
} else {
+4 -4
View File
@@ -151,7 +151,7 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error
var groups []string
if *cf.groups != "" {
for rg := range strings.SplitSeq(*cf.groups, ",") {
for _, rg := range strings.Split(*cf.groups, ",") {
g := strings.TrimSpace(rg)
if g != "" {
groups = append(groups, g)
@@ -171,7 +171,7 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error
}
if *cf.networks != "" {
for rs := range strings.SplitSeq(*cf.networks, ",") {
for _, rs := range strings.Split(*cf.networks, ",") {
rs := strings.Trim(rs, " ")
if rs != "" {
n, err := netip.ParsePrefix(rs)
@@ -193,7 +193,7 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error
}
if *cf.unsafeNetworks != "" {
for rs := range strings.SplitSeq(*cf.unsafeNetworks, ",") {
for _, rs := range strings.Split(*cf.unsafeNetworks, ",") {
rs := strings.Trim(rs, " ")
if rs != "" {
n, err := netip.ParsePrefix(rs)
@@ -221,7 +221,7 @@ func ca(args []string, out io.Writer, errOut io.Writer, pr PasswordReader) error
if !isP11 && *cf.encryption {
passphrase = []byte(os.Getenv("NEBULA_CA_PASSPHRASE"))
if len(passphrase) == 0 {
for range 5 {
for i := 0; i < 5; i++ {
errOut.Write([]byte("Enter passphrase: "))
passphrase, err = pr.ReadPassword()
+1
View File
@@ -1,4 +1,5 @@
//go:build !windows
// +build !windows
package main
+4 -4
View File
@@ -146,7 +146,7 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
passphrase = []byte(os.Getenv("NEBULA_CA_PASSPHRASE"))
if len(passphrase) == 0 {
// ask for a passphrase until we get one
for range 5 {
for i := 0; i < 5; i++ {
errOut.Write([]byte("Enter passphrase: "))
passphrase, err = pr.ReadPassword()
@@ -203,7 +203,7 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
}
if *sf.networks != "" {
for rs := range strings.SplitSeq(*sf.networks, ",") {
for _, rs := range strings.Split(*sf.networks, ",") {
rs := strings.Trim(rs, " ")
if rs != "" {
n, err := netip.ParsePrefix(rs)
@@ -228,7 +228,7 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
}
if *sf.unsafeNetworks != "" {
for rs := range strings.SplitSeq(*sf.unsafeNetworks, ",") {
for _, rs := range strings.Split(*sf.unsafeNetworks, ",") {
rs := strings.Trim(rs, " ")
if rs != "" {
n, err := netip.ParsePrefix(rs)
@@ -247,7 +247,7 @@ func signCert(args []string, out io.Writer, errOut io.Writer, pr PasswordReader)
var groups []string
if *sf.groups != "" {
for rg := range strings.SplitSeq(*sf.groups, ",") {
for _, rg := range strings.Split(*sf.groups, ",") {
g := strings.TrimSpace(rg)
if g != "" {
groups = append(groups, g)
+1
View File
@@ -1,4 +1,5 @@
//go:build !windows
// +build !windows
package main
+1
View File
@@ -1,4 +1,5 @@
//go:build !windows
// +build !windows
package main
+1
View File
@@ -1,4 +1,5 @@
//go:build !linux
// +build !linux
package main
+9 -6
View File
@@ -5,7 +5,6 @@ import (
"errors"
"fmt"
"log/slog"
"maps"
"math"
"os"
"os/signal"
@@ -155,7 +154,9 @@ func (c *C) ReloadConfig() {
defer c.reloadLock.Unlock()
c.oldSettings = make(map[string]any)
maps.Copy(c.oldSettings, c.Settings)
for k, v := range c.Settings {
c.oldSettings[k] = v
}
err := c.Load(c.path)
if err != nil {
@@ -176,7 +177,9 @@ func (c *C) ReloadConfigString(raw string) error {
defer c.reloadLock.Unlock()
c.oldSettings = make(map[string]any)
maps.Copy(c.oldSettings, c.Settings)
for k, v := range c.Settings {
c.oldSettings[k] = v
}
err := c.LoadString(raw)
if err != nil {
@@ -213,7 +216,7 @@ func (c *C) GetStringSlice(k string, d []string) []string {
}
v := make([]string, len(rv))
for i := range v {
for i := 0; i < len(v); i++ {
v[i] = fmt.Sprintf("%v", rv[i])
}
@@ -307,8 +310,8 @@ func (c *C) IsSet(k string) bool {
}
func (c *C) get(k string, v any) any {
parts := strings.SplitSeq(k, ".")
for p := range parts {
parts := strings.Split(k, ".")
for _, p := range parts {
m, ok := v.(map[string]any)
if !ok {
return nil
+4 -2
View File
@@ -174,8 +174,10 @@ func (cm *connectionManager) doTrafficCheck(localIndex uint32, p, nb, out []byte
switch decision {
case deleteTunnel:
// Local delete only, clearing the lighthouse cache if this was the last hostinfo for the vpn addr
cm.intf.closeTunnel(hostinfo)
if cm.hostMap.DeleteHostInfo(hostinfo) {
// Only clearing the lighthouse cache if this is the last hostinfo for this vpn ip in the hostmap
cm.intf.lightHouse.DeleteVpnAddrs(hostinfo.vpnAddrs)
}
case closeTunnel:
cm.intf.sendCloseTunnel(hostinfo)
-3
View File
@@ -7,7 +7,6 @@ import (
"testing"
"time"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/overlay/overlaytest"
@@ -22,8 +21,6 @@ func newTestLighthouse() *LightHouse {
l: test.NewLogger(),
addrMap: map[netip.Addr]*RemoteList{},
queryChan: make(chan netip.Addr, 10),
// Nothing drains queryChan here, so an 11th query is dropped.
queryDropped: metrics.NilCounter{},
}
lighthouses := []netip.Addr{}
staticList := map[netip.Addr]struct{}{}
+1 -120
View File
@@ -2,8 +2,6 @@ package nebula
import (
"net/netip"
"strconv"
"sync"
"testing"
"time"
@@ -13,7 +11,6 @@ import (
ct "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/handshake"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/overlay"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
@@ -100,7 +97,7 @@ func TestConnectionState_NextMessageCounter(t *testing.T) {
assert.Equal(t, RejectAfterMessages, cs.messageCounter.Load())
// Continued send attempts stay refused and the counter never wraps
for range 10 {
for i := 0; i < 10; i++ {
_, ok = cs.NextMessageCounter()
assert.False(t, ok)
}
@@ -150,122 +147,6 @@ func TestSendNoMetricsCloseTunnelKeepsRebindEpoch(t *testing.T) {
assert.True(t, hostinfo.markOut(1), "a CloseTunnel send must not consume the rebind epoch")
}
// capturingConn is a udp.Conn that records every WriteTo.
type capturingConn struct {
udp.NoopConn
writes [][]byte
addrs []netip.AddrPort
}
func (c *capturingConn) WriteTo(b []byte, addr netip.AddrPort) error {
c.writes = append(c.writes, append([]byte(nil), b...))
c.addrs = append(c.addrs, addr)
return nil
}
// TestSendNoMetricsViaRelay sends through a relay from the mtu sized buffer the cached packet flush uses. Payloads up
// to overlay.MaxMTU go out intact, anything bigger is dropped rather than outgrowing the buffer, which once panicked.
func TestSendNoMetricsViaRelay(t *testing.T) {
tests := []struct {
n int
sent bool
}{
{0, true},
{1300, true},
{overlay.MaxMTU, true},
{overlay.MaxMTU + 1, false}, // too big for the relay's tag
{mtu - 47, false}, // too big for the inner tag, this panicked
{9000, false}, // a full packet at tun.mtu 9000
}
for _, tt := range tests {
t.Run(strconv.Itoa(tt.n), func(t *testing.T) {
// One tunnel to the target, carried end to end, and one to the relay that wraps it
targetInit, targetResp := runTestHandshake(t)
relayInit, relayResp := runTestHandshake(t)
ci, err := newConnectionStateFromResult(targetInit)
require.NoError(t, err)
relayCI, err := newConnectionStateFromResult(relayInit)
require.NoError(t, err)
targetAddr := netip.MustParseAddr("10.0.0.2")
relayAddr := netip.MustParseAddr("10.0.0.3")
relayRemote := netip.MustParseAddrPort("192.0.2.1:4242")
relay := &Relay{Type: TerminalType, State: Established, LocalIndex: 5, RemoteIndex: 6, PeerAddr: targetAddr}
relayHI := &HostInfo{
vpnAddrs: []netip.Addr{relayAddr},
ConnectionState: relayCI,
relayState: RelayState{
relayForByAddr: map[netip.Addr]*Relay{targetAddr: relay},
relayForByIdx: map[uint32]*Relay{relay.LocalIndex: relay},
},
}
relayHI.remote.Store(&relayRemote)
// No remote of its own, so the send has to go via the relay
hostinfo := &HostInfo{
vpnAddrs: []netip.Addr{targetAddr},
ConnectionState: ci,
remoteIndexId: 7,
relayState: RelayState{
relays: []netip.Addr{relayAddr},
relayForByAddr: map[netip.Addr]*Relay{},
relayForByIdx: map[uint32]*Relay{},
},
}
l := test.NewLogger()
hm := newHostMap(l)
hm.Hosts[relayAddr] = relayHI
conn := &capturingConn{}
f := &Interface{
l: l,
hostMap: hm,
messageMetrics: &MessageMetrics{txExhausted: metrics.NewCounter()},
writers: []udp.Conn{conn},
}
f.connectionManager = &connectionManager{intf: f, relayUsed: map[uint32]struct{}{}, relayUsedLock: &sync.RWMutex{}}
payload := make([]byte, tt.n)
for i := range payload {
payload[i] = byte(i)
}
counter := ci.messageCounter.Load()
f.sendNoMetrics(header.Message, 0, ci, hostinfo, netip.AddrPort{}, payload, make([]byte, 12), make([]byte, mtu), 0)
if !tt.sent {
assert.Empty(t, conn.writes)
assert.Equal(t, counter, ci.messageCounter.Load(), "a dropped packet must not spend a counter")
return
}
require.Len(t, conn.writes, 1)
assert.Equal(t, relayRemote, conn.addrs[0])
pkt := conn.writes[0]
// The relay authenticates the whole packet under its own header
outer := &header.H{}
require.NoError(t, outer.Parse(pkt))
assert.Equal(t, header.MessageRelay, outer.Subtype)
assert.Equal(t, relay.RemoteIndex, outer.RemoteIndex)
relayPeer, err := newConnectionStateFromResult(relayResp)
require.NoError(t, err)
require.NoError(t, relayPeer.VerifyRelay(l, outer.MessageCounter, pkt, make([]byte, 12)))
// Inside it, the target decrypts the end to end packet back to the payload
inner := pkt[header.Len : len(pkt)-relayPeer.dKey.Overhead()]
h := &header.H{}
require.NoError(t, h.Parse(inner))
assert.Equal(t, header.Message, h.Type)
assert.Equal(t, hostinfo.remoteIndexId, h.RemoteIndex)
targetPeer, err := newConnectionStateFromResult(targetResp)
require.NoError(t, err)
got, err := targetPeer.Decrypt(l, h.MessageCounter, inner, make([]byte, 12))
require.NoError(t, err)
assert.Equal(t, payload, got)
})
}
}
func TestNewConnectionStateFromResult(t *testing.T) {
initR, respR := runTestHandshake(t)
+1 -1
View File
@@ -247,7 +247,7 @@ func TestControl_ConcurrentStopAndStart(t *testing.T) {
c, _, _ := newReadyControl(t)
var wg sync.WaitGroup
for range 2 {
for i := 0; i < 2; i++ {
wg.Go(func() { c.Stop() })
}
wg.Go(func() { _ = c.Start() })
-131
View File
@@ -1,131 +0,0 @@
# Post-quantum handshake transport compared with QUIC
Status: analysis, 2026-10-14. Compares the fragment transport in
[pq-handshake.md](pq-handshake.md) with how QUIC makes its handshake reliable. TLS is out of
scope; only the transport mechanics are compared.
## Summary
QUIC's handshake reliability is closed-loop: the receiver acknowledges, and the sender
repairs exactly what is missing. Ours is open-loop: repair symbols are sent up front, and a
timer sends fresh symbols. At handshake sizes (K <= 15) a QUIC-style acknowledgement
shrinks to a 16-bit bitmap, and with an MDS code to a single count. So the real comparison
is forward error correction versus acknowledgements, not QUIC versus us.
FEC is the right core. One piece of feedback is worth adding, initiator to responder and for
msg2 only (see [Worth taking from QUIC](#worth-taking-from-quic)).
## What QUIC does
- **Messages as a stream.** Handshake bytes travel as `CRYPTO` frames (offset and length)
and are reassembled in order. Initial and Handshake packets each have their own packet
number space.
- **Immediate acknowledgements.** Receivers must acknowledge Initial and Handshake packets
without delay. ACK frames list ranges of received packet numbers.
- **Loss detection** (RFC 9002). A packet is lost once 3 later packets are acknowledged, or
after 9/8 of the round-trip time. Lost bytes are resent in new packets.
- **Probe timeout (PTO).** If a whole flight vanishes, a probe timer fires. Before any RTT
sample it starts at about 1 s (333 ms initial RTT plus 4x its variance) and doubles each
time. Both endpoints run it.
- **Amplification limit and anti-deadlock.** The server may send at most 3x what it
received until the client's address is validated. The client must keep probing so a
server stuck at that limit cannot deadlock.
- **Retry.** A stateless round trip to validate the address. `NEW_TOKEN` lets later
connections skip it.
- **Congestion control from the first packet.** The initial window is 10 packets of 1200
bytes.
## Side by side
| | QUIC | pq-handshake |
|---|---|---|
| Receiver feedback | ACK ranges, sent immediately | None; the next handshake message is the only acknowledgement |
| Partial loss | +1 to 2 RTT (ACK, detect, resend) | 0 RTT while at most m symbols are lost |
| Total loss | PTO: about 1 s, then doubling | Next PQ tick: 300 ms, then 1 s (alternating), or 100/300/600 ms (sticky) |
| Spurious retransmit | Wasted bytes | Still useful: fresh repair indexes add to what the receiver holds |
| Reassembly | Every byte offset, in order | Any K of up to 256 symbols |
| Who retransmits | Both sides on their own timers | Initiator on a timer; the responder resends msg2 only when a repeat msg1 arrives (rate limited) |
| State from the first packet | Server buffers and acknowledges partial flights | Capped pre-authentication table that never acknowledges |
| Address validation | Retry, 1 RTT | Cookie under load, 1 RTT |
From the [simulation results](pq-handshake.md#simulation-results): phase 2 at 5% random
loss with no FEC completes on the first attempt 39.8% of the time. Under
acknowledgement-based recovery roughly 60% of handshakes would pay at least one extra round
trip. With FEC, 0.4% wait for a retry. Spending bandwidth to save round trips is the stated
design preference, and open-loop recovery delivers it.
## Where the designs already agree
- **1200-byte floor and the 3x amplification budget**, both taken directly from QUIC.
- **Cookie and Retry.** Both cost one round trip and both can be forged by an on-path
attacker. QUIC's Retry integrity tag uses a fixed, published key.
- **On-path injection.** QUIC's Initial keys are derived from the client's connection ID,
which an observer can read, so QUIC accepts the same handshake-blocking injection
weakness documented for our symbols in
[Injected symbols](pq-handshake.md#injected-symbols).
- **Phase 2 msg3 confirmation.** Resending msg3 until any authenticated packet arrives,
with a `Test` from the responder on completion, is QUIC's handshake confirmation and
`HANDSHAKE_DONE`.
## Worth taking from QUIC
### 1. A msg2 progress probe from the initiator
This is the main gap. Today, recovering a partially lost msg2 means the initiator resends a
full msg1 attempt; the responder decodes it, matches it in the answered-msg1 cache, and
resends msg2. In phase 2 that is about 12 fragments (~13 KB) uploaded just to act as an
acknowledgement, and recovery waits for the 300 ms or 1 s tick.
- The initiator already knows the responder's index: msg2 fragments carry it as `FragID`
(`sendFragmentedResponse` passes `hostinfo.localIndexId`).
- A one-packet probe, "msg2: have n of K", sent to that index would recover in about one
round trip. The responder finds it the way phase 2 plans to find msg3.
- Timing comes for free: msg1 send to first msg2 fragment is an RTT sample. Probe at
roughly RTT/8 plus the repair delay after the first msg2 fragment if decoding has not
completed.
- Forging a probe needs the 32-bit responder index, so only an on-path attacker can, and
the existing 3x budget bounds the reply.
- If no msg2 fragment arrived at all, the initiator keeps today's msg1 resend. That is the
same tail-loss case QUIC's PTO covers.
The reverse direction, the responder reporting progress on msg1, should not be added. It
needs per-entry timers on the pre-authentication path, which is exactly the state QUIC
servers carry and the reason QUIC needs Retry.
### 2. Buffer early data in phase 2
QUIC stacks hold a few packets they cannot decrypt yet. The phase 2 plan drops data that
reaches the responder before msg3 decodes. Because repair fragments trail by
`handshakes.pq.repair_delay` (10 ms), losing one msg3 data fragment drops about 10 ms of
the initiator's first data while the repair is in flight. A small queue on the pending
responder entry, replayed once msg3 verifies, avoids this.
### 3. Burst size across remotes
QUIC handshakes on a single path. `handleOutbound` sends the same `now` and `later` sets to
every known remote and to relays (`StartRelays`).
- Phase 1 is fine.
- In phase 2, four remotes means about 48 packets (~54 KB) leaving one uplink at once,
every PQ tick, about 4x QUIC's initial window.
- The Gilbert-Elliott simulation treats each path independently, so it does not capture
this self-inflicted burst on a constrained uplink such as cellular.
- Options: send the full FEC set only to the most likely remote and a reduced set to the
others, or interleave packets across remotes.
## Not worth taking
- **Packet numbers, ACK ranges, an RTT estimator, loss thresholds, congestion control.**
QUIC pays for this machinery because it also serves a data transport. Nebula carries
packets and leaves reliability to the overlay's own transports. Building it only for
handshakes of at most 15 fragments buys nothing a symbol count cannot.
- **Offset-based reassembly.** It gives up the any-K-of-256 property and makes spurious
retransmits wasteful again.
- **An RTT-adaptive retry timer.** Low value. The fixed 100 ms linear schedule already fires
sooner than QUIC's 1 s first PTO, and a spurious resend costs bandwidth, not correctness.
## Conclusion
Keep FEC as the core and add the msg2 progress probe. That gets QUIC's roughly one round
trip tail recovery without pre-authentication state, and cuts phase 2's retry cost on the
uplink by about 10x. Items 2 and 3 are phase 2 concerns; item 1 also helps phase 1.
-348
View File
@@ -1,348 +0,0 @@
# Post-quantum handshakes
Status: draft. Decisions below were made on 2026-10-13; open items are listed at the end.
Phase 1 is implemented (see [Implementation](#implementation-phase-1)).
## Goals
1. **Confidentiality against harvest-now-decrypt-later.** Traffic recorded today must stay
secret after a cryptographically relevant quantum computer exists. This needs a
post-quantum key exchange now.
2. **Post-quantum authentication, later.** Needs certificates whose CA signature and host
keys are post-quantum. This requires reissuing certificates, so it ships second.
3. **Never rely on IP fragmentation.** ML-KEM and ML-DSA do not fit in one packet, so the
handshake must carry messages larger than the path MTU itself.
Non-goals: identity hiding beyond what IX offers today, and signatures at handshake time.
The only signature in the system stays the CA signature inside the certificate.
## Primitives and sizes
| Primitive | Size |
|---|---|
| ML-KEM-1024 encapsulation key | 1568 B |
| ML-KEM-1024 ciphertext | 1568 B |
| ML-DSA-87 public key / signature | 2592 B / 4627 B |
| Today's IX msg1 / msg2 (v2 cert, 151 B) | 252 B / 287 B |
`crypto/mlkem` is available in Go 1.26 (the current `go.mod`); `crypto/mldsa` arrives in
Go 1.27 and is only needed for phase 2.
## Phase 1: ephemeral ML-KEM on today's certificates
A new handshake subtype, `ix_psk2_mlkem1024` (2), IX-shaped (2 messages; the responder
completes on msg1 as today), using the flynn/noise `psk2` modifier. The Noise protocol name
becomes `Noise_IXpsk2_...`, which separates it from the classical subtype.
- **msg1**: the initiator generates a fresh ML-KEM-1024 key pair and places the
encapsulation key in `NebulaHandshakeDetails.KEMKey` (field 9). msg1's payload is mixed
into the Noise transcript, so it cannot be stripped or replaced without msg2 failing.
With the psk modifier Noise also mixes `e` into the cipher key, so msg1's static key and
payload are encrypted, though only under a key derived from the public ephemeral.
- **Responder**: after `ReadMessage(msg1)` it encapsulates to that key, obtaining
`(ct, ss)`, and calls `HandshakeState.SetPresharedKey(psk)` before writing msg2, with
`psk = HKDF-Extract(SHA-256, salt = "nebula ix_psk2_mlkem1024 v1", ikm = ss || ct)`.
The ciphertext is bound in because, unlike the encapsulation key, it is not part of the
Noise transcript.
- **msg2** on the wire is `ct || noise_msg2`. The ciphertext travels outside the Noise
message because the initiator needs `ss` before processing msg2's `psk` token. Any change
to `ct` yields a different `ss` (ML-KEM implicit rejection) and the handshake fails.
Transport keys depend on both the X25519 DH outputs and the ML-KEM shared secret.
Authentication is unchanged and classical. With a minimal v2 certificate msg1 and msg2 are
about 1.84 KB each, so each travels as 2 data and 2 repair fragments of about 936 bytes.
A KEM key on the classical subtype, or a missing or malformed one on this subtype, is a fatal
protocol error; a forged ciphertext only fails that packet.
## Phase 2: pqIX hybrid with certificate v3
Certificate v3 is hybrid: the classical static key and CA signature plus a static
ML-KEM-1024 key and an ML-DSA-87 CA signature. Host authentication is by KEM rather than by
signature, following PQNoise's pqIX, while keeping the classical DH tokens:
- **msg1** (I to R): `e, s`, ephemeral ML-KEM key, initiator cert.
- **msg2** (R to I): `e, s`, ciphertext to I's ephemeral KEM key, ciphertext to I's static
KEM key, responder cert.
- **msg3** (I to R): ciphertext to R's static KEM key, plus first data. R authenticates I
when msg3 decrypts; I authenticates R implicitly when R's first reply decrypts.
msg3 is about 1.6 KB, 2 data and 2 repair fragments, and adds no new message types:
- **Responder.** After sending msg2 the responder keeps a pending Machine, found by its
local index like the initiator's today; this is the multi-message responder support the
manager lacks now. A repeat msg1 maps to it through the answered-msg1 cache and gets msg2
resent with fresh symbols, exactly as in phase 1. Until msg3 verifies, data for that
index is dropped silently: it must not draw a RecvError, which would tear down the
initiator's new tunnel. On completing it sends an empty `Test`/`TestReply` under the
new keys, so the initiator hears back even when there is no traffic.
- **Initiator.** Writing msg3 yields the transport keys, so the hostinfo moves to the main
hostmap and data flows at once, readable only by the real responder. The handshake entry
stays behind in a confirming state: each retry tick resends msg3 with fresh symbols until
any packet from the responder decrypts on the new tunnel, and if that never happens
within the handshake timeout the tunnel is torn down. Losing msg3 costs the data sent
behind it, which the overlay's own transport retransmits.
- msg3's fragments use the responder's index as RemoteIndex and land on the responder's
pending entry, so they never enter the pre-authentication table.
The initiator never caches the responder's certificate, so every handshake has three
messages and carries both full certificates. With ML-DSA-87:
| Message | Size | Data fragments (k) |
|---|---|---|
| msg1 | ~8.2 KB | 8 |
| msg2 | ~9.8 KB | 9 |
| msg3 | ~1.6 KB | 2 |
flynn/noise cannot mix multiple KEM secrets into the chaining key, so phase 2 needs a
KEM-aware Noise symmetric state written in this repository. Both subtypes coexist.
Cryptographic review of the hybrid composition will be done by in-house cryptographers.
## Fragment transport
### Requirements
- No IP fragmentation: every datagram's UDP payload is at most 1232 bytes **after** relay
wrapping (16 B header + 16 B tag), the IPv6 minimum MTU budget also used by DNS Flag Day
2020 and Solana's Turbine.
- Every fragment of a handshake uses the same 5-tuple.
- Bounded pre-authentication state, bounded amplification, selective loss recovery,
tolerance of reordering and duplication, and a strictly validated, fuzzable parser.
### Wire format
Every packet of a fragmented handshake subtype is:
```
Nebula header (16) Version | Type=Handshake | Subtype | Reserved=0 | RemoteIndex | Counter
Fragment header (8) FragID u32 | MsgLen u16 | Index u8 | Flags u8
Cookie (16) only when Flags has 0x01
Symbol SymbolSize bytes
```
- `Counter` is the Noise message number (1, 2, 3). `RemoteIndex` is 0 for msg1, as today.
- `FragID` is the sender's local index (random, non-zero).
- `MsgLen` is the length of the whole message. Both ends derive the layout from it:
`K = ceil(MsgLen / 1160)`, `SymbolSize = ceil(MsgLen / K)`. Every symbol has the same
size. Room for a cookie is always reserved, so a packet is at most
16 + 8 + 16 + 1160 = 1200 bytes with or without one.
- `Index < K` is a data symbol (a slice of the message, the last one zero padded);
`Index >= K` is a repair symbol.
- `Flags` bit 0x01 means a cookie follows the fragment header. Other bits must be zero.
A cookie reply, sent by a responder under load, is a bare 32 byte packet:
```
Nebula header (16) Version | Type=Handshake | Subtype | Reserved=0 | RemoteIndex=FragID | Counter=0
Cookie (16)
```
It names the initiator's FragID, so the initiator finds its attempt the same way it finds
msg2.
### Forward error correction
Messages are protected with a systematic Reed-Solomon code over GF(2^8), implemented in
`handshake/fec`:
- The repair rows form a Cauchy matrix, `coefficient(row, col) = 1 / (row XOR col)` with
`row >= K > col`, so every square submatrix is invertible and **any K distinct symbols
reconstruct the message**. Naive identity-over-Vandermonde constructions are not MDS
(the PAR1 flaw and the 1997 Plank tutorial erratum).
- A repair row depends only on its index, so repair symbols are generated on demand and
their indexes never need to be agreed in advance. Up to 256 symbols exist per message.
- Known-answer vectors in `fec_test.go` pin the encoding and were cross-checked against an
independent implementation; changing them is a wire format change.
- Cost for a 9.8 KB message (k=9): about 13 µs for three repair symbols, about 19 µs to
decode with three data symbols missing.
**The sender chooses m**, the number of repair symbols per attempt. The receiver only needs
any K symbols, so this is policy and can change without a protocol change. The starting
rule is `m = max(2, ceil(K/3))`.
**Retries send fresh symbols.** Attempt 0 sends indexes `0 .. K+m-1`; each retry sends
the next `K+m` unused indexes. The receiver merges all distinct symbols across attempts,
so no received fragment is wasted. Indexes are a deterministic function of the message,
so retransmits stay idempotent: the responder caches msg2 itself and regenerates any
symbol.
**Repair symbols trail the data.** Data symbols go out back to back; repair symbols follow
after a short delay, so a queue tail-drop at the end of the data run does not also take the
repair. In simulation this lifts small messages substantially (k=2 at 10% loss, burst 8:
91.4% to 98.2% first-attempt success). Pacing every fragment was also evaluated; it gains
about 5 more points for k=7..9 on bursty paths, which is not worth the extra machinery,
and the retry with fresh symbols covers the remainder.
### Receiver rules
Implemented in `handshake/fragment` (`Sender`, `Assembly`, and the capped `Table`).
- Fragments are stored individually as they arrive and charged against a memory cap.
Nothing is allocated from a claimed size alone (FragmentSmack, game-engine "fragment
bombs").
- `MsgLen` must be identical for every fragment of a `FragID`; a mismatch drops the whole
entry. msg1 is capped at `MsgLen <= 16384` (K <= 15).
- The decoder keeps at most K distinct symbols, so an entry never exceeds about MsgLen bytes
however many repair symbols arrive.
- When K symbols are present the message is decoded and handed to the handshake `Machine`
as a synthetic `[header | message]`. The Machine never sees fragments.
- A completed entry is forgotten immediately. A late duplicate fragment starts a new entry
and, once enough arrive, reconstructs the same message again; the handshake layer's
duplicate detection recognises it.
- If Noise rejects a decoded message, the entry is discarded and the next attempt
supplies fresh symbols. (Possible refinement: keep a few spare symbols so an injected
bad symbol can be identified by re-encoding instead of waiting for a retry.)
- Pre-authentication msg1 entries live in a table keyed by (source address or relay index,
FragID) with a total memory cap, per-source quotas (/32 for IPv4, /64 for IPv6), a
sliding TTL of about 2 s refreshed by progress, and random eviction under pressure.
msg2 and msg3 reassembly attaches to the existing pending handshake found by RemoteIndex.
### Retransmission and amplification
- Only timers cause transmission. Reactive resends (the responder resending its cached
msg2 when a duplicate msg1 completes) are rate limited to one per handshake per retry
interval, to avoid TFTP's Sorcerer's Apprentice amplification.
- Duplicate msg1 detection compares the decoded message to the cached one **before**
running Noise or certificate verification.
- Before an address is validated, a responder sends at most 3x the bytes it received from
it (QUIC's rule). Normal handshakes sit near 1.1-1.2x.
- In phase 2 the initiator retransmits msg3 until any authenticated packet arrives from
the responder; the responder sends a small confirmation on completion and holds tunnel
traffic until msg3 verifies.
### Denial of service
The responder can no longer complete statelessly on msg1, since msg1 arrives in pieces.
Defenses are the capped table above plus a **cookie under load**, in the style of
WireGuard's mac2, QUIC Retry and the SCTP state cookie:
- Once the msg1 table is at least half full (by bytes or by entries), every msg1 fragment
must carry a valid cookie. One without is answered with a cookie reply and leaves no
state. Fragments that already belong to an entry need one too, so an attacker cannot
hold old entries open.
- A cookie is `HMAC-SHA256(secret, source address and port, relay channel, FragID)`
truncated to 16 bytes. The secret rotates every 2 minutes and the previous one is still
accepted, so a cookie is good for 2 to 4 minutes.
- Replies are never larger than the packet that prompted them (no amplification) and are
rate limited as a whole (a burst of 256, then 1000 per second), so spoofed floods cannot
turn the responder into a reflector.
- The initiator attaches the cookie to its PQ attempt, so every later packet carries it,
and immediately resends fresh symbols to the address the reply came from, at most once
per retry interval. Under load a PQ handshake costs one extra round trip, which in
practice still beats the classical attempt on the next retry tick.
- An attacker on the path can forge a cookie reply; the bogus cookie is simply rejected
and answered with a real one.
A CA-keyed pre-filter MAC was considered and rejected: networks commonly trust many CAs.
#### Injected symbols
The decoder keeps the first K distinct symbols it sees, so one forged symbol in those K
spoils that attempt's decode; Noise rejects the result and the next attempt brings fresh
symbols. Forging one needs the attempt's FragID or index, which are random 32 bit values,
and for msg1 the initiator's exact address and port too, so in practice only an attacker
who sees the traffic can do it. An attacker who can also drop packets gains nothing new.
One who can only inject, and races a forged symbol into every attempt, can stop a PQ
handshake where it could not stop a classical one, since classical msg2s are tried one
packet at a time. Keeping spare symbols to find a consistent subset only raises the number
of forgeries needed per attempt, and authenticating symbols is impossible for msg1, which
travels before any key exists. So the decoder deliberately does neither. A non-sticky
initiator still gets a classical tunnel from its next attempt; a sticky one, by design,
does not downgrade.
## Rollout
- Separate settings for initiating and accepting PQ handshakes, both off by default:
`handshakes.pq.initiate`, `handshakes.pq.accept`. `handshakes.pq.repair_delay` (default
10ms) sets how long repair fragments trail the data fragments.
- `handshakes.pq.require` makes every handshake PQ: it implies `initiate` and `accept`,
never offers a classical attempt, and drops classical msg1s.
- Old nodes silently drop unknown subtypes, so discovery is speculative: an initiator with
`initiate` enabled alternates PQ and classical attempts on successive timer retries,
keeping both pending, and the first to complete wins. A path nothing has been sent over
yet gets the PQ attempt first: an early resend triggered by a lighthouse reply is PQ, and
a relay that comes up pushes the PQ attempt through at once, counting as the PQ turn.
Without that, relays set up during the first attempt handed every relayed handshake to
the classical retry (seen on real hosts: 11 of 11). Once a msg2 fragment or cookie reply
arrives for the PQ attempt the peer evidently speaks PQ, and classical stops getting
turns.
- All `handshakes.pq` settings reload without a restart. Handshakes in flight carry on;
the new settings apply to the next attempt and the next packet.
- `handshakes.pq.sticky` (default off): once a peer has completed a PQ handshake, never fall
back to classical for it again during this process lifetime. This resists active
downgrade after first contact.
- No lighthouse capability advertisement.
## Simulation results
Gilbert-Elliott channel (bad state drops everything), losses expressed as average rate and
mean burst length in packets. First-attempt success for the whole handshake with
`m = max(2, ceil(K/3))` and repair delayed 16 packet slots, compared with no FEC:
| Handshake | Loss | Burst 1 | Burst 2 | Burst 4 | Burst 8 |
|---|---|---|---|---|---|
| phase 1 (2, 2) no FEC | 5% | 81.4% | 85.6% | 87.9% | 89.0% |
| phase 1 (2, 2) FEC | 5% | 99.9% | 99.4% | 99.3% | 98.6% |
| phase 1 (2, 2) FEC | 10% | 99.3% | 97.5% | 97.4% | 96.3% |
| phase 2 (8, 9, 2) no FEC | 5% | 37.8% | 56.0% | 69.3% | 77.1% |
| phase 2 (8, 9, 2) FEC | 5% | 99.6% | 93.7% | 89.9% | 88.4% |
| phase 2 (8, 9, 2) FEC | 10% | 95.3% | 83.5% | 77.9% | 76.7% |
With fresh repair symbols, phase 2 completes within two attempts at least 99.1% of the time
in every scenario above, and within three at least 99.98%.
The simulators and their outputs are in [pq-handshake-sim](pq-handshake-sim/README.md),
including a comparison with classical handshakes and a model of time to a working tunnel
under Nebula's retry schedule.
## Prior art
- **WireGuard / Rosenpass**: fit every message in one packet; cookies and biscuits for
statelessness. Not possible at our sizes.
- **IKEv2**: RFC 7383 fragments only after keys exist, with each fragment authenticated;
RFC 9242/9370 move PQ key exchange into intermediate exchanges for that reason.
- **QUIC / DTLS 1.3**: offset-based reassembly with ACKs; QUIC's 3x amplification limit and
1200-byte floor.
- **Solana Turbine**: Reed-Solomon over 1232-byte UDP packets through untrusted relays;
moved to Merkle-authenticated shreds after unauthenticated repair symbols enabled
poisoning.
- **kcp-go, UDPspeeder, CM256**: RS FEC over UDP; parity must scale up for small groups.
- **gQUIC FEC (removed)**: FEC does not pay for bulk transfer under bursty loss; it does
for short, latency-critical transfers such as handshakes.
- **TFTP, NFS duplicate request cache, memcached UDP, IP reassembly bugs**: retransmit
storms, ID-wrap misassembly, reflection amplification, and allocation from claimed sizes.
## Implementation (phase 1)
- `handshake`: the `ix_psk2_mlkem1024` subtype (`patterns.go`, `kem.go`), the `KEMKey`
payload field, and the psk2 plumbing in `Machine`.
- `handshake/fec`, `handshake/fragment`: the code and the fragment transport.
- `handshake_manager_pq.go` and the manager:
- **Initiator.** Attempts alternate PQ and classical, starting with PQ (sticky peers get
only PQ). The PQ attempt has its own local index registered alongside the classical
one; whichever response completes first wins, and the other index is released
(`settleIndex`). A timed out or deleted pending handshake releases both.
- **Responder.** msg1 fragments go into the capped pre-authentication table. A decoded
msg1 is looked up by hash among recently answered ones before any Noise, KEM or
certificate work: the trailing repair fragments of an attempt rebuild a msg1 its data
fragments already completed, and a retrying initiator offers fresh symbols for the
same msg1. A repeat goes straight to the cached-response resend, which offers fresh
msg2 symbols, is rate limited to one per retry interval and is held to the 3x
amplification budget.
- **Relays** carry fragments unchanged; repair fragments trail via a timer.
- The pre-authentication table is capped at 4 MiB, 1024 entries, 4 per source (IPv4 /32,
IPv6 /64, or relay channel), 16 KiB messages and a 3 s idle timeout. Cookies are
demanded from half full.
- Tests: Noise-level KEM tests, codec and fragment tests and fuzzers, manager unit tests
for the response budget and rate limit, and e2e tests for a clean handshake, loss of
data fragments in both directions, fallback to a peer that does not accept PQ, a lost
msg2 recovered by fresh symbols, sticky mode under an attacker dropping PQ packets,
relays, index cleanup on completion, fallback and timeout, and a responder flooded from
spoofed sources demanding and accepting a cookie. Unit tests cover cookie binding and
rotation, the never-amplify rule, the initiator's rate limited resend, and reload. ML-KEM and HKDF also
pass under `GOEXPERIMENT=boringcrypto` and `GODEBUG=fips140=on`.
## Open items
None for phase 1. Phase 2 needs certificate v3, the KEM-aware Noise symmetric state, and
the msg3 design above.
+1
View File
@@ -1,4 +1,5 @@
//go:build e2e_testing
// +build e2e_testing
package e2e
+1
View File
@@ -1,4 +1,5 @@
//go:build e2e_testing
// +build e2e_testing
package e2e
+1
View File
@@ -1,4 +1,5 @@
//go:build e2e_testing
// +build e2e_testing
package e2e
+1
View File
@@ -1,4 +1,5 @@
//go:build e2e_testing
// +build e2e_testing
package e2e
+1
View File
@@ -1,4 +1,5 @@
//go:build e2e_testing
// +build e2e_testing
package e2e
-476
View File
@@ -1,476 +0,0 @@
//go:build e2e_testing
package e2e
import (
"net/netip"
"sync"
"testing"
"time"
"github.com/slackhq/nebula"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/e2e/router"
"github.com/slackhq/nebula/handshake/fragment"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func pqRequire() m {
return m{"handshakes": m{"pq": m{"require": true, "repair_delay": "0s"}}}
}
func pqConfig(initiate, accept, sticky bool) m {
return m{"handshakes": m{"pq": m{
"initiate": initiate,
"accept": accept,
"sticky": sticky,
"repair_delay": "0s",
}}}
}
// handshakeOf returns the handshake header p carries, looking inside relay
// wrapping, plus the fragment header when the handshake is fragmented.
func handshakeOf(p *udp.Packet) (*header.H, *fragment.Header) {
b := p.Data
h := &header.H{}
if h.Parse(b) != nil {
return nil, nil
}
if h.Type == header.Message && h.Subtype == header.MessageRelay {
// Relays authenticate but do not encrypt the inner packet.
b = b[header.Len:]
if h.Parse(b) != nil {
return nil, nil
}
}
if h.Type != header.Handshake {
return nil, nil
}
if !header.IsFragmented(h.Subtype) {
return h, nil
}
// A relayed packet still has the relay's AEAD tag on the end.
for _, tag := range []int{0, 16} {
if len(b) < header.Len+tag {
break
}
if fh, _, err := fragment.Parse(b[header.Len : len(b)-tag]); err == nil {
return h, &fh
}
}
return h, nil
}
// handshakeTally counts handshake packets by style as the router moves them.
type handshakeTally struct {
sync.Mutex
pq, classical int
maxPQLen int
}
func (ht *handshakeTally) observe(p *udp.Packet) {
h, _ := handshakeOf(p)
if h == nil {
return
}
ht.Lock()
defer ht.Unlock()
if h.Subtype == header.HandshakeIXPSK2MLKEM1024 {
ht.pq++
ht.maxPQLen = max(ht.maxPQLen, len(p.Data))
} else {
ht.classical++
}
}
// isTunnelData reports whether p is ordinary tunnel traffic headed for to,
// directly or as the last hop of a relay.
func isTunnelData(p *udp.Packet, receiver, to *nebula.Control) bool {
h := &header.H{}
if receiver != to || h.Parse(p.Data) != nil || h.Type != header.Message {
return false
}
if h.Subtype == header.MessageRelay {
if h.Parse(p.Data[header.Len:]) != nil {
return false
}
}
return h.Type == header.Message && h.Subtype == header.MessageNone
}
func newPQPair(t *testing.T, mine, theirs m) (myControl, theirControl *nebula.Control, myVpn, theirVpn []netip.Prefix, myUdp, theirUdp netip.AddrPort, r *router.R) {
t.Helper()
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
myControl, myVpn, myUdp, _ = newSimpleServer(cert.Version2, ca, caKey, "me", "10.128.0.1/24", mine)
theirControl, theirVpn, theirUdp, _ = newSimpleServer(cert.Version2, ca, caKey, "them", "10.128.0.2/24", theirs)
myControl.InjectLightHouseAddr(theirVpn[0].Addr(), theirUdp)
r = router.NewR(t, myControl, theirControl)
myControl.Start()
theirControl.Start()
return
}
func TestPQHandshake(t *testing.T) {
t.Parallel()
// They only accept, which is all a PQ initiator needs.
myControl, theirControl, myVpn, theirVpn, myUdp, theirUdp, r := newPQPair(t, pqConfig(true, true, false), pqConfig(false, true, false))
defer r.RenderFlow()
var tally handshakeTally
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpn[0].Addr(), 80, myVpn[0].Addr(), 80, []byte("Hi from me")))
require.True(t, r.RouteForAllExitFuncOrTimeout(5*time.Second, func(p *udp.Packet, receiver *nebula.Control) router.ExitType {
tally.observe(p)
if isTunnelData(p, receiver, theirControl) {
return router.RouteAndExit
}
return router.KeepRouting
}), "tunnel never came up")
assertUdpPacket(t, []byte("Hi from me"), theirControl.GetFromTun(true), myVpn[0].Addr(), theirVpn[0].Addr(), 80, 80)
assert.Zero(t, tally.classical, "no classical handshake should be needed")
assert.GreaterOrEqual(t, tally.pq, 4, "both messages travel as several fragments")
assert.LessOrEqual(t, tally.maxPQLen, fragment.MaxPacketLen)
assertHostInfoPair(t, myUdp, theirUdp, myVpn, theirVpn, myControl, theirControl)
assertTunnel(t, myVpn[0].Addr(), theirVpn[0].Addr(), myControl, theirControl, r)
assert.Empty(t, myControl.ListHostmapIndexes(true), "no pending index may outlive the handshake")
assert.Empty(t, theirControl.ListHostmapIndexes(true))
myControl.Stop()
theirControl.Stop()
}
// TestPQHandshakeLoss drops data fragments of both messages; the repair
// fragments must carry the handshake without a retry.
func TestPQHandshakeLoss(t *testing.T) {
t.Parallel()
myControl, theirControl, myVpn, theirVpn, _, _, r := newPQPair(t, pqConfig(true, true, false), pqConfig(false, true, false))
defer r.RenderFlow()
var tally handshakeTally
dropped := 0
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpn[0].Addr(), 80, myVpn[0].Addr(), 80, []byte("Hi from me")))
require.True(t, r.RouteForAllExitFuncOrTimeout(5*time.Second, func(p *udp.Packet, receiver *nebula.Control) router.ExitType {
if h, fh := handshakeOf(p); fh != nil {
// Every data fragment of msg1, and the first of msg2.
if (h.MessageCounter == 1 && int(fh.Index) < 2) || (h.MessageCounter == 2 && fh.Index == 0) {
dropped++
return router.Drop
}
}
tally.observe(p)
if isTunnelData(p, receiver, theirControl) {
return router.RouteAndExit
}
return router.KeepRouting
}), "tunnel never came up")
assertUdpPacket(t, []byte("Hi from me"), theirControl.GetFromTun(true), myVpn[0].Addr(), theirVpn[0].Addr(), 80, 80)
assert.Equal(t, 3, dropped)
assert.Zero(t, tally.classical, "the first attempt should succeed from repair fragments")
assertTunnel(t, myVpn[0].Addr(), theirVpn[0].Addr(), myControl, theirControl, r)
myControl.Stop()
theirControl.Stop()
}
// TestPQFallback has a peer that does not accept PQ handshakes. The initiator's
// next attempt is classical and the tunnel comes up anyway.
func TestPQFallback(t *testing.T) {
t.Parallel()
myControl, theirControl, myVpn, theirVpn, myUdp, theirUdp, r := newPQPair(t, pqConfig(true, true, false), nil)
defer r.RenderFlow()
var tally handshakeTally
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpn[0].Addr(), 80, myVpn[0].Addr(), 80, []byte("Hi from me")))
require.True(t, r.RouteForAllExitFuncOrTimeout(5*time.Second, func(p *udp.Packet, receiver *nebula.Control) router.ExitType {
tally.observe(p)
if isTunnelData(p, receiver, theirControl) {
return router.RouteAndExit
}
return router.KeepRouting
}), "tunnel never came up")
assertUdpPacket(t, []byte("Hi from me"), theirControl.GetFromTun(true), myVpn[0].Addr(), theirVpn[0].Addr(), 80, 80)
assert.Positive(t, tally.pq, "a PQ attempt comes first")
assert.Positive(t, tally.classical, "then a classical one")
assertHostInfoPair(t, myUdp, theirUdp, myVpn, theirVpn, myControl, theirControl)
assertTunnel(t, myVpn[0].Addr(), theirVpn[0].Addr(), myControl, theirControl, r)
assert.Empty(t, myControl.ListHostmapIndexes(true), "the losing PQ attempt's index must be released")
myControl.Stop()
theirControl.Stop()
}
// TestPQResendFreshSymbols loses the responder's whole first msg2. The
// initiator's next PQ attempt offers fresh msg1 symbols, the responder
// recognises the repeat and resends msg2 with symbols it has not sent before.
func TestPQResendFreshSymbols(t *testing.T) {
t.Parallel()
myControl, theirControl, myVpn, theirVpn, _, _, r := newPQPair(t, pqConfig(true, true, false), pqConfig(false, true, false))
defer r.RenderFlow()
var firstMsg2, freshMsg2, freshMsg1 int
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpn[0].Addr(), 80, myVpn[0].Addr(), 80, []byte("Hi from me")))
require.True(t, r.RouteForAllExitFuncOrTimeout(10*time.Second, func(p *udp.Packet, receiver *nebula.Control) router.ExitType {
h, fh := handshakeOf(p)
switch {
case h != nil && fh == nil:
// Keep the classical fallback out of the way.
return router.Drop
case fh != nil && h.MessageCounter == 2 && fh.Index < 4:
firstMsg2++
return router.Drop
case fh != nil && h.MessageCounter == 2:
freshMsg2++
case fh != nil && h.MessageCounter == 1 && fh.Index >= 4:
freshMsg1++
}
if isTunnelData(p, receiver, theirControl) {
return router.RouteAndExit
}
return router.KeepRouting
}), "tunnel never came up")
assertUdpPacket(t, []byte("Hi from me"), theirControl.GetFromTun(true), myVpn[0].Addr(), theirVpn[0].Addr(), 80, 80)
assert.Equal(t, 4, firstMsg2, "the first msg2 is 2 data and 2 repair fragments")
assert.Positive(t, freshMsg1, "the retry offers fresh msg1 symbols")
assert.Positive(t, freshMsg2, "the resend offers fresh msg2 symbols")
assertTunnel(t, myVpn[0].Addr(), theirVpn[0].Addr(), myControl, theirControl, r)
myControl.Stop()
theirControl.Stop()
}
// TestPQSticky checks a sticky initiator never offers a classical handshake to a
// peer it has already completed a PQ handshake with.
func TestPQSticky(t *testing.T) {
t.Parallel()
myControl, theirControl, myVpn, theirVpn, _, _, r := newPQPair(t, pqConfig(true, true, true), pqConfig(false, true, false))
defer r.RenderFlow()
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpn[0].Addr(), 80, myVpn[0].Addr(), 80, []byte("Hi from me")))
require.True(t, r.RouteForAllExitFuncOrTimeout(5*time.Second, func(p *udp.Packet, receiver *nebula.Control) router.ExitType {
if isTunnelData(p, receiver, theirControl) {
return router.RouteAndExit
}
return router.KeepRouting
}), "tunnel never came up")
theirControl.GetFromTun(true)
t.Log("Rehandshake while an attacker drops every PQ packet")
var tally handshakeTally
myControl.ReHandshake(theirVpn[0].Addr())
r.RouteForAllExitFuncOrTimeout(time.Second, func(p *udp.Packet, receiver *nebula.Control) router.ExitType {
tally.observe(p)
if h, _ := handshakeOf(p); h != nil && h.Subtype == header.HandshakeIXPSK2MLKEM1024 {
return router.Drop
}
return router.KeepRouting
})
assert.Positive(t, tally.pq, "PQ attempts keep being made")
assert.Zero(t, tally.classical, "sticky must never fall back")
myControl.Stop()
theirControl.Stop()
}
func TestPQRelays(t *testing.T) {
t.Parallel()
ca, _, caKey, _ := cert_test.NewTestCaCert(cert.Version1, cert.Curve_CURVE25519, time.Now(), time.Now().Add(10*time.Minute), nil, nil, []string{})
useRelays := pqConfig(true, true, false)
useRelays["relay"] = m{"use_relays": true}
amRelay := pqConfig(true, true, false)
amRelay["relay"] = m{"am_relay": true}
myControl, myVpnIpNet, _, _ := newSimpleServer(cert.Version1, ca, caKey, "me ", "10.128.0.1/24", useRelays)
relayControl, relayVpnIpNet, relayUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "relay ", "10.128.0.128/24", amRelay)
theirControl, theirVpnIpNet, theirUdpAddr, _ := newSimpleServer(cert.Version1, ca, caKey, "them ", "10.128.0.2/24", useRelays)
myControl.InjectLightHouseAddr(relayVpnIpNet[0].Addr(), relayUdpAddr)
myControl.InjectRelays(theirVpnIpNet[0].Addr(), []netip.Addr{relayVpnIpNet[0].Addr()})
relayControl.InjectLightHouseAddr(theirVpnIpNet[0].Addr(), theirUdpAddr)
r := router.NewR(t, myControl, relayControl, theirControl)
defer r.RenderFlow()
myControl.Start()
relayControl.Start()
theirControl.Start()
var tally handshakeTally
var relayedPQ, relayedClassical int
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpnIpNet[0].Addr(), 80, myVpnIpNet[0].Addr(), 80, []byte("Hi from me")))
require.True(t, r.RouteForAllExitFuncOrTimeout(10*time.Second, func(p *udp.Packet, receiver *nebula.Control) router.ExitType {
tally.observe(p)
outer := &header.H{}
if outer.Parse(p.Data) == nil && outer.Type == header.Message && outer.Subtype == header.MessageRelay {
if h, fh := handshakeOf(p); h != nil && fh != nil {
relayedPQ++
assert.LessOrEqual(t, len(p.Data), fragment.MaxPacketLen+32, "a relayed fragment fits the IPv6 minimum MTU")
} else if h != nil {
relayedClassical++
}
}
if isTunnelData(p, receiver, theirControl) {
return router.RouteAndExit
}
return router.KeepRouting
}), "tunnel never came up")
assertUdpPacket(t, []byte("Hi from me"), theirControl.GetFromTun(true), myVpnIpNet[0].Addr(), theirVpnIpNet[0].Addr(), 80, 80)
assert.Positive(t, relayedPQ, "the end to end handshake is PQ through the relay")
assert.Zero(t, relayedClassical, "the relay coming up carries the PQ attempt, not the classical retry")
r.RenderHostmaps("Final hostmaps", myControl, relayControl, theirControl)
myControl.Stop()
relayControl.Stop()
theirControl.Stop()
}
// TestPQTimeoutReleasesIndexes lets a handshake with both a PQ and a classical
// attempt pending time out, and checks neither index is left behind.
func TestPQTimeoutReleasesIndexes(t *testing.T) {
t.Parallel()
mine := pqConfig(true, true, false)
mine["handshakes"].(m)["retries"] = 2
myControl, theirControl, myVpn, theirVpn, _, _, r := newPQPair(t, mine, pqConfig(false, true, false))
defer r.RenderFlow()
var tally handshakeTally
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpn[0].Addr(), 80, myVpn[0].Addr(), 80, []byte("Hi from me")))
r.RouteForAllExitFuncOrTimeout(1500*time.Millisecond, func(p *udp.Packet, receiver *nebula.Control) router.ExitType {
tally.observe(p)
return router.Drop
})
assert.Positive(t, tally.pq)
assert.Positive(t, tally.classical)
assert.Nil(t, myControl.GetHostInfoByVpnAddr(theirVpn[0].Addr(), true), "the handshake should have timed out")
assert.Empty(t, myControl.ListHostmapIndexes(true))
myControl.Stop()
theirControl.Stop()
}
// TestPQCookieUnderLoad floods the responder with msg1 fragments from spoofed
// sources until it demands cookies, then checks a real initiator picks up the
// cookie and still completes a PQ handshake.
func TestPQCookieUnderLoad(t *testing.T) {
t.Parallel()
myControl, theirControl, myVpn, theirVpn, _, theirUdp, r := newPQPair(t, pqConfig(true, true, false), pqConfig(false, true, false))
defer r.RenderFlow()
// Half of the 1024 entry table is the cookie threshold. Stopping exactly there
// keeps them from answering the spoofed sources, which the router cannot reach.
t.Log("Flood them with the first fragment of msg1s nobody will finish")
for i := range 512 {
from := netip.AddrPortFrom(netip.AddrFrom4([4]byte{198, 18, byte(i >> 8), byte(i)}), 4242)
s, err := fragment.NewSender(make([]byte, 2000), header.HandshakeIXPSK2MLKEM1024, 0, 1, uint32(i+1))
require.NoError(t, err)
pkts, _, err := s.Attempt(0)
require.NoError(t, err)
theirControl.InjectUDPPacket(&udp.Packet{From: from, To: theirUdp, Data: pkts[0]})
}
var tally handshakeTally
var cookieReplies, msg2, withCookie int
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpn[0].Addr(), 80, myVpn[0].Addr(), 80, []byte("Hi from me")))
require.True(t, r.RouteForAllExitFuncOrTimeout(5*time.Second, func(p *udp.Packet, receiver *nebula.Control) router.ExitType {
tally.observe(p)
if h, fh := handshakeOf(p); h != nil && h.Subtype == header.HandshakeIXPSK2MLKEM1024 {
switch {
case h.MessageCounter == 0:
cookieReplies++
case h.MessageCounter == 2:
msg2++
case fh != nil && fh.HasCookie():
withCookie++
}
}
if isTunnelData(p, receiver, theirControl) {
return router.RouteAndExit
}
return router.KeepRouting
}), "tunnel never came up")
assertUdpPacket(t, []byte("Hi from me"), theirControl.GetFromTun(true), myVpn[0].Addr(), theirVpn[0].Addr(), 80, 80)
assert.Positive(t, cookieReplies, "a loaded responder asks for a cookie")
assert.Positive(t, withCookie, "the initiator resends with it")
assert.Positive(t, msg2, "and the handshake completes post-quantum")
assert.Zero(t, tally.classical, "the cookie round trip beats the classical fallback")
assertTunnel(t, myVpn[0].Addr(), theirVpn[0].Addr(), myControl, theirControl, r)
myControl.Stop()
theirControl.Stop()
}
// TestPQRequire checks require mode never handshakes classically, in either
// direction.
func TestPQRequire(t *testing.T) {
t.Parallel()
t.Run("initiator", func(t *testing.T) {
// They neither speak PQ nor require it; we never fall back.
myControl, theirControl, myVpn, theirVpn, _, _, r := newPQPair(t, pqRequire(), nil)
defer r.RenderFlow()
var tally handshakeTally
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpn[0].Addr(), 80, myVpn[0].Addr(), 80, []byte("Hi from me")))
r.RouteForAllExitFuncOrTimeout(time.Second, func(p *udp.Packet, receiver *nebula.Control) router.ExitType {
tally.observe(p)
return router.KeepRouting
})
assert.Positive(t, tally.pq)
assert.Zero(t, tally.classical)
assert.Nil(t, theirControl.GetHostInfoByVpnAddr(myVpn[0].Addr(), false), "no tunnel without PQ")
myControl.Stop()
theirControl.Stop()
})
t.Run("responder", func(t *testing.T) {
// We only speak classical; they refuse it.
myControl, theirControl, myVpn, theirVpn, _, _, r := newPQPair(t, nil, pqRequire())
defer r.RenderFlow()
var answers int
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpn[0].Addr(), 80, myVpn[0].Addr(), 80, []byte("Hi from me")))
r.RouteForAllExitFuncOrTimeout(time.Second, func(p *udp.Packet, receiver *nebula.Control) router.ExitType {
if h, _ := handshakeOf(p); h != nil && h.MessageCounter == 2 {
answers++
}
return router.KeepRouting
})
assert.Zero(t, answers, "a classical msg1 gets no answer")
assert.Nil(t, theirControl.GetHostInfoByVpnAddr(myVpn[0].Addr(), false))
myControl.Stop()
theirControl.Stop()
})
t.Run("both", func(t *testing.T) {
myControl, theirControl, myVpn, theirVpn, _, _, r := newPQPair(t, pqRequire(), pqRequire())
defer r.RenderFlow()
var tally handshakeTally
myControl.InjectTunPacket(BuildTunUDPPacket(theirVpn[0].Addr(), 80, myVpn[0].Addr(), 80, []byte("Hi from me")))
require.True(t, r.RouteForAllExitFuncOrTimeout(5*time.Second, func(p *udp.Packet, receiver *nebula.Control) router.ExitType {
tally.observe(p)
if isTunnelData(p, receiver, theirControl) {
return router.RouteAndExit
}
return router.KeepRouting
}), "tunnel never came up")
assertUdpPacket(t, []byte("Hi from me"), theirControl.GetFromTun(true), myVpn[0].Addr(), theirVpn[0].Addr(), 80, 80)
assert.Zero(t, tally.classical)
assertTunnel(t, myVpn[0].Addr(), theirVpn[0].Addr(), myControl, theirControl, r)
myControl.Stop()
theirControl.Stop()
})
}
+1
View File
@@ -1,4 +1,5 @@
//go:build e2e_testing
// +build e2e_testing
package e2e
+1
View File
@@ -1,4 +1,5 @@
//go:build e2e_testing
// +build e2e_testing
package e2e
+1
View File
@@ -1,4 +1,5 @@
//go:build e2e_testing
// +build e2e_testing
package router
+1
View File
@@ -1,4 +1,5 @@
//go:build e2e_testing
// +build e2e_testing
package router
+1
View File
@@ -1,4 +1,5 @@
//go:build e2e_testing
// +build e2e_testing
package e2e
+1
View File
@@ -1,4 +1,5 @@
//go:build e2e_testing
// +build e2e_testing
package e2e
+7 -26
View File
@@ -135,12 +135,12 @@ listen:
# superpacket, so the receive scratch is batch * 64KiB per listening socket (~4MiB per routine at the
# default of 64). Lower this to trade peak per-syscall throughput for memory on constrained hosts.
#batch: 64
# Linux only. Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel
# Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel
# Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default)
# Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide
# max, net.core.rmem_max and net.core.wmem_max
read_buffer: 3412512
write_buffer: 3412512
#read_buffer: 10485760
#write_buffer: 10485760
# On Windows only
# When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port.
@@ -262,10 +262,9 @@ tun:
drop_local_broadcast: false
# Toggles forwarding of multicast packets
drop_multicast: false
# Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 5000
tx_queue: 5000
# Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 500
tx_queue: 500
# Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic
# Anything above 8937, here or on a route below, is capped to 8937 with a warning
mtu: 1300
# the use_offloads setting controls if Nebula will attempt to enable GSO and GRO for the tun device. Linux only, not reloadable.
@@ -411,31 +410,13 @@ logging:
#try_interval: 100ms
#retries: 20
# query_buffer is the size of the buffer channel for querying lighthouses, drops are recorded in the lighthouse.query.dropped metric.
#query_buffer: 1024
# query_buffer is the size of the buffer channel for querying lighthouses
#query_buffer: 64
# trigger_buffer is the size of the buffer channel for quickly sending handshakes
# after receiving the response for lighthouse queries
#trigger_buffer: 64
# pq enables handshakes that mix an ephemeral ML-KEM-1024 key exchange into the session keys, protecting
# recorded traffic against a future quantum computer. Authentication is unchanged. These handshakes are too
# large for one packet and travel as fragments with forward error correction.
#pq:
# initiate starts post-quantum handshakes. Attempts alternate between post-quantum and classical until one is
# answered, so peers that do not support them still connect.
#initiate: false
# accept answers post-quantum handshakes from peers.
#accept: false
# require makes every handshake post-quantum: it implies initiate and accept, never offers a classical handshake
# and refuses classical handshakes from peers. Only enable it once every host can speak post-quantum handshakes.
#require: false
# sticky never falls back to a classical handshake with a peer that has completed a post-quantum one since
# nebula started, which defeats an attacker who drops post-quantum handshakes to force a downgrade.
#sticky: false
# repair_delay is how long the forward error correction fragments trail the rest of a handshake message.
#repair_delay: 10ms
# Tunnel manager settings
#tunnels:
# drop_inactive controls whether inactive tunnels are maintained or dropped after the inactive_timeout period has
+1 -1
View File
@@ -22,7 +22,7 @@ func newFixedTicker(t *testing.T, l *slog.Logger, cacheLen int) *ConntrackCacheT
l: l,
cache: make(ConntrackCache, cacheLen),
}
for i := range cacheLen {
for i := 0; i < cacheLen; i++ {
c.cache[Packet{LocalPort: uint16(i) + 1}] = struct{}{}
}
c.cacheTick.Store(1) // cacheV starts at 0, so Get() takes the reset path
+12 -10
View File
@@ -5,33 +5,33 @@ go 1.26.0
require (
dario.cat/mergo v1.0.2
filippo.io/bigmod v0.1.0
github.com/DefinedNet/netlink v1.3.2-dn.1
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be
github.com/armon/go-radix v1.0.0
github.com/flynn/noise v1.1.0
github.com/gaissmai/bart v0.29.0
github.com/gaissmai/bart v0.28.0
github.com/gogo/protobuf v1.3.2
github.com/google/gopacket v1.1.19
github.com/kardianos/service v1.3.0
github.com/miekg/dns v1.1.73
github.com/miekg/dns v1.1.72
github.com/miekg/pkcs11 v1.1.2
github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f
github.com/prometheus/client_golang v1.24.1
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
github.com/stretchr/testify v1.12.1
github.com/stretchr/testify v1.12.0
github.com/vishvananda/netlink v1.3.1
go.uber.org/goleak v1.3.0
go.yaml.in/yaml/v3 v3.0.5
golang.org/x/crypto v0.57.0
golang.org/x/net v0.59.0
golang.org/x/sync v0.23.0
golang.org/x/sys v0.48.0
golang.org/x/term v0.46.0
golang.org/x/crypto v0.54.0
golang.org/x/net v0.57.0
golang.org/x/sync v0.22.0
golang.org/x/sys v0.47.0
golang.org/x/term v0.45.0
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2
golang.zx2c4.com/wireguard v0.0.0-20230325221338-052af4a8072b
golang.zx2c4.com/wireguard/windows v1.0.1
google.golang.org/protobuf v1.36.12
google.golang.org/protobuf v1.36.11
gopkg.in/yaml.v3 v3.0.1
gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
)
@@ -45,5 +45,7 @@ require (
github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect
github.com/vishvananda/netns v0.0.5 // indirect
golang.org/x/mod v0.36.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.45.0 // indirect
)
+26 -20
View File
@@ -3,8 +3,6 @@ dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
filippo.io/bigmod v0.1.0 h1:UNzDk7y9ADKST+axd9skUpBQeW7fG2KrTZyOE4uGQy8=
filippo.io/bigmod v0.1.0/go.mod h1:OjOXDNlClLblvXdwgFFOQFJEocLhhtai8vGLy0JCZlI=
github.com/DefinedNet/netlink v1.3.2-dn.1 h1:Md2PsUE1Xuy1Zxs/Bm6356n1kt5bq2HspXlPJ6pEspg=
github.com/DefinedNet/netlink v1.3.2-dn.1/go.mod h1:o4YSn8Jw7zoDcYXDmV77TRd4eS5sSLuE+vu2zAN67C8=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -25,8 +23,8 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/flynn/noise v1.1.0 h1:KjPQoQCEFdZDiP03phOvGi11+SVVhBG2wOWAorLsstg=
github.com/flynn/noise v1.1.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag=
github.com/gaissmai/bart v0.29.0 h1:wO6HGE8g9YE0Wm0bCpYxwRzfQ4+fbJKOhL64e5ACGCI=
github.com/gaissmai/bart v0.29.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
github.com/gaissmai/bart v0.28.0 h1:89yZLo8NmyqD0RYgJ3QO9HhqqGGw+oWhf90cZm69Lko=
github.com/gaissmai/bart v0.28.0/go.mod h1:GREWQfTLRWz/c5FTOsIw+KkscuFkIV5t8Rp7Nd1Td5c=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
@@ -83,8 +81,8 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/miekg/dns v1.1.73 h1:uhT8nJxmTrPJYClxVxTCX+CVn6qnzSiybRk72Z6DgrE=
github.com/miekg/dns v1.1.73/go.mod h1:RW2Obtfd5NZHvOFe3zYG0W8koWOQtAzyHaLo8vASBuQ=
github.com/miekg/dns v1.1.72 h1:vhmr+TF2A3tuoGNkLDFK9zi36F2LS+hKTRW0Uf8kbzI=
github.com/miekg/dns v1.1.72/go.mod h1:+EuEPhdHOsfk6Wk5TT2CzssZdqkmFhf8r+aVyDEToIs=
github.com/miekg/pkcs11 v1.1.2 h1:/VxmeAX5qU6Q3EwafypogwWbYryHFmF2RpkJmw3m4MQ=
github.com/miekg/pkcs11 v1.1.2/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -138,8 +136,10 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE=
github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg=
github.com/stretchr/testify v1.12.0 h1:K6Mr6jO9JICuend/5xzTM03ydSV3vdNRYAdPSukj8uI=
github.com/stretchr/testify v1.12.0/go.mod h1:bOYBZb5qJ00vPzWfIqBUZPaxK8jWiXc6d3ErP4Ca9Gw=
github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW6bV0=
github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -155,12 +155,14 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.36.0 h1:JJjpVx6myfUsUdAzZuOSTTmRE0PfZeNWzzvKrP7amb4=
golang.org/x/mod v0.36.0/go.mod h1:moc6ELqsWcOw5Ef3xVprK5ul/MvtVvkIXLziUOICjUQ=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -171,8 +173,8 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -180,8 +182,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -195,11 +197,13 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -210,6 +214,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.45.0 h1:18qN3FAooORvApf5XjCXgsuayZOEtXf6JK18I3+ONa8=
golang.org/x/tools v0.45.0/go.mod h1:LuUGqqaXcXMEFEruIVJVm5mgDD8vww/z/SR1gQ4uE/0=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -228,8 +234,8 @@ google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miE
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc=
google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+5 -18
View File
@@ -31,7 +31,6 @@ package nebula
import (
"bufio"
"context"
"fmt"
"net"
"strconv"
@@ -52,29 +51,17 @@ type graphiteConfigExport struct {
Percentiles []float64 // Percentiles to export from timers and histograms
}
// graphiteTimeout bounds one submission, the connect and the write together. An export is a few KB, a host that can't
// take that in this long isn't going to.
const graphiteTimeout = 30 * time.Second
// graphiteOnce performs a single submission to Graphite, returning a non-nil
// error if it could not connect or write all of it within timeout. It gives up
// as soon as ctx is done.
func graphiteOnce(ctx context.Context, c graphiteConfigExport, timeout time.Duration) error {
// error on failed connections.
func graphiteOnce(c graphiteConfigExport) error {
now := time.Now().Unix()
du := float64(c.DurationUnit)
flushSeconds := float64(c.FlushInterval) / float64(time.Second)
deadline := time.Now().Add(timeout)
d := net.Dialer{Deadline: deadline}
conn, err := d.DialContext(ctx, "tcp", c.Addr.String())
conn, err := net.DialTCP("tcp", nil, c.Addr)
if err != nil {
return err
}
defer conn.Close()
if err := conn.SetDeadline(deadline); err != nil {
return err
}
stop := context.AfterFunc(ctx, func() { _ = conn.Close() })
defer stop()
w := bufio.NewWriter(conn)
c.Registry.Each(func(name string, i any) {
switch metric := i.(type) {
@@ -124,7 +111,7 @@ func graphiteOnce(ctx context.Context, c graphiteConfigExport, timeout time.Dura
fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, t.Rate15(), now)
fmt.Fprintf(w, "%s.%s.mean-rate %.2f %d\n", c.Prefix, name, t.RateMean(), now)
}
w.Flush()
})
// A failed write sticks, so this reports the first one
return w.Flush()
return nil
}
-315
View File
@@ -1,315 +0,0 @@
package nebula
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"net"
"strings"
"sync"
"testing"
"time"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// graphiteHost accepts connections and hands each one to the test
type graphiteHost struct {
ln net.Listener
conns chan net.Conn
mu sync.Mutex
closed bool
open []net.Conn
}
// newGraphiteHost accepts connections for the test. A non-zero window shrinks each connection's receive buffer, so a
// sender that is not read from blocks after a little whatever the host's TCP tuning. Only for tests that need a stall,
// a small window makes reading very slow on Linux.
func newGraphiteHost(t *testing.T, window int) *graphiteHost {
t.Helper()
ln, err := net.Listen("tcp", "127.0.0.1:0")
require.NoError(t, err)
h := &graphiteHost{ln: ln, conns: make(chan net.Conn, 8)}
done := make(chan struct{})
t.Cleanup(func() {
_ = ln.Close()
h.mu.Lock()
h.closed = true
for _, c := range h.open {
_ = c.Close()
}
h.mu.Unlock()
close(done)
})
go func() {
for {
c, err := ln.Accept()
if err != nil {
return
}
h.mu.Lock()
if h.closed {
h.mu.Unlock()
_ = c.Close()
return
}
h.open = append(h.open, c)
h.mu.Unlock()
if tc, ok := c.(*net.TCPConn); ok && window > 0 {
_ = tc.SetReadBuffer(window)
}
select {
case h.conns <- c:
case <-done:
return
}
}
}()
return h
}
func (h *graphiteHost) addr(t *testing.T) *net.TCPAddr {
a, err := net.ResolveTCPAddr("tcp", h.ln.Addr().String())
require.NoError(t, err)
return a
}
// readExport reads what one send wrote, bounded so a send that never finishes fails the test rather than hanging it
func readExport(t *testing.T, c net.Conn) []byte {
t.Helper()
require.NoError(t, c.SetReadDeadline(time.Now().Add(10*time.Second)))
b, err := io.ReadAll(c)
require.NoError(t, err)
return b
}
func (h *graphiteHost) next(t *testing.T, wait time.Duration) net.Conn {
t.Helper()
select {
case c := <-h.conns:
return c
case <-time.After(wait):
t.Fatal("no connection")
return nil
}
}
// lockedBuffer is a log destination a test can read while the code under test writes to it from another goroutine
type lockedBuffer struct {
mu sync.Mutex
buf bytes.Buffer
}
func (b *lockedBuffer) Write(p []byte) (int, error) {
b.mu.Lock()
defer b.mu.Unlock()
return b.buf.Write(p)
}
func (b *lockedBuffer) String() string {
b.mu.Lock()
defer b.mu.Unlock()
return b.buf.String()
}
func testGraphiteConfig(addr *net.TCPAddr, r metrics.Registry) graphiteConfigExport {
return graphiteConfigExport{Addr: addr, Registry: r, FlushInterval: time.Second, DurationUnit: time.Nanosecond, Prefix: "nebula"}
}
// Enough metrics that one export overflows the socket buffers on any platform, so a send to a host that stops
// reading blocks
func bigGraphiteRegistry(r metrics.Registry, t *testing.T) {
for i := range 200000 {
name := fmt.Sprintf("big.gauge.%d", i)
metrics.GetOrRegisterGauge(name, r).Update(int64(i))
t.Cleanup(func() { r.Unregister(name) })
}
}
func TestGraphiteOnce_sends(t *testing.T) {
r := metrics.NewRegistry()
metrics.GetOrRegisterGauge("g", r).Update(7)
metrics.GetOrRegisterCounter("c", r).Inc(3)
h := newGraphiteHost(t, 0)
require.NoError(t, graphiteOnce(t.Context(), testGraphiteConfig(h.addr(t), r), time.Minute))
b := readExport(t, h.next(t, 5*time.Second))
assert.Contains(t, string(b), "nebula.g.value 7 ")
assert.Contains(t, string(b), "nebula.c.count 3 ")
assert.Contains(t, string(b), "nebula.c.count_ps 3.00 ")
}
// A host that accepts and never reads is given up on, and the failed write is reported
func TestGraphiteOnce_stalledHostTimesOut(t *testing.T) {
r := metrics.NewRegistry()
bigGraphiteRegistry(r, t)
h := newGraphiteHost(t, 4096)
done := make(chan error, 1)
cfg := testGraphiteConfig(h.addr(t), r)
go func() { done <- graphiteOnce(t.Context(), cfg, 300*time.Millisecond) }()
h.next(t, 5*time.Second) // accepted, never read
select {
case err := <-done:
require.Error(t, err)
case <-time.After(10 * time.Second):
t.Fatal("the send did not give up on a host that stopped reading")
}
}
// The timeout covers the whole send, a host that keeps reading but too slowly is still given up on
func TestGraphiteOnce_timeoutCoversTheWholeSend(t *testing.T) {
r := metrics.NewRegistry()
bigGraphiteRegistry(r, t)
h := newGraphiteHost(t, 0)
done := make(chan error, 1)
start := time.Now()
cfg := testGraphiteConfig(h.addr(t), r)
go func() { done <- graphiteOnce(t.Context(), cfg, 500*time.Millisecond) }()
c := h.next(t, 5*time.Second)
stopReading := make(chan struct{})
defer close(stopReading)
// 32KB every 10ms keeps every write well inside the timeout, the whole export takes seconds
go func() {
buf := make([]byte, 32<<10)
for {
select {
case <-stopReading:
return
case <-time.After(10 * time.Millisecond):
}
if _, err := c.Read(buf); err != nil {
return
}
}
}()
select {
case err := <-done:
require.Error(t, err)
assert.Less(t, time.Since(start), 5*time.Second)
case <-time.After(10 * time.Second):
t.Fatal("a host that kept reading slowly held the send past its timeout")
}
}
func TestGraphiteOnce_cancelledCtxDoesNotConnect(t *testing.T) {
h := newGraphiteHost(t, 0)
ctx, cancel := context.WithCancel(t.Context())
cancel()
require.Error(t, graphiteOnce(ctx, testGraphiteConfig(h.addr(t), metrics.NewRegistry()), time.Minute))
select {
case <-h.conns:
t.Fatal("connected on a cancelled ctx")
case <-time.After(200 * time.Millisecond):
}
}
// A stop or reload abandons a stuck send rather than waiting out its timeout
func TestGraphiteOnce_cancelAbandonsSend(t *testing.T) {
r := metrics.NewRegistry()
bigGraphiteRegistry(r, t)
h := newGraphiteHost(t, 4096)
ctx, cancel := context.WithCancel(t.Context())
done := make(chan error, 1)
cfg := testGraphiteConfig(h.addr(t), r)
go func() { done <- graphiteOnce(ctx, cfg, time.Minute) }()
h.next(t, 5*time.Second)
select {
case err := <-done:
t.Fatalf("the send should be stuck on a host that is not reading, it ended with %v", err)
case <-time.After(200 * time.Millisecond):
}
cancel()
select {
case err := <-done:
require.Error(t, err)
case <-time.After(5 * time.Second):
t.Fatal("the send is still stuck after its ctx was cancelled")
}
}
// buildRuntime hands graphite the configured prefix and interval
func TestStatsServer_graphiteExportUsesConfig(t *testing.T) {
const name = "graphite.wiring.test"
metrics.GetOrRegisterCounter(name, nil).Inc(8)
t.Cleanup(func() { metrics.DefaultRegistry.Unregister(name) })
h := newGraphiteHost(t, 0)
s, c := newTestStatsServer(t)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "4s", "prefix": "pfx", "host": h.ln.Addr().String()})
cfg, err := loadStatsConfig(c)
require.NoError(t, err)
s.runMu.Lock()
fns, _ := s.buildRuntime(t.Context(), cfg)
s.runMu.Unlock()
for _, fn := range fns {
fn()
}
b := readExport(t, h.next(t, 5*time.Second))
assert.Contains(t, string(b), "pfx."+name+".count 8 ")
assert.Contains(t, string(b), "pfx."+name+".count_ps 2.00 ")
}
func TestStatsServer_graphiteFailureIsLogged(t *testing.T) {
logs := &lockedBuffer{}
s, c := newTestStatsServer(t)
s.l = test.NewLoggerWithOutput(logs)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "1h", "host": "127.0.0.1:" + freeTCPPort(t)})
cfg, err := loadStatsConfig(c)
require.NoError(t, err)
s.runMu.Lock()
fns, _ := s.buildRuntime(t.Context(), cfg)
s.runMu.Unlock()
for _, fn := range fns {
fn()
}
assert.Contains(t, logs.String(), "Graphite export failed")
}
// Stop abandons the running export, so it goes through Start rather than handing buildRuntime a ctx, and a stop is
// not logged as a failure
func TestStatsServer_stopAbandonsGraphiteSend(t *testing.T) {
bigGraphiteRegistry(metrics.DefaultRegistry, t)
h := newGraphiteHost(t, 4096)
logs := &lockedBuffer{}
s, c := newTestStatsServer(t)
s.l = test.NewLoggerWithOutput(logs)
setStatsConfig(c, map[string]any{"type": "graphite", "interval": "50ms", "host": h.ln.Addr().String()})
require.NoError(t, s.reload(c, true))
started := make(chan struct{})
go func() {
s.Start()
close(started)
}()
stuck := h.next(t, 30*time.Second)
s.Stop()
select {
case <-started:
case <-time.After(5 * time.Second):
t.Fatal("Start did not return after Stop")
}
// A closed sender resets the connection once it sees this, one still sending just buffers it. Either way the
// read below doesn't have to drain the export through the small window to find out
_, err := stuck.Write([]byte("x"))
require.NoError(t, err)
require.NoError(t, stuck.SetReadDeadline(time.Now().Add(10*time.Second)))
n, err := io.Copy(io.Discard, stuck)
var ne net.Error
if errors.As(err, &ne) && ne.Timeout() {
t.Fatalf("the send kept the connection after Stop, read %d bytes", n)
}
assert.Less(t, n, int64(1<<20), "the send kept going after Stop")
// The capture loop logs on its own goroutine after Start has returned, give it the chance
assert.Never(t, func() bool { return strings.Contains(logs.String(), "Graphite export failed") }, 500*time.Millisecond, 10*time.Millisecond, "a stop is not a failure")
}
+2 -4
View File
@@ -35,9 +35,7 @@ func NewCredential(
}
// buildHandshakeState creates a noise.HandshakeState from this credential.
// pskPlacement selects the noise psk modifier; 0 means none. A psk for
// placements 2 and up is supplied later with SetPresharedKey.
func (hc *Credential) buildHandshakeState(initiator bool, pattern noise.HandshakePattern, pskPlacement int) (*noise.HandshakeState, error) {
func (hc *Credential) buildHandshakeState(initiator bool, pattern noise.HandshakePattern) (*noise.HandshakeState, error) {
return noise.NewHandshakeState(noise.Config{
CipherSuite: hc.cipherSuite,
Random: rand.Reader,
@@ -45,7 +43,7 @@ func (hc *Credential) buildHandshakeState(initiator bool, pattern noise.Handshak
Initiator: initiator,
StaticKeypair: noise.DHKey{Private: hc.privateKey, Public: hc.Cert.PublicKey()},
PresharedKey: []byte{},
PresharedKeyPlacement: pskPlacement,
PresharedKeyPlacement: 0,
})
}
-216
View File
@@ -1,216 +0,0 @@
// Package fec implements the systematic Reed-Solomon erasure code used to carry
// handshake messages that do not fit in a single UDP packet.
//
// A message is split into K equally sized data symbols (the last one zero
// padded). Symbol indexes 0..K-1 are the data symbols themselves; indexes
// K..MaxSymbols-1 are repair symbols. Any K distinct symbols reconstruct the
// message. Repair symbols are generated on demand, so a sender can emit fresh
// ones on every retransmit and a receiver can merge everything it has seen.
package fec
import (
"errors"
"fmt"
)
// MaxSymbols is the number of distinct symbols, data and repair together, one
// message can be encoded into. It is bounded by the size of GF(2^8).
const MaxSymbols = 256
var (
ErrInvalidLayout = errors.New("invalid fec layout")
ErrIndexRange = errors.New("symbol index out of range")
ErrSymbolSize = errors.New("symbol has the wrong size")
ErrNotEnough = errors.New("not enough symbols to decode")
ErrCorrupt = errors.New("decoded message has non-zero padding")
)
// Layout describes how a message is split into symbols. Both ends derive it
// from the message length alone, so it never has to be sent.
type Layout struct {
MsgLen int // length of the message in bytes
K int // number of data symbols
SymbolSize int // size of every symbol, data and repair
}
// NewLayout returns the layout for a message of msgLen bytes carried in symbols
// of at most maxSymbolSize bytes. Symbols are balanced, so padding is always
// less than K bytes.
func NewLayout(msgLen, maxSymbolSize int) (Layout, error) {
if msgLen <= 0 || maxSymbolSize <= 0 {
return Layout{}, ErrInvalidLayout
}
k := (msgLen + maxSymbolSize - 1) / maxSymbolSize
if k >= MaxSymbols {
return Layout{}, fmt.Errorf("%w: %d data symbols", ErrInvalidLayout, k)
}
return Layout{
MsgLen: msgLen,
K: k,
SymbolSize: (msgLen + k - 1) / k,
}, nil
}
// Encoder produces symbols for one message.
type Encoder struct {
layout Layout
data [][]byte
}
// NewEncoder copies msg into padded data symbols.
func NewEncoder(msg []byte, maxSymbolSize int) (*Encoder, error) {
l, err := NewLayout(len(msg), maxSymbolSize)
if err != nil {
return nil, err
}
buf := make([]byte, l.K*l.SymbolSize)
copy(buf, msg)
data := make([][]byte, l.K)
for i := range data {
data[i] = buf[i*l.SymbolSize : (i+1)*l.SymbolSize]
}
return &Encoder{layout: l, data: data}, nil
}
func (e *Encoder) Layout() Layout {
return e.layout
}
// Symbol appends the symbol with the given index to dst and returns the result.
func (e *Encoder) Symbol(dst []byte, index int) ([]byte, error) {
if index < 0 || index >= MaxSymbols {
return nil, ErrIndexRange
}
if index < e.layout.K {
return append(dst, e.data[index]...), nil
}
start := len(dst)
dst = append(dst, make([]byte, e.layout.SymbolSize)...)
out := dst[start:]
for col, d := range e.data {
mulAdd(out, d, coefficient(index, col))
}
return dst, nil
}
// Decoder collects symbols for one message until it can reconstruct it.
// It stores at most K symbols, so its memory is bounded by the layout no matter
// how many symbols arrive.
type Decoder struct {
layout Layout
have [MaxSymbols]bool
syms map[int][]byte
}
// NewDecoder returns a decoder for a message of msgLen bytes that was encoded
// with the same maxSymbolSize.
func NewDecoder(msgLen, maxSymbolSize int) (*Decoder, error) {
l, err := NewLayout(msgLen, maxSymbolSize)
if err != nil {
return nil, err
}
return &Decoder{layout: l, syms: make(map[int][]byte, l.K)}, nil
}
func (d *Decoder) Layout() Layout {
return d.layout
}
// Stored returns the number of symbols held.
func (d *Decoder) Stored() int {
return len(d.syms)
}
// Has reports whether the symbol with the given index is held.
func (d *Decoder) Has(index int) bool {
return index >= 0 && index < MaxSymbols && d.have[index]
}
// Ready reports whether enough symbols have arrived to decode.
func (d *Decoder) Ready() bool {
return len(d.syms) >= d.layout.K
}
// Add stores a copy of the symbol. Duplicates, and anything beyond the first K
// distinct symbols, are ignored. It returns whether the decoder is ready.
func (d *Decoder) Add(index int, sym []byte) (bool, error) {
if index < 0 || index >= MaxSymbols {
return d.Ready(), ErrIndexRange
}
if len(sym) != d.layout.SymbolSize {
return d.Ready(), ErrSymbolSize
}
if d.have[index] || d.Ready() {
return d.Ready(), nil
}
d.have[index] = true
d.syms[index] = append([]byte(nil), sym...)
return d.Ready(), nil
}
// Decode reconstructs the message from the stored symbols.
func (d *Decoder) Decode() ([]byte, error) {
if !d.Ready() {
return nil, ErrNotEnough
}
l := d.layout
out := make([]byte, l.K*l.SymbolSize)
var missing, repair []int
for i := range l.K {
if s, ok := d.syms[i]; ok {
copy(out[i*l.SymbolSize:], s)
} else {
missing = append(missing, i)
}
}
if len(missing) > 0 {
for i := l.K; i < MaxSymbols && len(repair) < len(missing); i++ {
if d.have[i] {
repair = append(repair, i)
}
}
if len(repair) < len(missing) {
return nil, ErrNotEnough
}
// Each repair symbol r satisfies r = sum_col coefficient(r, col) * data[col].
// Move the known data columns to the left hand side, leaving a square
// Cauchy system in the missing columns.
rhs := make([][]byte, len(repair))
m := make([][]byte, len(repair))
for ri, row := range repair {
rhs[ri] = append([]byte(nil), d.syms[row]...)
m[ri] = make([]byte, len(missing))
mi := 0
for col := range l.K {
if mi < len(missing) && missing[mi] == col {
m[ri][mi] = coefficient(row, col)
mi++
continue
}
mulAdd(rhs[ri], out[col*l.SymbolSize:(col+1)*l.SymbolSize], coefficient(row, col))
}
}
if !invert(m) {
// Unreachable for a Cauchy matrix; kept so a broken invariant fails loudly.
return nil, errors.New("fec: singular decode matrix")
}
for mi, col := range missing {
dst := out[col*l.SymbolSize : (col+1)*l.SymbolSize]
for ri := range repair {
mulAdd(dst, rhs[ri], m[mi][ri])
}
}
}
for _, b := range out[l.MsgLen:] {
if b != 0 {
return nil, ErrCorrupt
}
}
return out[:l.MsgLen], nil
}
-318
View File
@@ -1,318 +0,0 @@
package fec
import (
"crypto/sha256"
"encoding/hex"
"math/rand/v2"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// slowMul is carry-less multiplication reduced by gfPoly, used to check the tables.
func slowMul(a, b byte) byte {
var p int
x, y := int(a), int(b)
for y > 0 {
if y&1 != 0 {
p ^= x
}
x <<= 1
if x&0x100 != 0 {
x ^= gfPoly
}
y >>= 1
}
return byte(p)
}
func TestGF(t *testing.T) {
for a := range 256 {
for b := range 256 {
require.Equal(t, slowMul(byte(a), byte(b)), gfMul(byte(a), byte(b)), "%d*%d", a, b)
}
if a != 0 {
require.Equal(t, byte(1), gfMul(byte(a), gfInv(byte(a))), "inv(%d)", a)
}
}
src := make([]byte, 256)
for i := range src {
src[i] = byte(i)
}
for c := range 256 {
dst := make([]byte, 256)
mulAdd(dst, src, byte(c))
for i := range src {
require.Equal(t, gfMul(byte(c), src[i]), dst[i])
}
}
}
func TestLayout(t *testing.T) {
for msgLen := 1; msgLen <= 20000; msgLen++ {
l, err := NewLayout(msgLen, 1176)
require.NoError(t, err)
require.Equal(t, (msgLen+1175)/1176, l.K)
require.LessOrEqual(t, l.SymbolSize, 1176)
require.GreaterOrEqual(t, l.K*l.SymbolSize, msgLen)
require.Less(t, l.K*l.SymbolSize-msgLen, l.K, "padding must be less than one byte per symbol")
}
l, err := NewLayout(65535, 1176)
require.NoError(t, err)
assert.Equal(t, 56, l.K)
_, err = NewLayout(0, 1176)
require.ErrorIs(t, err, ErrInvalidLayout)
_, err = NewLayout(100, 0)
require.ErrorIs(t, err, ErrInvalidLayout)
_, err = NewLayout(256, 1)
require.ErrorIs(t, err, ErrInvalidLayout)
_, err = NewLayout(255, 1)
require.NoError(t, err)
}
func randMsg(r *rand.Rand, n int) []byte {
b := make([]byte, n)
for i := range b {
b[i] = byte(r.Uint32())
}
return b
}
func decodeWith(t *testing.T, enc *Encoder, msgLen, maxSym int, indexes []int) []byte {
t.Helper()
dec, err := NewDecoder(msgLen, maxSym)
require.NoError(t, err)
for _, idx := range indexes {
sym, err := enc.Symbol(nil, idx)
require.NoError(t, err)
_, err = dec.Add(idx, sym)
require.NoError(t, err)
}
require.True(t, dec.Ready())
out, err := dec.Decode()
require.NoError(t, err)
return out
}
// TestExhaustiveSubsets decodes every k-subset of the first k+m symbols for
// small codes, which checks the MDS property directly.
func TestExhaustiveSubsets(t *testing.T) {
r := rand.New(rand.NewPCG(1, 2))
const maxSym = 16
for k := 1; k <= 10; k++ {
for m := 1; m <= 5; m++ {
msgLen := (k-1)*maxSym + 1 + r.IntN(maxSym)
msg := randMsg(r, msgLen)
enc, err := NewEncoder(msg, maxSym)
require.NoError(t, err)
require.Equal(t, k, enc.Layout().K)
n := k + m
for mask := 0; mask < 1<<n; mask++ {
var idx []int
for i := range n {
if mask&(1<<i) != 0 {
idx = append(idx, i)
}
}
if len(idx) != k {
continue
}
require.Equal(t, msg, decodeWith(t, enc, msgLen, maxSym, idx), "k=%d m=%d idx=%v", k, m, idx)
}
}
}
}
// TestRandomHighIndexes draws symbols from the whole index space, the shape
// retransmits produce once fresh repair symbols are in play.
func TestRandomHighIndexes(t *testing.T) {
r := rand.New(rand.NewPCG(3, 4))
for range 2000 {
msgLen := 1 + r.IntN(65535)
msg := randMsg(r, msgLen)
enc, err := NewEncoder(msg, 1176)
require.NoError(t, err)
k := enc.Layout().K
idx := r.Perm(MaxSymbols)[:k]
require.Equal(t, msg, decodeWith(t, enc, msgLen, 1176, idx), "k=%d idx=%v", k, idx)
}
}
func TestDecoderBounds(t *testing.T) {
r := rand.New(rand.NewPCG(5, 6))
msg := randMsg(r, 3000)
enc, err := NewEncoder(msg, 1176)
require.NoError(t, err)
l := enc.Layout()
require.Equal(t, 3, l.K)
require.Equal(t, 1000, l.SymbolSize)
dec, err := NewDecoder(3000, 1176)
require.NoError(t, err)
_, err = dec.Decode()
require.ErrorIs(t, err, ErrNotEnough)
_, err = dec.Add(-1, make([]byte, 1000))
require.ErrorIs(t, err, ErrIndexRange)
_, err = dec.Add(MaxSymbols, make([]byte, 1000))
require.ErrorIs(t, err, ErrIndexRange)
_, err = dec.Add(0, make([]byte, 999))
require.ErrorIs(t, err, ErrSymbolSize)
s5, _ := enc.Symbol(nil, 5)
ready, err := dec.Add(5, s5)
require.NoError(t, err)
require.False(t, ready)
// A duplicate index is ignored, even with different contents.
ready, err = dec.Add(5, make([]byte, 1000))
require.NoError(t, err)
require.False(t, ready)
s0, _ := enc.Symbol(nil, 0)
s9, _ := enc.Symbol(nil, 9)
_, err = dec.Add(0, s0)
require.NoError(t, err)
ready, err = dec.Add(9, s9)
require.NoError(t, err)
require.True(t, ready)
// Symbols beyond K are not stored.
ready, err = dec.Add(1, make([]byte, 1000))
require.NoError(t, err)
require.True(t, ready)
require.Len(t, dec.syms, 3)
out, err := dec.Decode()
require.NoError(t, err)
require.Equal(t, msg, out)
}
func TestCorruptPadding(t *testing.T) {
// 3001 bytes in 3 symbols of 1001 leaves 2 bytes of padding in the last one.
dec, err := NewDecoder(3001, 1176)
require.NoError(t, err)
require.Equal(t, 1001, dec.Layout().SymbolSize)
last := make([]byte, 1001)
last[1000] = 1
for i, s := range [][]byte{make([]byte, 1001), make([]byte, 1001), last} {
_, err = dec.Add(i, s)
require.NoError(t, err)
}
_, err = dec.Decode()
require.ErrorIs(t, err, ErrCorrupt)
}
// TestKnownAnswer pins the encoding. Both ends of a handshake must produce
// identical repair symbols, so any change here is a wire format change.
func TestKnownAnswer(t *testing.T) {
msg := make([]byte, 3000)
for i := range msg {
msg[i] = byte(i*7 + 3)
}
enc, err := NewEncoder(msg, 1176)
require.NoError(t, err)
// Cross-checked against an independent bit-serial GF(2^8) implementation.
want := map[int]string{
3: "52124f58276ec6338d3b3f4eed273bc7cabfe4f0f467928962b046ad1b348f50",
4: "a50928b04738981985a7d4a17ffaf8d7e30014ca64fc53c99800ffb61aec4aa7",
5: "ada4c95a7facbc542aaef584e6f0d081b7b0d0eed69ded03b57d0753a119cfc9",
255: "839066cdf94f963b223fb1c1c1360f698c14f6349b25653ff5b521791a4021b1",
}
for idx, h := range want {
sym, err := enc.Symbol(nil, idx)
require.NoError(t, err)
sum := sha256.Sum256(sym)
assert.Equal(t, h, hex.EncodeToString(sum[:]), "symbol %d", idx)
}
}
func FuzzRoundTrip(f *testing.F) {
f.Add([]byte("hello"), uint16(2), uint64(0))
f.Add(make([]byte, 9000), uint16(1176), uint64(7))
f.Fuzz(func(t *testing.T, msg []byte, maxSym uint16, seed uint64) {
if len(msg) == 0 || maxSym == 0 {
return
}
enc, err := NewEncoder(msg, int(maxSym))
if err != nil {
return
}
r := rand.New(rand.NewPCG(seed, seed))
idx := r.Perm(MaxSymbols)[:enc.Layout().K]
require.Equal(t, msg, decodeWith(t, enc, len(msg), int(maxSym), idx))
})
}
// FuzzDecoder feeds arbitrary symbols to a decoder; it must never panic and
// must either fail or return a message of the declared length.
func FuzzDecoder(f *testing.F) {
f.Add(uint16(3000), []byte{0, 5, 9, 200}, []byte("some symbol bytes"))
f.Fuzz(func(t *testing.T, msgLen uint16, indexes []byte, fill []byte) {
dec, err := NewDecoder(int(msgLen), 1176)
if err != nil {
return
}
size := dec.Layout().SymbolSize
for i, idx := range indexes {
sym := make([]byte, size)
for j := range sym {
if len(fill) > 0 {
sym[j] = fill[(i+j)%len(fill)]
}
}
if _, err := dec.Add(int(idx), sym); err != nil {
t.Fatal(err)
}
}
out, err := dec.Decode()
if err == nil {
require.Len(t, out, int(msgLen))
}
})
}
// BenchmarkRepair and BenchmarkDecode use a v3-sized msg2 (~9.8KB, k=9) with
// three repair symbols, decoding with three data symbols missing.
func BenchmarkRepair(b *testing.B) {
msg := randMsg(rand.New(rand.NewPCG(7, 8)), 9800)
enc, err := NewEncoder(msg, 1176)
require.NoError(b, err)
buf := make([]byte, 0, 1176)
b.SetBytes(int64(len(msg)))
for b.Loop() {
for idx := 9; idx < 12; idx++ {
buf, _ = enc.Symbol(buf[:0], idx)
}
}
}
func BenchmarkDecode(b *testing.B) {
msg := randMsg(rand.New(rand.NewPCG(7, 8)), 9800)
enc, err := NewEncoder(msg, 1176)
require.NoError(b, err)
idx := []int{0, 1, 2, 3, 5, 6, 9, 10, 11}
syms := make([][]byte, len(idx))
for i, x := range idx {
syms[i], _ = enc.Symbol(nil, x)
}
b.SetBytes(int64(len(msg)))
for b.Loop() {
dec, _ := NewDecoder(len(msg), 1176)
for i, x := range idx {
_, _ = dec.Add(x, syms[i])
}
if _, err := dec.Decode(); err != nil {
b.Fatal(err)
}
}
}
-115
View File
@@ -1,115 +0,0 @@
package fec
// Arithmetic over GF(2^8) with the reducing polynomial x^8+x^4+x^3+x^2+1 (0x11d).
// Addition is XOR. Multiplication uses log/exp tables; the exp table is doubled so
// the sum of two logs never needs reducing mod 255.
const gfPoly = 0x11d
var (
gfExp [510]byte
gfLog [256]byte
)
func init() {
x := 1
for i := range 255 {
gfExp[i] = byte(x)
gfExp[i+255] = byte(x)
gfLog[x] = byte(i)
x <<= 1
if x&0x100 != 0 {
x ^= gfPoly
}
}
}
func gfMul(a, b byte) byte {
if a == 0 || b == 0 {
return 0
}
return gfExp[int(gfLog[a])+int(gfLog[b])]
}
// gfInv returns the multiplicative inverse of a. a must be non-zero.
func gfInv(a byte) byte {
return gfExp[255-int(gfLog[a])]
}
// mulAdd computes dst ^= c*src element-wise. dst must be at least as long as src.
func mulAdd(dst, src []byte, c byte) {
switch c {
case 0:
return
case 1:
for i, s := range src {
dst[i] ^= s
}
return
}
var t [256]byte
lc := int(gfLog[c])
for s := 1; s < 256; s++ {
t[s] = gfExp[int(gfLog[s])+lc]
}
for i, s := range src {
dst[i] ^= t[s]
}
}
// coefficient returns the generator matrix entry for repair symbol index row and
// data symbol index col. Repair rows are a Cauchy matrix 1/(x_row + y_col) with
// x_row = row and y_col = col: rows are always >= k and columns < k so the two
// sets are disjoint, which makes every square submatrix invertible and the
// systematic code [I; C] MDS. A row depends only on its index, never on how many
// repair symbols exist, so new repair symbols can be generated at any time.
func coefficient(row, col int) byte {
return gfInv(byte(row) ^ byte(col))
}
// invert inverts the n x n matrix m in place with Gauss-Jordan elimination.
// It returns false if m is singular.
func invert(m [][]byte) bool {
n := len(m)
inv := make([][]byte, n)
for i := range inv {
inv[i] = make([]byte, n)
inv[i][i] = 1
}
for col := range n {
pivot := -1
for r := col; r < n; r++ {
if m[r][col] != 0 {
pivot = r
break
}
}
if pivot < 0 {
return false
}
m[col], m[pivot] = m[pivot], m[col]
inv[col], inv[pivot] = inv[pivot], inv[col]
scale := gfInv(m[col][col])
for c := range n {
m[col][c] = gfMul(m[col][c], scale)
inv[col][c] = gfMul(inv[col][c], scale)
}
for r := 0; r < n; r++ {
if r == col || m[r][col] == 0 {
continue
}
f := m[r][col]
for c := range n {
m[r][c] ^= gfMul(f, m[col][c])
inv[r][c] ^= gfMul(f, inv[col][c])
}
}
}
copy(m, inv)
return true
}
-67
View File
@@ -1,67 +0,0 @@
package fragment
import (
"errors"
"github.com/slackhq/nebula/handshake/fec"
"github.com/slackhq/nebula/header"
)
// ErrMismatch is returned when a fragment disagrees with the message it claims to
// belong to. The caller should discard the whole assembly.
var ErrMismatch = errors.New("fragment does not match its message")
// Assembly collects the fragments of one incoming message. Its memory is bounded
// by the message length: at most K symbols are kept however many arrive.
//
// An Assembly is not safe for concurrent use.
type Assembly struct {
fragID uint32
msgLen uint16
dec *fec.Decoder
received int
}
// NewAssembly starts an assembly for the message described by h. The fragment
// itself still has to be passed to Add.
func NewAssembly(h Header) (*Assembly, error) {
dec, err := fec.NewDecoder(int(h.MsgLen), MaxSymbolSize)
if err != nil {
return nil, err
}
return &Assembly{fragID: h.FragID, msgLen: h.MsgLen, dec: dec}, nil
}
// Add records a fragment parsed by Parse. It returns true once enough symbols
// are held to call Decode.
func (a *Assembly) Add(h Header, sym []byte) (bool, error) {
if h.FragID != a.fragID || h.MsgLen != a.msgLen {
return a.dec.Ready(), ErrMismatch
}
a.received += header.Len + HeaderLen + len(sym)
return a.dec.Add(int(h.Index), sym)
}
// wouldStore returns how many bytes Add would retain for this fragment.
func (a *Assembly) wouldStore(h Header, sym []byte) int {
if h.FragID != a.fragID || h.MsgLen != a.msgLen || a.dec.Ready() || a.dec.Has(int(h.Index)) {
return 0
}
return len(sym)
}
// Decode reconstructs the message. It requires Add to have reported ready.
func (a *Assembly) Decode() ([]byte, error) {
return a.dec.Decode()
}
// Stored returns the number of symbol bytes held.
func (a *Assembly) Stored() int {
return a.dec.Stored() * a.dec.Layout().SymbolSize
}
// Received returns the number of packet bytes received for this message,
// duplicates included. It is the basis for the amplification budget.
func (a *Assembly) Received() int {
return a.received
}
-515
View File
@@ -1,515 +0,0 @@
package fragment
import (
"bytes"
"math/rand/v2"
"sync"
"testing"
"time"
"github.com/slackhq/nebula/header"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
const testSubtype = header.MessageSubType(9)
type testKey struct {
src string
fragID uint32
}
func randMsg(r *rand.Rand, n int) []byte {
b := make([]byte, n)
for i := range b {
b[i] = byte(r.Uint32())
}
return b
}
func testConfig() TableConfig {
return TableConfig{
MaxBytes: 1 << 20,
MaxEntries: 64,
MaxPerSource: 4,
MaxMsgLen: 16384,
TTL: 2 * time.Second,
}
}
// checkInvariants verifies the table's bookkeeping against its contents.
func checkInvariants[K, S comparable](t *testing.T, tb *Table[K, S]) {
t.Helper()
tb.mu.Lock()
defer tb.mu.Unlock()
require.Len(t, tb.order, len(tb.entries))
require.LessOrEqual(t, len(tb.entries), tb.cfg.MaxEntries)
bytes := 0
perSource := map[S]int{}
for i, key := range tb.order {
e, ok := tb.entries[key]
require.True(t, ok)
require.Equal(t, i, e.pos)
bytes += e.a.Stored()
perSource[e.source]++
}
require.Equal(t, bytes, tb.bytes)
require.LessOrEqual(t, tb.bytes, tb.cfg.MaxBytes)
require.Equal(t, perSource, tb.perSource)
for _, n := range perSource {
require.LessOrEqual(t, n, tb.cfg.MaxPerSource)
}
}
func parse(t *testing.T, p []byte) (header.H, Header, []byte) {
t.Helper()
var nh header.H
require.NoError(t, nh.Parse(p))
h, sym, err := Parse(p[header.Len:])
require.NoError(t, err)
return nh, h, sym
}
func TestParse(t *testing.T) {
h := Header{FragID: 0xdeadbeef, MsgLen: 3000, Index: 7}
b := h.Encode(nil)
require.Len(t, b, HeaderLen)
b = append(b, make([]byte, 1000)...)
got, sym, err := Parse(b)
require.NoError(t, err)
assert.Equal(t, h, got)
assert.Len(t, sym, 1000)
_, _, err = Parse(b[:HeaderLen-1])
require.ErrorIs(t, err, ErrShortFragment)
_, _, err = Parse(b[:len(b)-1])
require.ErrorIs(t, err, ErrSymbolSize)
_, _, err = Parse(append(b, 0))
require.ErrorIs(t, err, ErrSymbolSize)
bad := Header{FragID: 0, MsgLen: 3000}.Encode(nil)
_, _, err = Parse(append(bad, make([]byte, 1000)...))
require.ErrorIs(t, err, ErrFragID)
bad = Header{FragID: 1, MsgLen: 0}.Encode(nil)
_, _, err = Parse(bad)
require.ErrorIs(t, err, ErrMsgLen)
bad = Header{FragID: 1, MsgLen: 3000, Flags: 0x02}.Encode(nil)
_, _, err = Parse(append(bad, make([]byte, 1000)...))
require.ErrorIs(t, err, ErrFlags)
t.Run("cookie", func(t *testing.T) {
h := Header{FragID: 7, MsgLen: 3000, Index: 4, Flags: FlagCookie}
for i := range h.Cookie {
h.Cookie[i] = byte(i + 1)
}
b := h.Encode(nil)
require.Len(t, b, HeaderLen+CookieLen)
b = append(b, make([]byte, 1000)...)
got, sym, err := Parse(b)
require.NoError(t, err)
assert.Equal(t, h, got)
assert.True(t, got.HasCookie())
assert.Len(t, sym, 1000)
_, _, err = Parse(b[:HeaderLen+CookieLen-1])
require.ErrorIs(t, err, ErrNoCookie)
})
}
func TestDefaultRepair(t *testing.T) {
for k, want := range map[int]int{1: 2, 2: 2, 6: 2, 7: 3, 9: 3, 10: 4, 14: 5} {
assert.Equal(t, want, DefaultRepair(k), "k=%d", k)
}
}
func TestSender(t *testing.T) {
_, err := NewSender(nil, testSubtype, 0, 1, 1)
require.ErrorIs(t, err, ErrMsgLen)
_, err = NewSender(make([]byte, 65536), testSubtype, 0, 1, 1)
require.ErrorIs(t, err, ErrMsgLen)
_, err = NewSender(make([]byte, 10), testSubtype, 0, 1, 0)
require.ErrorIs(t, err, ErrFragID)
msg := randMsg(rand.New(rand.NewPCG(1, 1)), 9800)
s, err := NewSender(msg, testSubtype, 77, 2, 1234)
require.NoError(t, err)
require.Equal(t, 9, s.Layout().K)
indexes := func(ps [][]byte) []int {
var out []int
for _, p := range ps {
require.LessOrEqual(t, len(p), MaxPacketLen)
nh, h, _ := parse(t, p)
assert.Equal(t, header.Handshake, nh.Type)
assert.Equal(t, testSubtype, nh.Subtype)
assert.Equal(t, uint32(77), nh.RemoteIndex)
assert.Equal(t, uint64(2), nh.MessageCounter)
assert.Equal(t, uint32(1234), h.FragID)
assert.Equal(t, uint16(9800), h.MsgLen)
out = append(out, int(h.Index))
}
return out
}
now, later, err := s.Attempt(3)
require.NoError(t, err)
assert.Equal(t, []int{0, 1, 2, 3, 4, 5, 6, 7, 8}, indexes(now))
assert.Equal(t, []int{9, 10, 11}, indexes(later))
// A retry offers only indexes the receiver has not seen.
now, later, err = s.Attempt(3)
require.NoError(t, err)
assert.Equal(t, []int{12, 13, 14, 15, 16, 17, 18, 19, 20}, indexes(now))
assert.Equal(t, []int{21, 22, 23}, indexes(later))
// A sender rebuilt from the same message resumes where it was told to.
s2, err := NewSender(msg, testSubtype, 77, 2, 1234)
require.NoError(t, err)
s2.SetNext(s.Next())
now2, _, err := s2.Attempt(0)
require.NoError(t, err)
assert.Equal(t, []int{24, 25, 26, 27, 28, 29, 30, 31, 32}, indexes(now2))
s2.SetNext(-1)
assert.Equal(t, 255, s2.Next())
// A cookie rides on every later packet, which still fits MaxPacketLen even
// with a full size symbol.
full, err := NewSender(make([]byte, 2*MaxSymbolSize), testSubtype, 0, 1, 9)
require.NoError(t, err)
full.SetCookie(Cookie{1, 2, 3})
pkts, _, err := full.Attempt(1)
require.NoError(t, err)
for _, p := range pkts {
assert.Len(t, p, MaxPacketLen)
_, h, _ := parse(t, p)
assert.True(t, h.HasCookie())
assert.Equal(t, Cookie{1, 2, 3}, h.Cookie)
}
// Indexes wrap once the space is used up.
for range 19 {
_, _, err = s.Attempt(3)
require.NoError(t, err)
}
now, _, err = s.Attempt(3)
require.NoError(t, err)
assert.Equal(t, []int{252, 253, 254, 255, 0, 1, 2, 3, 4}, indexes(now))
}
// TestLossy runs whole messages through heavy loss, reordering and duplication,
// merging across attempts, and checks the table ends empty.
func TestLossy(t *testing.T) {
r := rand.New(rand.NewPCG(2, 2))
tb := NewTable[testKey, string](testConfig())
now := time.Unix(1000, 0)
for i := range 300 {
msg := randMsg(r, 1+r.IntN(16384))
fragID := uint32(i + 1)
s, err := NewSender(msg, testSubtype, 0, 1, fragID)
require.NoError(t, err)
var got *Result
for attempt := 0; got == nil; attempt++ {
require.Less(t, attempt, 20, "message never completed")
a, b, err := s.Attempt(DefaultRepair(s.Layout().K))
require.NoError(t, err)
var pkts [][]byte
for _, p := range append(a, b...) {
if r.Float64() < 0.3 {
continue
}
pkts = append(pkts, p)
if r.Float64() < 0.2 {
pkts = append(pkts, p)
}
}
r.Shuffle(len(pkts), func(i, j int) { pkts[i], pkts[j] = pkts[j], pkts[i] })
for _, p := range pkts {
_, h, sym := parse(t, p)
res, err := tb.Add(testKey{"a", h.FragID}, "a", now, h, sym)
require.NoError(t, err)
if res != nil {
// The table keeps no memory of completed messages, so a late
// duplicate can complete one again; the handshake layer
// recognises the repeat.
require.True(t, bytes.Equal(msg, res.Msg))
got = res
}
checkInvariants(t, tb)
}
now = now.Add(100 * time.Millisecond)
}
require.Positive(t, got.Received)
// Late fragments of a completed message start a fresh entry; drop it.
tb.Expire(now.Add(time.Hour))
}
assert.Zero(t, tb.Len())
assert.Zero(t, tb.Bytes())
}
// TestConcurrent spreads each message's fragments across goroutines, as
// multiple UDP reader routines would.
func TestConcurrent(t *testing.T) {
tb := NewTable[testKey, string](testConfig())
now := time.Unix(1000, 0)
r := rand.New(rand.NewPCG(3, 3))
const msgs = 50
want := make(map[uint32][]byte, msgs)
var pkts [][]byte
for id := uint32(1); id <= msgs; id++ {
msg := randMsg(r, 1+r.IntN(16384))
want[id] = msg
s, err := NewSender(msg, testSubtype, 0, 1, id)
require.NoError(t, err)
a, b, err := s.Attempt(DefaultRepair(s.Layout().K))
require.NoError(t, err)
pkts = append(pkts, a...)
pkts = append(pkts, b...)
}
r.Shuffle(len(pkts), func(i, j int) { pkts[i], pkts[j] = pkts[j], pkts[i] })
var mu sync.Mutex
done := map[uint32]bool{}
var wg sync.WaitGroup
for w := range 8 {
wg.Go(func() {
for i := w; i < len(pkts); i += 8 {
h, sym, err := Parse(pkts[i][16:])
if err != nil {
t.Error(err)
return
}
src := string(rune('a' + h.FragID))
res, err := tb.Add(testKey{src, h.FragID}, src, now, h, sym)
if err != nil {
t.Error(err)
return
}
if res != nil {
if !bytes.Equal(want[h.FragID], res.Msg) {
t.Errorf("message %d decoded wrong", h.FragID)
}
mu.Lock()
done[h.FragID] = true
mu.Unlock()
}
}
})
}
wg.Wait()
assert.Len(t, done, msgs)
checkInvariants(t, tb)
}
func firstFragment(t *testing.T, msgLen int, fragID uint32) (Header, []byte) {
t.Helper()
s, err := NewSender(make([]byte, msgLen), testSubtype, 0, 1, fragID)
require.NoError(t, err)
now, _, err := s.Attempt(0)
require.NoError(t, err)
_, h, sym := parse(t, now[0])
return h, sym
}
func TestTableLimits(t *testing.T) {
now := time.Unix(1000, 0)
t.Run("max message length", func(t *testing.T) {
tb := NewTable[testKey, string](testConfig())
h, sym := firstFragment(t, 16385, 1)
_, err := tb.Add(testKey{"a", 1}, "a", now, h, sym)
require.ErrorIs(t, err, ErrTooLarge)
assert.Zero(t, tb.Len())
})
t.Run("per source quota", func(t *testing.T) {
cfg := testConfig()
cfg.MaxPerSource = 2
tb := NewTable[testKey, string](cfg)
for id := uint32(1); id <= 3; id++ {
h, sym := firstFragment(t, 5000, id)
_, err := tb.Add(testKey{"a", id}, "a", now, h, sym)
if id <= 2 {
require.NoError(t, err)
} else {
require.ErrorIs(t, err, ErrSourceQuota)
}
}
h, sym := firstFragment(t, 5000, 4)
_, err := tb.Add(testKey{"b", 4}, "b", now, h, sym)
require.NoError(t, err)
assert.Equal(t, 3, tb.Len())
checkInvariants(t, tb)
})
t.Run("max entries evicts", func(t *testing.T) {
cfg := testConfig()
cfg.MaxEntries = 4
tb := NewTable[testKey, string](cfg)
for id := uint32(1); id <= 10; id++ {
h, sym := firstFragment(t, 5000, id)
src := string(rune('a' + id))
_, err := tb.Add(testKey{src, id}, src, now, h, sym)
require.NoError(t, err)
checkInvariants(t, tb)
}
assert.Equal(t, 4, tb.Len())
})
t.Run("max bytes evicts others", func(t *testing.T) {
cfg := testConfig()
cfg.MaxBytes = 3 * 1000
tb := NewTable[testKey, string](cfg)
for id := uint32(1); id <= 10; id++ {
h, sym := firstFragment(t, 5000, id) // 5 symbols of 1000
src := string(rune('a' + id))
_, err := tb.Add(testKey{src, id}, src, now, h, sym)
require.NoError(t, err)
checkInvariants(t, tb)
}
assert.Equal(t, 3000, tb.Bytes())
})
t.Run("no room removes new entry", func(t *testing.T) {
cfg := testConfig()
cfg.MaxBytes = 500
tb := NewTable[testKey, string](cfg)
h, sym := firstFragment(t, 5000, 1)
_, err := tb.Add(testKey{"a", 1}, "a", now, h, sym)
require.ErrorIs(t, err, ErrNoRoom)
assert.Zero(t, tb.Len())
checkInvariants(t, tb)
})
t.Run("duplicates are not charged", func(t *testing.T) {
tb := NewTable[testKey, string](testConfig())
h, sym := firstFragment(t, 5000, 1)
for range 5 {
_, err := tb.Add(testKey{"a", 1}, "a", now, h, sym)
require.NoError(t, err)
}
assert.Equal(t, 1000, tb.Bytes())
})
t.Run("mismatch drops the entry", func(t *testing.T) {
tb := NewTable[testKey, string](testConfig())
h, sym := firstFragment(t, 5000, 1)
_, err := tb.Add(testKey{"a", 1}, "a", now, h, sym)
require.NoError(t, err)
h2, sym2 := firstFragment(t, 6000, 1)
_, err = tb.Add(testKey{"a", 1}, "a", now, h2, sym2)
require.ErrorIs(t, err, ErrMismatch)
assert.Zero(t, tb.Len())
assert.Zero(t, tb.Bytes())
checkInvariants(t, tb)
})
t.Run("load", func(t *testing.T) {
cfg := testConfig()
cfg.MaxEntries = 4
cfg.MaxBytes = 10000
tb := NewTable[testKey, string](cfg)
assert.Zero(t, tb.Load())
h, sym := firstFragment(t, 5000, 1)
_, err := tb.Add(testKey{"a", 1}, "a", now, h, sym)
require.NoError(t, err)
assert.InDelta(t, 0.25, tb.Load(), 0.001, "one of four entries")
h, sym = firstFragment(t, 5000, 2)
_, err = tb.Add(testKey{"b", 2}, "b", now, h, sym)
require.NoError(t, err)
assert.InDelta(t, 0.5, tb.Load(), 0.001)
})
t.Run("ttl", func(t *testing.T) {
tb := NewTable[testKey, string](testConfig())
s, err := NewSender(make([]byte, 5000), testSubtype, 0, 1, 1)
require.NoError(t, err)
pkts, _, err := s.Attempt(0)
require.NoError(t, err)
_, h, sym := parse(t, pkts[0])
_, err = tb.Add(testKey{"a", 1}, "a", now, h, sym)
require.NoError(t, err)
// Activity within the TTL keeps the entry and slides its deadline.
_, h, sym = parse(t, pkts[1])
_, err = tb.Add(testKey{"a", 1}, "a", now.Add(1500*time.Millisecond), h, sym)
require.NoError(t, err)
assert.Equal(t, 2000, tb.Bytes())
tb.Expire(now.Add(3 * time.Second))
assert.Equal(t, 1, tb.Len())
// A stale entry is restarted rather than merged.
_, h, sym = parse(t, pkts[2])
_, err = tb.Add(testKey{"a", 1}, "a", now.Add(10*time.Second), h, sym)
require.NoError(t, err)
assert.Equal(t, 1000, tb.Bytes())
tb.Expire(now.Add(time.Minute))
assert.Zero(t, tb.Len())
checkInvariants(t, tb)
})
}
func FuzzParse(f *testing.F) {
f.Add(append(Header{FragID: 1, MsgLen: 3000}.Encode(nil), make([]byte, 1000)...))
f.Add([]byte{0, 0, 0, 1, 0, 1, 0, 0, 7})
f.Fuzz(func(t *testing.T, b []byte) {
h, sym, err := Parse(b)
if err != nil {
return
}
require.Equal(t, b, append(h.Encode(nil), sym...))
l, err := h.Layout()
require.NoError(t, err)
require.LessOrEqual(t, l.SymbolSize, MaxSymbolSize)
})
}
// FuzzTable drives a small table with arbitrary fragments and checks its
// bookkeeping after every step.
func FuzzTable(f *testing.F) {
f.Add([]byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12})
f.Fuzz(func(t *testing.T, ops []byte) {
tb := NewTable[testKey, string](TableConfig{
MaxBytes: 4000,
MaxEntries: 3,
MaxPerSource: 2,
MaxMsgLen: 6000,
TTL: time.Second,
})
now := time.Unix(0, 0)
for len(ops) >= 4 {
src := string(rune('a' + ops[0]%4))
fragID := uint32(ops[1]%5) + 1
msgLen := uint16(ops[2])*32 + 1
index := ops[3]
ops = ops[4:]
h := Header{FragID: fragID, MsgLen: msgLen, Index: index}
l, err := h.Layout()
require.NoError(t, err)
sym := make([]byte, l.SymbolSize)
if index%7 == 0 {
now = now.Add(600 * time.Millisecond)
tb.Expire(now)
}
res, err := tb.Add(testKey{src, fragID}, src, now, h, sym)
if res != nil {
require.NoError(t, err)
require.Len(t, res.Msg, int(msgLen))
}
checkInvariants(t, tb)
}
})
}
-118
View File
@@ -1,118 +0,0 @@
package fragment
import (
"errors"
"math"
"github.com/slackhq/nebula/handshake/fec"
"github.com/slackhq/nebula/header"
)
// DefaultRepair is the starting number of repair symbols per attempt for a
// message of k data symbols. Small messages get proportionally more repair.
func DefaultRepair(k int) int {
return max(2, (k+2)/3)
}
// Sender produces the fragment packets of one outgoing handshake message.
//
// The first attempt sends the K data symbols followed by repair symbols. Every
// later attempt sends symbols with indexes the receiver has not been offered yet,
// so whatever it already holds still counts toward K. Symbols are a deterministic
// function of the message, which keeps retransmits idempotent. Once all
// fec.MaxSymbols indexes are used the sequence wraps and repeats.
//
// A Sender is not safe for concurrent use.
type Sender struct {
enc *fec.Encoder
subtype header.MessageSubType
remoteIndex uint32
counter uint64
fragID uint32
next int
cookie *Cookie
}
// NewSender prepares msg for sending. subtype, remoteIndex and counter fill the
// Nebula header of every fragment; fragID is the sender's local index.
func NewSender(msg []byte, subtype header.MessageSubType, remoteIndex uint32, counter uint64, fragID uint32) (*Sender, error) {
if len(msg) == 0 || len(msg) > math.MaxUint16 {
return nil, ErrMsgLen
}
if fragID == 0 {
return nil, ErrFragID
}
enc, err := fec.NewEncoder(msg, MaxSymbolSize)
if err != nil {
return nil, err
}
return &Sender{
enc: enc,
subtype: subtype,
remoteIndex: remoteIndex,
counter: counter,
fragID: fragID,
}, nil
}
func (s *Sender) Layout() fec.Layout {
return s.enc.Layout()
}
// SetNext sets the index the next attempt starts from, so a sender rebuilt from
// a cached message can carry on offering fresh symbols.
func (s *Sender) SetNext(index int) {
s.next = ((index % fec.MaxSymbols) + fec.MaxSymbols) % fec.MaxSymbols
}
// SetCookie attaches a cookie from a responder under load to every later packet.
func (s *Sender) SetCookie(c Cookie) {
s.cookie = &c
}
// Next returns the index the next attempt starts from.
func (s *Sender) Next() int {
return s.next
}
// Attempt returns the packets for one transmission attempt carrying K+repair
// symbols. now should be sent immediately and later after the repair delay, so
// a tail drop at the end of the first burst does not also take the repair.
func (s *Sender) Attempt(repair int) (now, later [][]byte, err error) {
if repair < 0 {
return nil, nil, errors.New("negative repair count")
}
k := s.enc.Layout().K
now = make([][]byte, 0, k)
later = make([][]byte, 0, repair)
for i := range k + repair {
p, err := s.packet(s.next)
if err != nil {
return nil, nil, err
}
s.next = (s.next + 1) % fec.MaxSymbols
if i < k {
now = append(now, p)
} else {
later = append(later, p)
}
}
return now, later, nil
}
func (s *Sender) packet(index int) ([]byte, error) {
l := s.enc.Layout()
p := make([]byte, header.Len, header.Len+HeaderLen+CookieLen+l.SymbolSize)
header.Encode(p, header.Version, header.Handshake, s.subtype, s.remoteIndex, s.counter)
h := Header{
FragID: s.fragID,
MsgLen: uint16(l.MsgLen),
Index: uint8(index),
}
if s.cookie != nil {
h.Flags |= FlagCookie
h.Cookie = *s.cookie
}
p = h.Encode(p)
return s.enc.Symbol(p, index)
}
-220
View File
@@ -1,220 +0,0 @@
package fragment
import (
"errors"
"math/rand/v2"
"sync"
"time"
)
var (
ErrTooLarge = errors.New("fragmented message exceeds the size limit")
ErrSourceQuota = errors.New("source has too many messages in progress")
ErrNoRoom = errors.New("no room for fragment")
)
// TableConfig bounds a Table.
type TableConfig struct {
MaxBytes int // symbol bytes held across all entries
MaxEntries int // messages in progress across all sources
MaxPerSource int // messages in progress per source
MaxMsgLen int // largest message accepted
TTL time.Duration // an entry idle this long is dropped
}
// Table reassembles messages from senders that have not authenticated yet, the
// responder's view of msg1. Every resource is capped. When a cap is hit a
// random entry is evicted; random eviction is harder to aim than LRU, which a
// flood can use to flush specific entries.
//
// K identifies one message, typically the source plus FragID. S identifies the
// source for quotas, typically the address masked to /32 or /64.
//
// A Table is safe for concurrent use.
type Table[K, S comparable] struct {
mu sync.Mutex
cfg TableConfig
entries map[K]*tableEntry[K, S]
order []K
perSource map[S]int
bytes int
}
type tableEntry[K, S comparable] struct {
a *Assembly
source S
lastSeen time.Time
pos int
}
func NewTable[K, S comparable](cfg TableConfig) *Table[K, S] {
return &Table[K, S]{
cfg: cfg,
entries: make(map[K]*tableEntry[K, S]),
perSource: make(map[S]int),
}
}
// Result describes a completed message.
type Result struct {
Msg []byte // the decoded message
Received int // packet bytes received for it, for the amplification budget
}
// Add records a fragment parsed by Parse. When the message is complete its entry
// is removed and the result returned; otherwise the result is nil. A decode
// error means the message is lost and must be retransmitted.
func (t *Table[K, S]) Add(key K, source S, now time.Time, h Header, sym []byte) (*Result, error) {
if int(h.MsgLen) > t.cfg.MaxMsgLen {
return nil, ErrTooLarge
}
t.mu.Lock()
e, ok := t.entries[key]
if ok && now.Sub(e.lastSeen) > t.cfg.TTL {
t.remove(key, e)
ok = false
}
if !ok {
var err error
e, err = t.insert(key, source, h)
if err != nil {
t.mu.Unlock()
return nil, err
}
}
e.lastSeen = now
if !t.makeRoom(key, e.a.wouldStore(h, sym)) {
if e.a.Stored() == 0 {
t.remove(key, e)
}
t.mu.Unlock()
return nil, ErrNoRoom
}
before := e.a.Stored()
ready, err := e.a.Add(h, sym)
t.bytes += e.a.Stored() - before
if err != nil || !ready {
if errors.Is(err, ErrMismatch) || (err != nil && e.a.Stored() == 0) {
t.remove(key, e)
}
t.mu.Unlock()
return nil, err
}
t.remove(key, e)
t.mu.Unlock()
msg, err := e.a.Decode()
if err != nil {
return nil, err
}
return &Result{Msg: msg, Received: e.a.Received()}, nil
}
// Expire drops every entry idle for longer than the TTL.
func (t *Table[K, S]) Expire(now time.Time) {
t.mu.Lock()
defer t.mu.Unlock()
for i := 0; i < len(t.order); {
key := t.order[i]
e := t.entries[key]
if now.Sub(e.lastSeen) > t.cfg.TTL {
// remove swaps the last entry into position i, so look at i again.
t.remove(key, e)
continue
}
i++
}
}
// Len returns the number of messages in progress.
func (t *Table[K, S]) Len() int {
t.mu.Lock()
defer t.mu.Unlock()
return len(t.entries)
}
// Load returns how full the table is, from 0 to 1, by whichever of its byte and
// entry limits is closer.
func (t *Table[K, S]) Load() float64 {
t.mu.Lock()
defer t.mu.Unlock()
return max(float64(t.bytes)/float64(t.cfg.MaxBytes), float64(len(t.entries))/float64(t.cfg.MaxEntries))
}
// Bytes returns the number of symbol bytes held.
func (t *Table[K, S]) Bytes() int {
t.mu.Lock()
defer t.mu.Unlock()
return t.bytes
}
func (t *Table[K, S]) insert(key K, source S, h Header) (*tableEntry[K, S], error) {
if t.perSource[source] >= t.cfg.MaxPerSource {
return nil, ErrSourceQuota
}
a, err := NewAssembly(h)
if err != nil {
return nil, err
}
for len(t.entries) >= t.cfg.MaxEntries {
if !t.evictRandom(nil) {
return nil, ErrNoRoom
}
}
e := &tableEntry[K, S]{a: a, source: source, pos: len(t.order)}
t.entries[key] = e
t.order = append(t.order, key)
t.perSource[source]++
return e, nil
}
// makeRoom evicts random entries other than keep until n more bytes fit.
func (t *Table[K, S]) makeRoom(keep K, n int) bool {
for t.bytes+n > t.cfg.MaxBytes {
if !t.evictRandom(&keep) {
return false
}
}
return true
}
func (t *Table[K, S]) evictRandom(keep *K) bool {
candidates := len(t.order)
if keep != nil {
if _, ok := t.entries[*keep]; ok {
candidates--
}
}
if candidates <= 0 {
return false
}
for {
key := t.order[rand.IntN(len(t.order))]
if keep != nil && key == *keep {
continue
}
t.remove(key, t.entries[key])
return true
}
}
func (t *Table[K, S]) remove(key K, e *tableEntry[K, S]) {
last := len(t.order) - 1
moved := t.order[last]
t.order[e.pos] = moved
t.entries[moved].pos = e.pos
t.order = t.order[:last]
delete(t.entries, key)
t.bytes -= e.a.Stored()
if t.perSource[e.source] <= 1 {
delete(t.perSource, e.source)
} else {
t.perSource[e.source]--
}
}
-121
View File
@@ -1,121 +0,0 @@
// Package fragment carries handshake messages that do not fit in one UDP
// packet. A message is Reed-Solomon coded into equally sized symbols (see
// handshake/fec) and each symbol travels in its own packet:
//
// Nebula header (16) | Fragment header (8) | [cookie (16)] | symbol
//
// The fragment header is FragID u32 | MsgLen u16 | Index u8 | Flags u8, all big
// endian. Both ends derive the symbol layout from MsgLen, so any MsgLen-derived
// value never has to be sent. FlagCookie marks a cookie a responder under load
// asked for; room for it is always reserved, so a fragment never outgrows
// MaxPacketLen.
package fragment
import (
"encoding/binary"
"errors"
"github.com/slackhq/nebula/handshake/fec"
"github.com/slackhq/nebula/header"
)
const (
// HeaderLen is the size of the fragment header that follows the Nebula header.
HeaderLen = 8
// CookieLen is the size of the cookie a fragment carries when FlagCookie is set.
CookieLen = 16
// FlagCookie marks a fragment that carries a cookie after its header.
FlagCookie uint8 = 0x01
// MaxPacketLen bounds every fragment packet, Nebula header included. A relayed
// fragment gains a 16 byte header and a 16 byte tag, which lands exactly on
// the 1232 byte UDP payload available on an IPv6 minimum MTU path.
MaxPacketLen = 1200
// MaxSymbolSize is the largest symbol that fits in MaxPacketLen alongside a cookie.
MaxSymbolSize = MaxPacketLen - header.Len - HeaderLen - CookieLen
)
var (
ErrShortFragment = errors.New("fragment is too short")
ErrFragID = errors.New("fragment has a zero FragID")
ErrMsgLen = errors.New("fragment has a zero MsgLen")
ErrFlags = errors.New("fragment has reserved flags set")
ErrNoCookie = errors.New("fragment is too short for its cookie")
ErrSymbolSize = errors.New("fragment symbol size does not match its MsgLen")
)
// Cookie proves a sender received a responder's reply at its address.
type Cookie [CookieLen]byte
// Header is the fragment header.
type Header struct {
FragID uint32 // the sender's local index for this handshake, never zero
MsgLen uint16 // length of the whole message
Index uint8 // symbol index, data below K and repair from K
Flags uint8 // FlagCookie, other bits reserved and zero
Cookie Cookie // valid when Flags has FlagCookie
}
func (h Header) HasCookie() bool {
return h.Flags&FlagCookie != 0
}
// Encode appends the header, and its cookie if it has one, to b.
func (h Header) Encode(b []byte) []byte {
b = binary.BigEndian.AppendUint32(b, h.FragID)
b = binary.BigEndian.AppendUint16(b, h.MsgLen)
b = append(b, h.Index, h.Flags)
if h.HasCookie() {
b = append(b, h.Cookie[:]...)
}
return b
}
// Layout returns the symbol layout the header's MsgLen implies.
func (h Header) Layout() (fec.Layout, error) {
return fec.NewLayout(int(h.MsgLen), MaxSymbolSize)
}
// Parse validates the bytes following the Nebula header and splits them into
// the fragment header and the symbol. The symbol aliases b.
func Parse(b []byte) (Header, []byte, error) {
if len(b) < HeaderLen {
return Header{}, nil, ErrShortFragment
}
h := Header{
FragID: binary.BigEndian.Uint32(b[0:4]),
MsgLen: binary.BigEndian.Uint16(b[4:6]),
Index: b[6],
Flags: b[7],
}
if h.FragID == 0 {
return Header{}, nil, ErrFragID
}
if h.MsgLen == 0 {
return Header{}, nil, ErrMsgLen
}
if h.Flags&^FlagCookie != 0 {
return Header{}, nil, ErrFlags
}
b = b[HeaderLen:]
if h.HasCookie() {
if len(b) < CookieLen {
return Header{}, nil, ErrNoCookie
}
copy(h.Cookie[:], b)
b = b[CookieLen:]
}
l, err := h.Layout()
if err != nil {
return Header{}, nil, err
}
sym := b
if len(sym) != l.SymbolSize {
return Header{}, nil, ErrSymbolSize
}
return h, sym, nil
}
-3
View File
@@ -24,9 +24,6 @@ message NebulaHandshakeDetails {
uint64 Cookie = 4 [deprecated = true];
uint64 Time = 5;
uint32 CertVersion = 8;
// KEMKey is the initiator's ephemeral ML-KEM-1024 encapsulation key. It is
// only present in msg1 of the ix_psk2_mlkem1024 subtype.
bytes KEMKey = 9;
// reserved for WIP multiport
reserved 6, 7;
}
-68
View File
@@ -1,68 +0,0 @@
package handshake
import (
"crypto/hkdf"
"crypto/mlkem"
"crypto/sha256"
"errors"
"fmt"
)
// kemPSKSalt separates the psk derivation from any other use of the secret.
const kemPSKSalt = "nebula ix_psk2_mlkem1024 v1"
var ErrInvalidKEMKey = errors.New("peer sent an invalid ML-KEM encapsulation key")
// kemPSK derives the noise psk from an ML-KEM shared secret. The ciphertext is
// bound in because, unlike the encapsulation key, it travels outside the noise
// transcript.
func kemPSK(sharedKey, ciphertext []byte) ([]byte, error) {
secret := make([]byte, 0, len(sharedKey)+len(ciphertext))
secret = append(secret, sharedKey...)
secret = append(secret, ciphertext...)
return hkdf.Extract(sha256.New, secret, []byte(kemPSKSalt))
}
// kemEncapsulate is the responder half: encapsulate to the initiator's key,
// install the psk, and remember the ciphertext to prefix the response with.
func (m *Machine) kemEncapsulate(encapsulationKey []byte) error {
ek, err := mlkem.NewEncapsulationKey1024(encapsulationKey)
if err != nil {
return fmt.Errorf("%w: %w", ErrInvalidKEMKey, err)
}
sharedKey, ct := ek.Encapsulate()
psk, err := kemPSK(sharedKey, ct)
if err != nil {
return err
}
if err := m.hs.SetPresharedKey(psk); err != nil {
return err
}
m.kemCiphertext = ct
return nil
}
// kemDecapsulate is the initiator half: recover the shared secret from the
// ciphertext prefixing the response and install the psk. It returns the rest of
// the message, which is the noise message.
func (m *Machine) kemDecapsulate(msg []byte) ([]byte, error) {
if len(msg) < mlkem.CiphertextSize1024 {
return nil, ErrPacketTooShort
}
if m.kemKey == nil {
return nil, errors.New("no ML-KEM decapsulation key for this handshake")
}
ct := msg[:mlkem.CiphertextSize1024]
sharedKey, err := m.kemKey.Decapsulate(ct)
if err != nil {
return nil, err
}
psk, err := kemPSK(sharedKey, ct)
if err != nil {
return nil, err
}
if err := m.hs.SetPresharedKey(psk); err != nil {
return nil, err
}
return msg[mlkem.CiphertextSize1024:], nil
}
-208
View File
@@ -1,208 +0,0 @@
package handshake
import (
"bytes"
"crypto/mlkem"
"net/netip"
"testing"
"time"
"github.com/flynn/noise"
"github.com/slackhq/nebula/cert"
ct "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/header"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func newKEMMachine(t *testing.T, cs *testCertState, v CertVerifier, initiator bool, index uint32) *Machine {
t.Helper()
m, err := NewMachine(
cs.version, cs.getCredential, v,
func() (uint32, error) { return index, nil },
initiator, header.HandshakeIXPSK2MLKEM1024,
)
require.NoError(t, err)
return m
}
type kemFixture struct {
initCS, respCS *testCertState
v CertVerifier
}
func newKEMFixture(t *testing.T, cipher noise.CipherFunc) kemFixture {
ca, _, caKey, _ := ct.NewTestCaCert(
cert.Version2, cert.Curve_CURVE25519, time.Time{}, time.Time{}, nil, nil, nil,
)
return kemFixture{
initCS: newTestCertStateWithCipher(t, ca, caKey, "initiator", []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}, cipher),
respCS: newTestCertStateWithCipher(t, ca, caKey, "responder", []netip.Prefix{netip.MustParsePrefix("10.0.0.2/24")}, cipher),
v: testVerifier(ct.NewTestCAPool(ca)),
}
}
func requireKeysMatch(t *testing.T, a, b *Result) {
t.Helper()
c, err := a.EKey.Encrypt(nil, nil, []byte("hello"))
require.NoError(t, err)
p, err := b.DKey.Decrypt(nil, nil, c)
require.NoError(t, err)
assert.Equal(t, []byte("hello"), p)
c, err = b.EKey.Encrypt(nil, nil, []byte("world"))
require.NoError(t, err)
p, err = a.DKey.Decrypt(nil, nil, c)
require.NoError(t, err)
assert.Equal(t, []byte("world"), p)
}
func TestKEMHandshake(t *testing.T) {
for _, cipher := range []noise.CipherFunc{noise.CipherChaChaPoly, noise.CipherAESGCM} {
t.Run(cipher.CipherName(), func(t *testing.T) {
f := newKEMFixture(t, cipher)
initM := newKEMMachine(t, f.initCS, f.v, true, 1000)
respM := newKEMMachine(t, f.respCS, f.v, false, 2000)
msg1, err := initM.Initiate(nil)
require.NoError(t, err)
assert.Equal(t, header.HandshakeIXPSK2MLKEM1024, header.MessageSubType(msg1[1]))
assert.Greater(t, len(msg1), header.Len+mlkem.EncapsulationKeySize1024)
msg2, respR, err := respM.ProcessPacket(nil, msg1)
require.NoError(t, err)
require.NotNil(t, respR)
assert.Greater(t, len(msg2), header.Len+mlkem.CiphertextSize1024)
_, initR, err := initM.ProcessPacket(nil, msg2)
require.NoError(t, err)
require.NotNil(t, initR)
assert.Equal(t, "responder", initR.RemoteCert.Certificate.Name())
assert.Equal(t, "initiator", respR.RemoteCert.Certificate.Name())
assert.Equal(t, uint32(2000), initR.RemoteIndex)
assert.Equal(t, uint32(1000), respR.RemoteIndex)
requireKeysMatch(t, initR, respR)
})
}
}
// TestKEMSecretMatters checks the ML-KEM secret really reaches the transport
// keys: an initiator holding a different decapsulation key cannot complete even
// though every DH input is genuine.
func TestKEMSecretMatters(t *testing.T) {
f := newKEMFixture(t, noise.CipherChaChaPoly)
initM := newKEMMachine(t, f.initCS, f.v, true, 1000)
respM := newKEMMachine(t, f.respCS, f.v, false, 2000)
msg1, err := initM.Initiate(nil)
require.NoError(t, err)
msg2, _, err := respM.ProcessPacket(nil, msg1)
require.NoError(t, err)
other, err := mlkem.GenerateKey1024()
require.NoError(t, err)
initM.kemKey = other
_, res, err := initM.ProcessPacket(nil, msg2)
require.Error(t, err)
assert.Nil(t, res)
}
func TestKEMTamperedCiphertext(t *testing.T) {
f := newKEMFixture(t, noise.CipherChaChaPoly)
initM := newKEMMachine(t, f.initCS, f.v, true, 1000)
respM := newKEMMachine(t, f.respCS, f.v, false, 2000)
msg1, err := initM.Initiate(nil)
require.NoError(t, err)
msg2, respR, err := respM.ProcessPacket(nil, msg1)
require.NoError(t, err)
t.Run("truncated", func(t *testing.T) {
_, res, err := initM.ProcessPacket(nil, msg2[:header.Len+mlkem.CiphertextSize1024-1])
require.ErrorIs(t, err, ErrPacketTooShort)
assert.Nil(t, res)
assert.False(t, initM.Failed())
})
t.Run("flipped ciphertext bit", func(t *testing.T) {
bad := bytes.Clone(msg2)
bad[header.Len+10] ^= 1
_, res, err := initM.ProcessPacket(nil, bad)
require.Error(t, err)
assert.Nil(t, res)
assert.False(t, initM.Failed(), "a forged ciphertext must not kill the handshake")
})
t.Run("genuine response still completes", func(t *testing.T) {
_, initR, err := initM.ProcessPacket(nil, msg2)
require.NoError(t, err)
require.NotNil(t, initR)
requireKeysMatch(t, initR, respR)
})
}
// TestKEMBadMsg1 drives processPayload by hand. With the psk modifier msg1's
// payload is encrypted under a key derived from the public ephemeral, so a
// forged payload would need re-encrypting; the parsing rules are what matter.
func TestKEMBadMsg1(t *testing.T) {
f := newKEMFixture(t, noise.CipherChaChaPoly)
msg1Flags := subtypeInfos[header.HandshakeIXPSK2MLKEM1024].msgs[0]
cert := f.initCS.creds[cert.Version2]
payload := func(kemKey []byte) []byte {
return MarshalPayload(nil, Payload{
Cert: cert.Bytes,
CertVersion: uint32(cert.Cert.Version()),
InitiatorIndex: 1000,
Time: 1,
KEMKey: kemKey,
})
}
dk, err := mlkem.GenerateKey1024()
require.NoError(t, err)
t.Run("missing KEM key", func(t *testing.T) {
m := newKEMMachine(t, f.respCS, f.v, false, 2000)
err := m.processPayload(payload(nil), msg1Flags)
require.ErrorIs(t, err, ErrUnexpectedContent)
assert.True(t, m.Failed())
})
t.Run("invalid KEM key", func(t *testing.T) {
m := newKEMMachine(t, f.respCS, f.v, false, 2000)
// Coefficients of 0xfff exceed the modulus, which ML-KEM must reject.
bad := bytes.Repeat([]byte{0xff}, mlkem.EncapsulationKeySize1024)
err := m.kemEncapsulate(bad)
require.ErrorIs(t, err, ErrInvalidKEMKey)
err = m.kemEncapsulate(bad[:100])
require.ErrorIs(t, err, ErrInvalidKEMKey)
})
t.Run("valid KEM key", func(t *testing.T) {
m := newKEMMachine(t, f.respCS, f.v, false, 2000)
require.NoError(t, m.kemEncapsulate(dk.EncapsulationKey().Bytes()))
assert.Len(t, m.kemCiphertext, mlkem.CiphertextSize1024)
})
t.Run("KEM key on the classical subtype", func(t *testing.T) {
m := newTestMachine(t, f.respCS, f.v, false, 2000)
classical := subtypeInfos[header.HandshakeIXPSK0].msgs[0]
err := m.processPayload(payload(dk.EncapsulationKey().Bytes()), classical)
require.ErrorIs(t, err, ErrUnexpectedContent)
assert.True(t, m.Failed())
})
}
// TestKEMDistinctFromClassical makes sure the two subtypes cannot be mixed: a
// classical responder cannot answer a KEM initiator.
func TestKEMDistinctFromClassical(t *testing.T) {
f := newKEMFixture(t, noise.CipherChaChaPoly)
initM := newKEMMachine(t, f.initCS, f.v, true, 1000)
respM := newTestMachine(t, f.respCS, f.v, false, 2000)
msg1, err := initM.Initiate(nil)
require.NoError(t, err)
_, _, err = respM.ProcessPacket(nil, msg1)
require.ErrorIs(t, err, ErrSubtypeMismatch)
}
+5 -54
View File
@@ -2,8 +2,6 @@ package handshake
import (
"bytes"
"crypto/mlkem"
"errors"
"fmt"
"slices"
"time"
@@ -69,9 +67,6 @@ type Machine struct {
remoteCertSet bool
payloadSet bool
failed bool
kemKey *mlkem.DecapsulationKey1024 // initiator's ephemeral ML-KEM key, until completion
kemCiphertext []byte // responder's ciphertext to prefix its response with
}
// NewMachine creates a handshake state machine. The subtype determines both
@@ -96,7 +91,7 @@ func NewMachine(
return nil, fmt.Errorf("%w: %v", ErrNoCredential, version)
}
hs, err := cred.buildHandshakeState(initiator, info.pattern, info.pskPlacement)
hs, err := cred.buildHandshakeState(initiator, info.pattern)
if err != nil {
return nil, fmt.Errorf("build noise state: %w", err)
}
@@ -142,8 +137,7 @@ func (m *Machine) requireComplete() error {
return nil
}
// myMsgFlags returns the flags for the current outgoing message, which are
// also the flags of the next incoming message before it is read.
// myMsgFlags returns the flags for the current outgoing message.
func (m *Machine) myMsgFlags() msgFlags {
idx := m.hs.MessageIndex()
if idx < len(m.msgs) {
@@ -231,18 +225,7 @@ func (m *Machine) ProcessPacket(out, packet []byte) ([]byte, *Result, error) {
// noise returns (cs1, cs2) where cs1 is the initiator->responder cipher.
// For 3-message patterns where a responder finishes by reading the final
// message, this ordering would be wrong; revisit when XX/pqIX lands.
noiseMsg := packet[header.Len:]
if m.myMsgFlags().kemCiphertext {
var err error
// Installing a psk derived from a forged ciphertext is harmless: ReadMessage
// then fails without advancing, and the next packet installs its own.
noiseMsg, err = m.kemDecapsulate(noiseMsg)
if err != nil {
return nil, nil, err
}
}
msg, eKey, dKey, err := m.hs.ReadMessage(nil, noiseMsg)
msg, eKey, dKey, err := m.hs.ReadMessage(nil, packet[header.Len:])
if err != nil {
// Noise ReadMessage failed. The noise library checkpoints and rolls back
// on failure, so the Machine is still alive. The caller can retry with
@@ -293,8 +276,6 @@ func (m *Machine) ProcessPacket(out, packet []byte) ([]byte, *Result, error) {
}
func (m *Machine) completed(eKey, dKey *noise.CipherState) *Result {
m.kemKey = nil
m.kemCiphertext = nil
m.result.EKey = eKey
m.result.DKey = dKey
m.result.MessageIndex = uint64(m.hs.MessageIndex())
@@ -303,7 +284,7 @@ func (m *Machine) completed(eKey, dKey *noise.CipherState) *Result {
func (m *Machine) processPayload(msg []byte, flags msgFlags) error {
if len(msg) == 0 {
if flags.expectsPayload || flags.expectsCert || flags.kemKey {
if flags.expectsPayload || flags.expectsCert {
m.failed = true
return ErrMissingContent
}
@@ -329,12 +310,6 @@ func (m *Machine) processPayload(msg []byte, flags msgFlags) error {
return ErrUnexpectedContent
}
hasKEMKey := len(payload.KEMKey) > 0
if hasKEMKey != flags.kemKey {
m.failed = true
return ErrUnexpectedContent
}
// Process payload
if flags.expectsPayload {
var remoteIndex uint32
@@ -361,13 +336,6 @@ func (m *Machine) processPayload(msg []byte, flags msgFlags) error {
}
}
if flags.kemKey {
if err := m.kemEncapsulate(payload.KEMKey); err != nil {
m.failed = true
return err
}
}
return nil
}
@@ -412,21 +380,11 @@ func (m *Machine) validateCert(payload Payload) error {
}
func (m *Machine) marshalOutgoing(flags msgFlags) ([]byte, error) {
if !flags.expectsPayload && !flags.expectsCert && !flags.kemKey {
if !flags.expectsPayload && !flags.expectsCert {
return nil, nil
}
var p Payload
if flags.kemKey {
if m.kemKey == nil {
dk, err := mlkem.GenerateKey1024()
if err != nil {
return nil, fmt.Errorf("generate ML-KEM key: %w", err)
}
m.kemKey = dk
}
p.KEMKey = m.kemKey.EncapsulationKey().Bytes()
}
if flags.expectsPayload {
if !m.indexAllocated {
index, err := m.allocIndex()
@@ -478,13 +436,6 @@ func (m *Machine) buildResponse(out []byte) ([]byte, *noise.CipherState, *noise.
uint64(m.hs.MessageIndex()+1),
)
if flags.kemCiphertext {
if m.kemCiphertext == nil {
return nil, nil, nil, errors.New("no ML-KEM ciphertext to send")
}
out = append(out, m.kemCiphertext...)
}
// noise.WriteMessage appends the encrypted handshake message to out,
// reusing capacity when present.
//
+2 -21
View File
@@ -11,16 +11,13 @@ import (
type msgFlags struct {
expectsPayload bool // message carries indexes and time
expectsCert bool // message carries the certificate
kemKey bool // payload carries an ML-KEM encapsulation key
kemCiphertext bool // message is prefixed with the ML-KEM ciphertext for the peer's key
}
// subtypeInfo bundles the noise pattern with the per-message flags for a
// given handshake subtype.
type subtypeInfo struct {
pattern noise.HandshakePattern
pskPlacement int // noise psk modifier position, 0 means no psk token
msgs []msgFlags
pattern noise.HandshakePattern
msgs []msgFlags
}
// subtypeInfos defines the noise pattern and message content layout for each
@@ -35,22 +32,6 @@ var subtypeInfos = map[header.MessageSubType]subtypeInfo{
},
},
// IX with an ephemeral ML-KEM-1024 exchange:
// msg1 (I->R): payload + cert + encapsulation key
// msg2 (R->I): ciphertext || noise(payload + cert)
// The responder encapsulates after reading msg1 and mixes the shared secret in
// through the psk2 token at the end of msg2. The ciphertext travels outside the
// noise message because the initiator needs the secret before it can process
// that token.
header.HandshakeIXPSK2MLKEM1024: {
pattern: noise.HandshakeIX,
pskPlacement: 2,
msgs: []msgFlags{
{expectsPayload: true, expectsCert: true, kemKey: true},
{expectsPayload: true, expectsCert: true, kemCiphertext: true},
},
},
// XX: 3 messages
// msg1 (I->R): payload only
// msg2 (R->I): payload + cert
-16
View File
@@ -20,7 +20,6 @@ type Payload struct {
ResponderIndex uint32
Time uint64
CertVersion uint32
KEMKey []byte // ML-KEM encapsulation key, only in subtypes that carry one
}
// Proto field numbers for NebulaHandshakeDetails
@@ -30,7 +29,6 @@ const (
fieldResponderIndex = 3 // uint32
fieldTime = 5 // uint64
fieldCertVersion = 8 // uint32
fieldKEMKey = 9 // bytes
)
// MarshalPayload encodes a handshake payload in protobuf wire format compatible
@@ -59,10 +57,6 @@ func MarshalPayload(out []byte, p Payload) []byte {
details = protowire.AppendTag(details, fieldCertVersion, protowire.VarintType)
details = protowire.AppendVarint(details, uint64(p.CertVersion))
}
if len(p.KEMKey) > 0 {
details = protowire.AppendTag(details, fieldKEMKey, protowire.BytesType)
details = protowire.AppendBytes(details, p.KEMKey)
}
out = protowire.AppendTag(out, 1, protowire.BytesType)
out = protowire.AppendBytes(out, details)
@@ -167,16 +161,6 @@ func unmarshalPayloadDetails(p *Payload, b []byte) error {
}
p.CertVersion = uint32(v)
b = b[n:]
case fieldKEMKey:
if typ != protowire.BytesType {
return errInvalidHandshakeDetails
}
v, n := protowire.ConsumeBytes(b)
if n < 0 {
return errInvalidHandshakeDetails
}
p.KEMKey = append([]byte(nil), v...)
b = b[n:]
default:
n := protowire.ConsumeFieldValue(num, typ, b)
if n < 0 {
+60 -162
View File
@@ -10,15 +10,12 @@ import (
"net/netip"
"slices"
"sync"
"sync/atomic"
"time"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/handshake"
"github.com/slackhq/nebula/handshake/fragment"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/ratelimit"
"github.com/slackhq/nebula/udp"
)
@@ -52,14 +49,12 @@ type HandshakeConfig struct {
tryInterval time.Duration
retries int64
triggerBuffer int
pq pqHandshakeConfig
messageMetrics *MessageMetrics
}
type HandshakeManager struct {
// Mutex for interacting with the vpnIps and indexes maps.
// Lock order is hostmap, then this lock, then the lighthouse, then a RemoteList.
// Mutex for interacting with the vpnIps and indexes maps
sync.RWMutex
vpnIps map[netip.Addr]*HandshakeHostInfo
@@ -78,18 +73,6 @@ type HandshakeManager struct {
// can be used to trigger outbound handshake for the given vpnIp
trigger chan netip.Addr
// pq holds the post-quantum settings, which change on reload.
pq atomic.Pointer[pqHandshakeConfig]
// pqTable reassembles post-quantum msg1s from senders that have not authenticated yet.
pqTable *fragment.Table[pqFragKey, pqFragSource]
// pqPeers holds vpn addresses that completed a post-quantum handshake, for sticky mode.
pqPeers sync.Map
// pqAnswered recognises repeats of post-quantum msg1s already answered.
pqAnswered *pqAnswered
// pqCookies and pqCookieLimiter serve cookies while pqTable is under load.
pqCookies *pqCookies
pqCookieLimiter *ratelimit.Limiter
}
type HandshakeHostInfo struct {
@@ -105,8 +88,6 @@ type HandshakeHostInfo struct {
hostinfo *HostInfo
machine *handshake.Machine // The handshake state machine, set during stage 0 (initiator) or beginHandshake (responder multi-message)
pq *pqAttempt // The initiator's post-quantum attempt, pending alongside the classical one
lastPQ bool // Whether the latest attempt was post-quantum
}
func (hh *HandshakeHostInfo) cachePacket(l *slog.Logger, t header.MessageType, st header.MessageSubType, packet []byte, f packetCallback, m *cachedPacketMetrics) {
@@ -135,7 +116,7 @@ func (hh *HandshakeHostInfo) cachePacket(l *slog.Logger, t header.MessageType, s
}
func NewHandshakeManager(l *slog.Logger, mainHostMap *HostMap, lightHouse *LightHouse, outside udp.Conn, config HandshakeConfig) *HandshakeManager {
hm := &HandshakeManager{
return &HandshakeManager{
vpnIps: map[netip.Addr]*HandshakeHostInfo{},
indexes: map[uint32]*HandshakeHostInfo{},
mainHostMap: mainHostMap,
@@ -148,16 +129,7 @@ func NewHandshakeManager(l *slog.Logger, mainHostMap *HostMap, lightHouse *Light
metricInitiated: metrics.GetOrRegisterCounter("handshake_manager.initiated", nil),
metricTimedOut: metrics.GetOrRegisterCounter("handshake_manager.timed_out", nil),
l: l,
pqTable: fragment.NewTable[pqFragKey, pqFragSource](pqTableConfig),
pqAnswered: newPQAnswered(hsTimeout(config.retries, config.tryInterval)),
}
pq := config.pq
hm.pq.Store(&pq)
hm.pqCookies = newPQCookies(time.Now())
// Cookie replies are tiny, but a flood of spoofed fragments must not turn
// into a flood of replies.
hm.pqCookieLimiter, _ = ratelimit.New(256, time.Millisecond)
return hm
}
func (hm *HandshakeManager) Run(ctx context.Context) {
@@ -172,8 +144,6 @@ func (hm *HandshakeManager) Run(ctx context.Context) {
hm.handleOutbound(vpnIP, true)
case now := <-clockSource.C:
hm.NextOutboundHandshakeTimerTick(now)
hm.pqTable.Expire(now)
hm.pqAnswered.expire(now)
}
}
}
@@ -183,7 +153,7 @@ func (hm *HandshakeManager) HandleIncoming(via ViaSender, packet []byte, h *head
// don't yet support) are dropped here rather than silently routed through
// the IX path. Add a case when introducing a new pattern.
switch h.Subtype {
case header.HandshakeIXPSK0, pqSubtype:
case header.HandshakeIXPSK0:
// supported
default:
hm.l.Debug("dropping handshake with unsupported subtype",
@@ -199,11 +169,6 @@ func (hm *HandshakeManager) HandleIncoming(via ViaSender, packet []byte, h *head
}
}
if header.IsFragmented(h.Subtype) {
hm.handleFragment(via, packet, h)
return
}
// First message of a new handshake. The wire format requires RemoteIndex
// to be zero here (the initiator has no responder index to fill in yet),
// and generateIndex never allocates 0, so any non-zero RemoteIndex on a
@@ -215,11 +180,7 @@ func (hm *HandshakeManager) HandleIncoming(via ViaSender, packet []byte, h *head
"from", via, "remoteIndex", h.RemoteIndex)
return
}
if hm.pqConfig().require {
hm.l.Debug("dropping classical handshake, post-quantum handshakes are required", "from", via)
return
}
hm.beginHandshake(via, packet, h, 0)
hm.beginHandshake(via, packet, h)
return
}
@@ -268,9 +229,6 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
"style": header.SubTypeName(header.Handshake, hh.machine.Subtype()),
})
}
if hh.pq != nil {
fields = append(fields, "pqInitiatorIndex", hh.pq.index)
}
hh.hostinfo.logger(hm.l).Info("Handshake timed out", fields...)
hm.metricTimedOut.Inc(1)
hm.DeleteHostInfo(hostinfo)
@@ -280,45 +238,33 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
// Increment the counter to increase our delay, linear backoff
hh.counter++
// Pick this attempt's style and make sure its first message exists. A PQ
// attempt sends msg1 as fragments: now goes out immediately, later after the
// repair delay, and every attempt offers fresh repair symbols.
var now, later [][]byte
var machine *handshake.Machine
hh.lastPQ = hm.usePQ(hh, vpnIp, lighthouseTriggered)
if hh.lastPQ {
if hh.pq == nil && !hm.buildPQAttempt(hh) {
// Check if we have a handshake packet to transmit yet
if !hh.ready {
if !hm.buildStage0Packet(hh) {
hm.OutboundHandshakeTimer.Add(vpnIp, hm.config.tryInterval*time.Duration(hh.counter))
return
}
var err error
now, later, err = hh.pq.sender.Attempt(fragment.DefaultRepair(hh.pq.sender.Layout().K))
if err != nil {
hostinfo.logger(hm.l).Error("Failed to fragment handshake", "error", err)
hm.OutboundHandshakeTimer.Add(vpnIp, hm.config.tryInterval*time.Duration(hh.counter))
return
}
machine = hh.pq.machine
} else {
if !hh.ready {
if !hm.buildStage0Packet(hh) {
hm.OutboundHandshakeTimer.Add(vpnIp, hm.config.tryInterval*time.Duration(hh.counter))
return
}
}
// TODO: this hardcodes "always retransmit stage 0", which is correct for
// IX (the initiator only ever sends one packet, msg1) but wrong the
// moment a 3+ message pattern lands. The retry loop should resend the
// most recent outgoing message, not always stage 0. That implies
// HandshakeHostInfo tracking a single "currentOutbound" packet (bytes +
// header metadata) that gets replaced as the handshake progresses,
// instead of indexing into HandshakePacket.
now = [][]byte{hostinfo.HandshakePacket[handshakePacketStage0]}
machine = hh.machine
}
// TODO: this hardcodes "always retransmit stage 0", which is correct for
// IX (the initiator only ever sends one packet, msg1) but wrong the
// moment a 3+ message pattern lands. The retry loop should resend the
// most recent outgoing message, not always stage 0. That implies
// HandshakeHostInfo tracking a single "currentOutbound" packet (bytes +
// header metadata) that gets replaced as the handshake progresses,
// instead of indexing into HandshakePacket.
stage0 := hostinfo.HandshakePacket[handshakePacketStage0]
hsFields := m{
"stage": uint64(machine.MessageIndex()),
"style": header.SubTypeName(header.Handshake, machine.Subtype()),
"stage": uint64(hh.machine.MessageIndex()),
"style": header.SubTypeName(header.Handshake, hh.machine.Subtype()),
}
// Get a remotes object if we don't already have one.
// This is mainly to protect us as this should never be the case
// NB ^ This comment doesn't jive. It's how the thing gets initialized.
// It's the common path. Should it update every time, in case a future LH query/queries give us more info?
if hostinfo.remotes == nil {
hostinfo.remotes = hm.lightHouse.QueryCache([]netip.Addr{vpnIp})
}
remotes := hostinfo.remotes.CopyAddrs(hm.mainHostMap.GetPreferredRanges())
@@ -346,10 +292,8 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
// Send the handshake to all known ips, stage 2 takes care of assigning the hostinfo.remote based on the first to reply
var sentTo []netip.AddrPort
hostinfo.remotes.ForEach(hm.mainHostMap.GetPreferredRanges(), func(addr netip.AddrPort, _ bool) {
hm.messageMetrics.Tx(header.Handshake, machine.Subtype(), int64(len(now)+len(later)))
err := hm.sendFragmented(now, later, func(p []byte) error {
return hm.outside.WriteTo(p, addr)
})
hm.messageMetrics.Tx(header.Handshake, hh.machine.Subtype(), 1)
err := hm.outside.WriteTo(stage0, addr)
if err != nil {
// These repeat every attempt, so match the success log below and only shout when the remotes changed
level := slog.LevelDebug
@@ -385,7 +329,7 @@ func (hm *HandshakeManager) handleOutbound(vpnIp netip.Addr, lighthouseTriggered
)
}
hm.f.relayManager.StartRelays(hm.f, vpnIp, hh, now, later)
hm.f.relayManager.StartRelays(hm.f, vpnIp, hh, stage0)
// If a lighthouse triggered this attempt then we are still in the timer wheel and do not need to re-add
if !lighthouseTriggered {
@@ -427,9 +371,6 @@ func (hm *HandshakeManager) StartHandshake(vpnAddr netip.Addr, cacheCb func(*Han
hostinfo := &HostInfo{
vpnAddrs: []netip.Addr{vpnAddr},
HandshakePacket: make(map[uint8][]byte, 0),
// Attached here so a pending hostinfo never exists without one: SetRemote dereferences
// it, and create-tunnel -address can reach SetRemote before the handshake loop runs.
remotes: hm.lightHouse.QueryCache([]netip.Addr{vpnAddr}),
relayState: RelayState{
relays: nil,
relayForByAddr: map[netip.Addr]*Relay{},
@@ -595,10 +536,6 @@ func (hm *HandshakeManager) DeleteHostInfo(hostinfo *HostInfo) {
func (hm *HandshakeManager) unlockedDeleteHostInfo(hostinfo *HostInfo) {
for _, addr := range hostinfo.vpnAddrs {
if cur, ok := hm.vpnIps[addr]; ok && cur.hostinfo == hostinfo {
// A PQ attempt holds its own index alongside the hostinfo's.
if idx := cur.pqIndex(); idx != 0 && hm.indexes[idx] == cur {
delete(hm.indexes, idx)
}
delete(hm.vpnIps, addr)
}
}
@@ -709,8 +646,9 @@ func hsTimeout(tries int64, interval time.Duration) time.Duration {
return time.Duration(tries / 2 * ((2 * int64(interval)) + (tries-1)*int64(interval)))
}
// initiatingVersion picks the certificate version to initiate a handshake with.
func (hm *HandshakeManager) initiatingVersion(hh *HandshakeHostInfo, cs *CertState) cert.Version {
// buildStage0Packet creates the initial handshake packet for the initiator.
func (hm *HandshakeManager) buildStage0Packet(hh *HandshakeHostInfo) bool {
cs := hm.f.pki.getCertState()
v := cs.DefaultVersion()
if hh.initiatingVersionOverride != cert.VersionPre1 {
v = hh.initiatingVersionOverride
@@ -722,13 +660,6 @@ func (hm *HandshakeManager) initiatingVersion(hh *HandshakeHostInfo, cs *CertSta
}
}
}
return v
}
// buildStage0Packet creates the initial handshake packet for the initiator.
func (hm *HandshakeManager) buildStage0Packet(hh *HandshakeHostInfo) bool {
cs := hm.f.pki.getCertState()
v := hm.initiatingVersion(hh, cs)
cred := cs.GetCredential(v)
if cred == nil {
@@ -767,10 +698,7 @@ func (hm *HandshakeManager) buildStage0Packet(hh *HandshakeHostInfo) bool {
// beginHandshake handles an incoming handshake packet that doesn't match any
// existing pending handshake. It creates a new responder Machine and processes
// the first message.
//
// received is the number of bytes a fragmented msg1 arrived in, which bounds the
// response sent back before the address is validated; 0 for unfragmented ones.
func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *header.H, received int) {
func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *header.H) {
f := hm.f
cs := f.pki.getCertState()
@@ -784,7 +712,7 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
machine, err := handshake.NewMachine(
v, cs.GetCredential,
hm.certVerifier(), func() (uint32, error) { return generateIndex(f.l) },
false, h.Subtype,
false, header.HandshakeIXPSK0,
)
if err != nil {
f.l.Error("Failed to create handshake machine", "from", via, "error", err)
@@ -874,16 +802,12 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
existing, err := hm.CheckAndComplete(hostinfo, handshakePacketStage0, f)
if err != nil {
hm.handleCheckAndCompleteError(err, existing, hostinfo, via, machine.Subtype(), received)
hm.handleCheckAndCompleteError(err, existing, hostinfo, via)
return
}
hm.sendHandshakeResponse(via, response, hostinfo, false, received)
if header.IsFragmented(machine.Subtype()) {
hm.pqAnswered.add(packet[header.Len:], hostinfo, time.Now())
}
hm.notePQPeer(machine.Subtype(), vpnAddrs)
hostinfo.remotes.RefreshFromHandshake(vpnAddrs)
hm.sendHandshakeResponse(via, response, hostinfo, false)
hostinfo.remotes.RefreshFromHandshake(vpnAddrs, remoteCert.Certificate.Version())
// Don't wait for UpdateWorker
if f.lightHouse.IsAnyLighthouseAddr(vpnAddrs) {
@@ -893,32 +817,23 @@ func (hm *HandshakeManager) beginHandshake(via ViaSender, packet []byte, h *head
// continueHandshake feeds an incoming packet to an existing pending handshake Machine.
func (hm *HandshakeManager) continueHandshake(via ViaSender, hh *HandshakeHostInfo, packet []byte) {
f := hm.f
hh.Lock()
defer hh.Unlock()
if !hm.stillPending(hh, hh.hostinfo.localIndexId) {
// Re-verify hh is still tracked. Between queryIndex returning and us taking
// hh.Lock, handleOutbound may have timed out and deleted it. Once we hold
// hh.Lock no other deleter can race our index: handleOutbound also takes
// hh.Lock first, and handleRecvError targets a main-hostmap entry with a
// different localIndexId.
hm.RLock()
cur, ok := hm.indexes[hh.hostinfo.localIndexId]
hm.RUnlock()
if !ok || cur != hh {
return
}
hm.processResponse(via, hh, hh.machine, packet)
}
// stillPending re-verifies hh is still tracked under index. Between queryIndex
// returning and the caller taking hh.Lock, handleOutbound may have timed out and
// deleted it. Once hh.Lock is held no other deleter can race the index:
// handleOutbound also takes hh.Lock first, and handleRecvError targets a
// main-hostmap entry with a different localIndexId. The caller must hold hh.Lock.
func (hm *HandshakeManager) stillPending(hh *HandshakeHostInfo, index uint32) bool {
hm.RLock()
defer hm.RUnlock()
cur, ok := hm.indexes[index]
return ok && cur == hh
}
// processResponse runs a responder's message through one of the initiator's
// pending attempts and completes the tunnel when the handshake finishes. The
// caller must hold hh.Lock.
func (hm *HandshakeManager) processResponse(via ViaSender, hh *HandshakeHostInfo, machine *handshake.Machine, packet []byte) {
f := hm.f
hostinfo := hh.hostinfo
if !via.IsRelayed {
if !f.lightHouse.GetRemoteAllowList().AllowAll(hostinfo.vpnAddrs, via.UdpAddr.Addr()) {
@@ -928,6 +843,7 @@ func (hm *HandshakeManager) processResponse(via ViaSender, hh *HandshakeHostInfo
}
}
machine := hh.machine
if machine == nil {
f.l.Error("No handshake machine available for continuation",
"vpnAddrs", hostinfo.vpnAddrs, "from", via)
@@ -950,7 +866,7 @@ func (hm *HandshakeManager) processResponse(via ViaSender, hh *HandshakeHostInfo
}
if response != nil {
hm.sendHandshakeResponse(via, response, hostinfo, false, 0)
hm.sendHandshakeResponse(via, response, hostinfo, false)
}
if result == nil {
@@ -1030,14 +946,13 @@ func (hm *HandshakeManager) processResponse(via ViaSender, hh *HandshakeHostInfo
hm.DeleteHostInfo(hostinfo)
hm.StartHandshake(hostinfo.vpnAddrs[0], func(newHH *HandshakeHostInfo) {
newHH.hostinfo.remotes = hostinfo.remotes
newHH.hostinfo.remotes.BlockRemote(via)
newHH.packetStore = hh.packetStore
hh.packetStore = []*cachedPacket{}
hostinfo.vpnAddrs = vpnAddrs
f.sendCloseTunnel(hostinfo)
})
// sendCloseTunnel may take the hostmap lock, so it must not run under the handshake
// manager lock that StartHandshake holds around the callback.
hostinfo.vpnAddrs = vpnAddrs
f.sendCloseTunnel(hostinfo)
return
}
@@ -1063,10 +978,7 @@ func (hm *HandshakeManager) processResponse(via ViaSender, hh *HandshakeHostInfo
hostinfo.vpnAddrs = vpnAddrs
hostinfo.buildNetworks(f.myVpnNetworksTable, remoteCert.Certificate)
// The classical and PQ attempts each hold an index; keep the one that won.
hm.settleIndex(hh, result.LocalIndex)
hm.Complete(hostinfo, f)
hm.notePQPeer(machine.Subtype(), vpnAddrs)
if len(hh.packetStore) > 0 {
if f.l.Enabled(context.Background(), slog.LevelDebug) {
@@ -1083,7 +995,7 @@ func (hm *HandshakeManager) processResponse(via ViaSender, hh *HandshakeHostInfo
f.cachedPacketMetrics.sent.Inc(int64(len(hh.packetStore)))
}
hostinfo.remotes.RefreshFromHandshake(vpnAddrs)
hostinfo.remotes.RefreshFromHandshake(vpnAddrs, remoteCert.Certificate.Version())
f.metricHandshakes.Update(duration)
// Don't wait for UpdateWorker
@@ -1145,23 +1057,20 @@ func (hm *HandshakeManager) validatePeerCert(via ViaSender, remoteCert *cert.Cac
// cached is true when msg is a stored response being retransmitted because
// the peer's stage-1 retransmit landed (the ErrAlreadySeen path); false on a
// fresh response.
//
// received is the number of bytes a fragmented msg1 arrived in; it bounds what a
// fragmented response may send back. It is ignored for unfragmented subtypes.
func (hm *HandshakeManager) sendHandshakeResponse(via ViaSender, msg []byte, hostinfo *HostInfo, cached bool, received int) {
func (hm *HandshakeManager) sendHandshakeResponse(via ViaSender, msg []byte, hostinfo *HostInfo, cached bool) {
if msg == nil {
return
}
f := hm.f
subtype := header.MessageSubType(msg[1])
f.messageMetrics.Tx(header.Handshake, header.MessageSubType(msg[1]), 1)
// Common log fields. peerCert may be nil during intermediate
// multi-message flows (handshake hasn't completed yet); skip the cert
// block if so.
logFields := []any{
"vpnAddrs", hostinfo.vpnAddrs,
"handshake", m{"stage": uint64(2), "style": header.SubTypeName(header.Handshake, subtype)},
"handshake", m{"stage": uint64(2), "style": header.SubTypeName(header.Handshake, header.HandshakeIXPSK0)},
"cached", cached,
"initiatorIndex", hostinfo.remoteIndexId,
"responderIndex", hostinfo.localIndexId,
@@ -1175,17 +1084,6 @@ func (hm *HandshakeManager) sendHandshakeResponse(via ViaSender, msg []byte, hos
)
}
if header.IsFragmented(subtype) {
fields := append(logFields, "from", via)
if err := hm.sendFragmentedResponse(via, msg, hostinfo, cached, received); err != nil {
f.l.Debug("Did not send handshake message", append(fields, "error", err)...)
} else {
f.l.Info("Handshake message sent", fields...)
}
return
}
f.messageMetrics.Tx(header.Handshake, subtype, 1)
if !via.IsRelayed {
fields := append(logFields, "from", via)
err := f.outside.WriteTo(msg, via.UdpAddr)
@@ -1212,10 +1110,10 @@ func (hm *HandshakeManager) sendHandshakeResponse(via ViaSender, msg []byte, hos
// This only fires from the responder-side beginHandshake path, after the
// peer cert has been validated and ConnectionState populated, so peerCert
// is always non-nil for the cases that log it.
func (hm *HandshakeManager) handleCheckAndCompleteError(err error, existing, hostinfo *HostInfo, via ViaSender, subtype header.MessageSubType, received int) {
func (hm *HandshakeManager) handleCheckAndCompleteError(err error, existing, hostinfo *HostInfo, via ViaSender) {
f := hm.f
peerCert := hostinfo.ConnectionState.peerCert
hsFields := m{"stage": uint64(1), "style": header.SubTypeName(header.Handshake, subtype)}
hsFields := m{"stage": uint64(1), "style": header.SubTypeName(header.Handshake, header.HandshakeIXPSK0)}
switch err {
case ErrAlreadySeen:
@@ -1226,7 +1124,7 @@ func (hm *HandshakeManager) handleCheckAndCompleteError(err error, existing, hos
// ephemeral keys; a freshly-built one would have different keys and break
// the tunnel even though both sides "completed" the handshake.
if msg := existing.HandshakePacket[handshakePacketStage2]; msg != nil {
hm.sendHandshakeResponse(via, msg, existing, true, received)
hm.sendHandshakeResponse(via, msg, existing, true)
}
case ErrExistingHostInfo:
-699
View File
@@ -1,699 +0,0 @@
package nebula
import (
"context"
"crypto/hmac"
"crypto/rand"
"crypto/sha256"
"encoding/binary"
"errors"
"log/slog"
"net/netip"
"sync"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/handshake"
"github.com/slackhq/nebula/handshake/fragment"
"github.com/slackhq/nebula/header"
)
const (
// pqSubtype is the post-quantum handshake this node speaks.
pqSubtype = header.HandshakeIXPSK2MLKEM1024
// DefaultPQRepairDelay is how long repair fragments trail the data fragments.
DefaultPQRepairDelay = 10 * time.Millisecond
// pqAmplification bounds what a responder sends in reply to an unvalidated
// address, as a multiple of what it received from it.
pqAmplification = 3
)
// pqTableConfig bounds the responder's reassembly of msg1 fragments from senders
// that have not authenticated yet.
var pqTableConfig = fragment.TableConfig{
MaxBytes: 4 << 20,
MaxEntries: 1024,
MaxPerSource: 4,
MaxMsgLen: 16384,
TTL: 3 * time.Second,
}
type pqHandshakeConfig struct {
initiate bool // start PQ handshakes
accept bool // answer PQ handshakes
require bool // only PQ handshakes, in either direction
sticky bool // never fall back to classical for a peer that completed PQ
repairDelay time.Duration // delay between data and repair fragments
}
func pqHandshakeConfigFrom(c *config.C) pqHandshakeConfig {
cfg := pqHandshakeConfig{
initiate: c.GetBool("handshakes.pq.initiate", false),
accept: c.GetBool("handshakes.pq.accept", false),
require: c.GetBool("handshakes.pq.require", false),
sticky: c.GetBool("handshakes.pq.sticky", false),
repairDelay: c.GetDuration("handshakes.pq.repair_delay", DefaultPQRepairDelay),
}
if cfg.require {
cfg.initiate = true
cfg.accept = true
}
return cfg
}
// pqConfig returns the post-quantum settings in effect.
func (hm *HandshakeManager) pqConfig() *pqHandshakeConfig {
return hm.pq.Load()
}
// reloadPQ applies changed post-quantum settings. Handshakes already in flight
// keep going; the new settings apply to the next attempt and packet.
func (hm *HandshakeManager) reloadPQ(c *config.C) {
if !c.HasChanged("handshakes.pq") {
return
}
old := hm.pqConfig()
cfg := pqHandshakeConfigFrom(c)
hm.pq.Store(&cfg)
hm.l.Info("Post-quantum handshake settings changed",
"initiate", cfg.initiate, "oldInitiate", old.initiate,
"accept", cfg.accept, "oldAccept", old.accept,
"require", cfg.require, "oldRequire", old.require,
"sticky", cfg.sticky, "oldSticky", old.sticky,
"repairDelay", cfg.repairDelay, "oldRepairDelay", old.repairDelay,
)
}
// pqAttempt is the initiator's post-quantum attempt for a pending handshake. It
// lives alongside the classical attempt in HandshakeHostInfo and has its own
// local index, so whichever answer arrives first wins.
type pqAttempt struct {
machine *handshake.Machine
index uint32
sender *fragment.Sender
msg2 *fragment.Assembly
cookieResendAt time.Time // last resend prompted by a cookie reply
heard bool // a msg2 fragment or cookie reply arrived, so the peer speaks PQ
}
// pqFragKey identifies one msg1 being reassembled.
type pqFragKey struct {
addr netip.AddrPort
relay uint32
fragID uint32
}
// pqFragSource groups msg1 reassemblies for the per-source quota: an IPv4 /32,
// an IPv6 /64, or one relay channel.
type pqFragSource struct {
prefix netip.Prefix
relay uint32
}
func pqFragKeyFor(via ViaSender, fragID uint32) (pqFragKey, pqFragSource) {
if via.IsRelayed {
return pqFragKey{relay: via.relay.LocalIndex, fragID: fragID},
pqFragSource{relay: via.relay.LocalIndex}
}
addr := via.UdpAddr.Addr().Unmap()
bits := 64
if addr.Is4() {
bits = 32
}
prefix, _ := addr.Prefix(bits)
return pqFragKey{addr: via.UdpAddr, fragID: fragID}, pqFragSource{prefix: prefix}
}
// reassembled rebuilds the packet the handshake Machine expects from the header
// of a fragment and the decoded message.
func reassembled(h *header.H, msg []byte) []byte {
p := make([]byte, header.Len, header.Len+len(msg))
header.Encode(p, header.Version, header.Handshake, h.Subtype, h.RemoteIndex, h.MessageCounter)
return append(p, msg...)
}
// usePQ decides whether this outbound attempt is post-quantum. A peer that does
// not speak PQ silently drops it, so timer retries alternate between PQ and
// classical, starting with PQ. An attempt triggered early by a lighthouse reply
// is sending to addresses nothing has been sent to yet, and a new path gets the
// PQ attempt first. With require, or for a peer that completed a PQ handshake
// when sticky is on, every attempt is PQ. Once anything has come back for the PQ
// attempt the peer evidently speaks PQ, and a classical attempt would only race
// it, so classical stops getting turns.
func (hm *HandshakeManager) usePQ(hh *HandshakeHostInfo, vpnIp netip.Addr, triggered bool) bool {
cfg := hm.pqConfig()
switch {
case !cfg.initiate:
return false
case cfg.require:
return true
case hh.pq != nil && hh.pq.heard:
return true
case cfg.sticky:
if _, ok := hm.pqPeers.Load(vpnIp); ok {
return true
}
}
return hh.counter <= 1 || triggered || !hh.lastPQ
}
// relayEstablished pushes the pending handshake with target through a relay that
// just came up, rather than leaving it for the next retry. Nothing has gone over
// the relay yet, so like any new path it gets the PQ attempt first, which then
// counts as the PQ turn: the next retry is classical, as if it had been sent on
// schedule. Without this, relays coming up while the first attempt is in flight
// handed every relayed handshake to the classical retry.
func (hm *HandshakeManager) relayEstablished(target netip.Addr, relayHI *HostInfo, relay *Relay) {
hh := hm.queryVpnIp(target)
if hh == nil {
return
}
hh.Lock()
defer hh.Unlock()
if hm.queryVpnIp(target) != hh {
return
}
var now, later [][]byte
switch {
case hh.pq != nil:
var err error
now, later, err = hh.pq.sender.Attempt(fragment.DefaultRepair(hh.pq.sender.Layout().K))
if err != nil {
return
}
hh.lastPQ = true
case hh.ready:
now = [][]byte{hh.hostinfo.HandshakePacket[handshakePacketStage0]}
default:
return
}
f := hm.f
hh.hostinfo.logger(hm.l).Info("Send handshake via new relay", "relay", relayHI.vpnAddrs[0])
_ = hm.sendFragmented(now, later, func(p []byte) error {
f.SendVia(relayHI, relay, p, make([]byte, 12), make([]byte, mtu), false, 0)
return nil
})
}
// buildPQAttempt creates the initiator's PQ Machine and msg1 sender.
func (hm *HandshakeManager) buildPQAttempt(hh *HandshakeHostInfo) bool {
cs := hm.f.pki.getCertState()
v := hm.initiatingVersion(hh, cs)
if cs.GetCredential(v) == nil {
hm.f.l.Error("Unable to handshake with host because no certificate is available",
"vpnAddrs", hh.hostinfo.vpnAddrs, "certVersion", v)
return false
}
pq := &pqAttempt{}
machine, err := handshake.NewMachine(
v, cs.GetCredential,
hm.certVerifier(), func() (uint32, error) { return hm.allocatePQIndex(hh, pq) },
true, pqSubtype,
)
if err != nil {
hm.f.l.Error("Failed to create handshake machine",
"vpnAddrs", hh.hostinfo.vpnAddrs, "error", err)
return false
}
msg, err := machine.Initiate(nil)
if err != nil {
hm.f.l.Error("Failed to initiate handshake",
"vpnAddrs", hh.hostinfo.vpnAddrs, "error", err)
hm.releasePQIndex(hh, pq)
return false
}
sender, err := fragment.NewSender(msg[header.Len:], pqSubtype, 0, 1, pq.index)
if err != nil {
hm.f.l.Error("Failed to fragment handshake",
"vpnAddrs", hh.hostinfo.vpnAddrs, "error", err)
hm.releasePQIndex(hh, pq)
return false
}
pq.machine = machine
pq.sender = sender
hh.pq = pq
return true
}
// allocatePQIndex registers a local index for the PQ attempt. Unlike
// allocateIndex it leaves hostinfo.localIndexId alone; that is settled when one
// of the attempts completes.
func (hm *HandshakeManager) allocatePQIndex(hh *HandshakeHostInfo, pq *pqAttempt) (uint32, error) {
hm.mainHostMap.RLock()
defer hm.mainHostMap.RUnlock()
hm.Lock()
defer hm.Unlock()
for range 32 {
index, err := generateIndex(hm.l)
if err != nil {
return 0, err
}
_, inPending := hm.indexes[index]
_, inMain := hm.mainHostMap.Indexes[index]
if !inMain && !inPending {
pq.index = index
hm.indexes[index] = hh
return index, nil
}
}
return 0, errors.New("failed to generate unique localIndexId")
}
func (hm *HandshakeManager) releasePQIndex(hh *HandshakeHostInfo, pq *pqAttempt) {
hm.Lock()
defer hm.Unlock()
if pq.index != 0 && hm.indexes[pq.index] == hh {
delete(hm.indexes, pq.index)
}
}
// settleIndex makes winner the hostinfo's local index and releases the index of
// the attempt that lost.
func (hm *HandshakeManager) settleIndex(hh *HandshakeHostInfo, winner uint32) {
hm.Lock()
defer hm.Unlock()
for _, idx := range []uint32{hh.hostinfo.localIndexId, hh.pqIndex()} {
if idx != 0 && idx != winner && hm.indexes[idx] == hh {
delete(hm.indexes, idx)
}
}
hh.hostinfo.localIndexId = winner
}
func (hh *HandshakeHostInfo) pqIndex() uint32 {
if hh.pq == nil {
return 0
}
return hh.pq.index
}
// handleFragment routes one fragment of a PQ handshake message.
func (hm *HandshakeManager) handleFragment(via ViaSender, packet []byte, h *header.H) {
if h.MessageCounter == pqCookieCounter {
hm.handleCookie(via, packet, h)
return
}
fh, sym, err := fragment.Parse(packet[header.Len:])
if err != nil {
hm.l.Debug("dropping malformed handshake fragment", "from", via, "error", err)
return
}
switch h.MessageCounter {
case 1:
if !hm.pqConfig().accept {
hm.l.Debug("dropping post-quantum handshake, not accepting them", "from", via)
return
}
if h.RemoteIndex != 0 {
hm.l.Debug("dropping stage-1 handshake with non-zero RemoteIndex",
"from", via, "remoteIndex", h.RemoteIndex)
return
}
now := time.Now()
key, src := pqFragKeyFor(via, fh.FragID)
if hm.pqTable.Load() >= pqCookieLoad && !(fh.HasCookie() && hm.pqCookies.valid(key, fh.Cookie, now)) {
hm.sendCookie(via, fh, key, len(packet), now)
return
}
res, err := hm.pqTable.Add(key, src, now, fh, sym)
if err != nil {
hm.l.Debug("dropping handshake fragment", "from", via, "error", err)
return
}
if res == nil {
return
}
if existing := hm.pqAnswered.lookup(res.Msg, time.Now()); existing != nil {
// Already answered: resend, rate limited, with fresh symbols.
if msg := existing.HandshakePacket[handshakePacketStage2]; msg != nil {
hm.sendHandshakeResponse(via, msg, existing, true, res.Received)
}
return
}
hm.beginHandshake(via, reassembled(h, res.Msg), h, res.Received)
case 2:
if hh := hm.queryIndex(h.RemoteIndex); hh != nil {
hm.continuePQ(via, hh, h, fh, sym)
}
}
}
// continuePQ collects msg2 fragments for the initiator's PQ attempt.
func (hm *HandshakeManager) continuePQ(via ViaSender, hh *HandshakeHostInfo, h *header.H, fh fragment.Header, sym []byte) {
hh.Lock()
defer hh.Unlock()
pq := hh.pq
if pq == nil || pq.index != h.RemoteIndex || !hm.stillPending(hh, pq.index) {
return
}
pq.heard = true
if pq.msg2 == nil {
a, err := fragment.NewAssembly(fh)
if err != nil {
return
}
pq.msg2 = a
}
ready, err := pq.msg2.Add(fh, sym)
if errors.Is(err, fragment.ErrMismatch) {
// The responder's FragID and MsgLen never change, so this is a stray or a
// forgery. Start over from this fragment rather than wedge.
pq.msg2 = nil
return
}
if err != nil || !ready {
return
}
msg, err := pq.msg2.Decode()
pq.msg2 = nil
if err != nil {
hm.l.Debug("failed to decode handshake fragments", "from", via, "error", err)
return
}
hm.processResponse(via, hh, pq.machine, reassembled(h, msg))
}
// sendFragmented sends packets to a peer: now immediately and later after the
// repair delay. It returns the first error from the immediate sends; errors from
// delayed sends are only logged.
func (hm *HandshakeManager) sendFragmented(now, later [][]byte, send func([]byte) error) error {
var first error
for _, p := range now {
if err := send(p); err != nil && first == nil {
first = err
}
}
sendLater := func() {
for _, p := range later {
if err := send(p); err != nil {
hm.l.Debug("Failed to send handshake repair fragment", "error", err)
}
}
}
delay := hm.pqConfig().repairDelay
switch {
case len(later) == 0:
case delay <= 0:
sendLater()
default:
time.AfterFunc(delay, sendLater)
}
return first
}
// sendFragmentedResponse sends the responder's msg2 as fragments. A resend of a
// cached response (cached) offers fresh symbols and is rate limited to one per
// retry interval. received is how many bytes the triggering msg1 took, which
// bounds what may be sent back before the address is validated.
func (hm *HandshakeManager) sendFragmentedResponse(via ViaSender, msg []byte, hostinfo *HostInfo, cached bool, received int) error {
now := time.Now()
if cached {
last := hostinfo.handshakeResendAt.Load()
if now.UnixNano()-last < int64(hm.config.tryInterval) ||
!hostinfo.handshakeResendAt.CompareAndSwap(last, now.UnixNano()) {
return errors.New("handshake response resend rate limited")
}
} else {
hostinfo.handshakeResendAt.Store(now.UnixNano())
}
var h header.H
if err := h.Parse(msg); err != nil {
return err
}
s, err := fragment.NewSender(msg[header.Len:], h.Subtype, h.RemoteIndex, h.MessageCounter, hostinfo.localIndexId)
if err != nil {
return err
}
l := s.Layout()
repair := fragment.DefaultRepair(l.K)
if received > 0 {
pktLen := header.Len + fragment.HeaderLen + l.SymbolSize
budget := pqAmplification * received / pktLen
if budget < l.K {
return errors.New("handshake response exceeds the amplification budget")
}
repair = min(repair, budget-l.K)
}
n := uint32(l.K + repair)
s.SetNext(int(hostinfo.handshakeNextSymbol.Add(n) - n))
pkts, later, err := s.Attempt(repair)
if err != nil {
return err
}
if via.IsRelayed {
if via.relay == nil {
return errors.New("both addr and via.relay are nil")
}
// We received a valid handshake on this relay, so make sure the relay
// state reflects that, in case it had been marked Disestablished.
hostinfo.relayState.InsertRelayTo(via.relayHI.vpnAddrs[0])
via.relayHI.relayState.UpdateRelayForByIdxState(via.relay.LocalIndex, Established)
}
hm.f.messageMetrics.Tx(header.Handshake, h.Subtype, int64(len(pkts)+len(later)))
return hm.sendFragmented(pkts, later, hm.sendTo(via))
}
// notePQPeer remembers peers that completed a PQ handshake, for sticky mode.
func (hm *HandshakeManager) notePQPeer(subtype header.MessageSubType, vpnAddrs []netip.Addr) {
if !hm.pqConfig().sticky || subtype != pqSubtype {
return
}
for _, a := range vpnAddrs {
hm.pqPeers.Store(a, struct{}{})
}
if hm.l.Enabled(context.Background(), slog.LevelDebug) {
hm.l.Debug("Remembering post-quantum peer", "vpnAddrs", vpnAddrs)
}
}
// pqAnswered remembers the msg1s a responder recently answered, keyed by hash,
// so a repeat is recognised before any noise, KEM or certificate work. Repeats
// are routine: the trailing repair fragments of an attempt rebuild a msg1 that
// its data fragments already completed, and a retrying initiator offers fresh
// symbols for the same msg1. Entries only exist for completed handshakes, which
// need a valid certificate, and live as long as an initiator keeps retrying.
type pqAnswered struct {
sync.Mutex
ttl time.Duration
entries map[[sha256.Size]byte]pqAnsweredEntry
}
type pqAnsweredEntry struct {
hostinfo *HostInfo
at time.Time
}
const pqAnsweredMax = 4096
func newPQAnswered(ttl time.Duration) *pqAnswered {
return &pqAnswered{ttl: ttl, entries: map[[sha256.Size]byte]pqAnsweredEntry{}}
}
func (a *pqAnswered) add(msg1 []byte, hostinfo *HostInfo, now time.Time) {
a.Lock()
defer a.Unlock()
if len(a.entries) >= pqAnsweredMax {
for k := range a.entries {
delete(a.entries, k)
break
}
}
a.entries[sha256.Sum256(msg1)] = pqAnsweredEntry{hostinfo: hostinfo, at: now}
}
func (a *pqAnswered) lookup(msg1 []byte, now time.Time) *HostInfo {
a.Lock()
defer a.Unlock()
e, ok := a.entries[sha256.Sum256(msg1)]
if !ok || now.Sub(e.at) > a.ttl {
return nil
}
return e.hostinfo
}
func (a *pqAnswered) expire(now time.Time) {
a.Lock()
defer a.Unlock()
for k, e := range a.entries {
if now.Sub(e.at) > a.ttl {
delete(a.entries, k)
}
}
}
// A responder whose msg1 table is under load stops keeping state for senders
// that have not shown they receive at their address. It answers a fragment
// without a valid cookie with a cookie reply and keeps nothing; the initiator
// resends with the cookie attached. This is WireGuard's under-load cookie, QUIC's
// Retry and SCTP's state cookie: one extra round trip, only while under attack.
const (
// pqCookieLoad is the table load above which msg1 fragments need a cookie.
pqCookieLoad = 0.5
// pqCookieCounter marks a cookie reply in the header's message counter.
// Handshake messages count from 1.
pqCookieCounter = 0
// pqCookieRotate is how often the cookie secret changes. A cookie stays
// valid for between one and two rotations.
pqCookieRotate = 2 * time.Minute
pqCookieReplyLen = header.Len + fragment.CookieLen
)
// pqCookies mints and checks cookies: an HMAC over the sender's address (or relay
// channel) and FragID under a rotating secret, with the previous secret still
// accepted.
type pqCookies struct {
sync.Mutex
secrets [2][32]byte
rotatedAt time.Time
}
func newPQCookies(now time.Time) *pqCookies {
c := &pqCookies{rotatedAt: now}
_, _ = rand.Read(c.secrets[0][:])
_, _ = rand.Read(c.secrets[1][:])
return c
}
func (c *pqCookies) rotate(now time.Time) {
if now.Sub(c.rotatedAt) < pqCookieRotate {
return
}
c.secrets[1] = c.secrets[0]
_, _ = rand.Read(c.secrets[0][:])
c.rotatedAt = now
}
func cookieMAC(secret []byte, key pqFragKey) fragment.Cookie {
mac := hmac.New(sha256.New, secret)
addr := key.addr.Addr().As16()
mac.Write(addr[:])
var b [10]byte
binary.BigEndian.PutUint16(b[0:], key.addr.Port())
binary.BigEndian.PutUint32(b[2:], key.relay)
binary.BigEndian.PutUint32(b[6:], key.fragID)
mac.Write(b[:])
var c fragment.Cookie
copy(c[:], mac.Sum(nil))
return c
}
func (c *pqCookies) make(key pqFragKey, now time.Time) fragment.Cookie {
c.Lock()
defer c.Unlock()
c.rotate(now)
return cookieMAC(c.secrets[0][:], key)
}
func (c *pqCookies) valid(key pqFragKey, cookie fragment.Cookie, now time.Time) bool {
c.Lock()
defer c.Unlock()
c.rotate(now)
for _, s := range c.secrets {
want := cookieMAC(s[:], key)
if hmac.Equal(want[:], cookie[:]) {
return true
}
}
return false
}
// sendCookie answers a msg1 fragment that arrived without a valid cookie while
// under load. The reply is never larger than what triggered it, and replies are
// rate limited as a whole.
func (hm *HandshakeManager) sendCookie(via ViaSender, fh fragment.Header, key pqFragKey, received int, now time.Time) {
if received < pqCookieReplyLen {
return
}
if _, ok := hm.pqCookieLimiter.Allow(); !ok {
return
}
c := hm.pqCookies.make(key, now)
reply := make([]byte, header.Len, pqCookieReplyLen)
header.Encode(reply, header.Version, header.Handshake, pqSubtype, fh.FragID, pqCookieCounter)
reply = append(reply, c[:]...)
hm.f.messageMetrics.Tx(header.Handshake, pqSubtype, 1)
if err := hm.sendTo(via)(reply); err != nil {
hm.l.Debug("Failed to send handshake cookie", "to", via, "error", err)
}
}
// handleCookie attaches a responder's cookie to the PQ attempt it names and
// resends right away, at most once per retry interval, rather than waiting for
// the next attempt.
func (hm *HandshakeManager) handleCookie(via ViaSender, packet []byte, h *header.H) {
if len(packet) != pqCookieReplyLen {
return
}
hh := hm.queryIndex(h.RemoteIndex)
if hh == nil {
return
}
hh.Lock()
defer hh.Unlock()
pq := hh.pq
if pq == nil || pq.index != h.RemoteIndex || !hm.stillPending(hh, pq.index) {
return
}
var c fragment.Cookie
copy(c[:], packet[header.Len:])
pq.sender.SetCookie(c)
pq.heard = true
now := time.Now()
if now.Sub(pq.cookieResendAt) < hm.config.tryInterval {
return
}
pq.cookieResendAt = now
pkts, later, err := pq.sender.Attempt(fragment.DefaultRepair(pq.sender.Layout().K))
if err != nil {
return
}
hm.messageMetrics.Tx(header.Handshake, pqSubtype, int64(len(pkts)+len(later)))
if err := hm.sendFragmented(pkts, later, hm.sendTo(via)); err != nil {
hm.l.Debug("Failed to resend handshake with cookie", "to", via, "error", err)
}
}
// sendTo returns a function sending a packet back the way via came: directly, or
// through the relay it arrived on.
func (hm *HandshakeManager) sendTo(via ViaSender) func([]byte) error {
f := hm.f
if !via.IsRelayed {
return func(p []byte) error {
return f.outside.WriteTo(p, via.UdpAddr)
}
}
return func(p []byte) error {
if via.relay == nil {
return errors.New("both addr and via.relay are nil")
}
f.SendVia(via.relayHI, via.relay, p, make([]byte, 12), make([]byte, mtu), false, 0)
return nil
}
}
-398
View File
@@ -1,398 +0,0 @@
package nebula
import (
"math/rand/v2"
"net/netip"
"sync"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/handshake/fragment"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// recordingConn captures what is written to it.
type recordingConn struct {
udp.NoopConn
sync.Mutex
sent [][]byte
}
func (c *recordingConn) WriteTo(b []byte, _ netip.AddrPort) error {
c.Lock()
defer c.Unlock()
c.sent = append(c.sent, append([]byte(nil), b...))
return nil
}
func (c *recordingConn) take() [][]byte {
c.Lock()
defer c.Unlock()
s := c.sent
c.sent = nil
return s
}
func newPQTestManager(t *testing.T) (*HandshakeManager, *recordingConn) {
l := test.NewLogger()
conn := &recordingConn{}
cfg := defaultHandshakeConfig
cfg.pq = pqHandshakeConfig{initiate: true, accept: true}
hm := NewHandshakeManager(l, newHostMap(l), newTestLighthouse(), conn, cfg)
hm.f = &Interface{
handshakeManager: hm,
outside: conn,
messageMetrics: newMessageMetricsOnlyRecvError(),
l: l,
}
return hm, conn
}
// fragmentIndexes checks every packet is a fragment of the expected response and
// returns their symbol indexes.
func fragmentIndexes(t *testing.T, pkts [][]byte, fragID, remoteIndex uint32) []int {
t.Helper()
var out []int
for _, p := range pkts {
require.LessOrEqual(t, len(p), fragment.MaxPacketLen)
var h header.H
require.NoError(t, h.Parse(p))
assert.Equal(t, pqSubtype, h.Subtype)
assert.Equal(t, remoteIndex, h.RemoteIndex)
assert.Equal(t, uint64(2), h.MessageCounter)
fh, _, err := fragment.Parse(p[header.Len:])
require.NoError(t, err)
assert.Equal(t, fragID, fh.FragID)
out = append(out, int(fh.Index))
}
return out
}
func TestSendFragmentedResponse(t *testing.T) {
hm, conn := newPQTestManager(t)
via := ViaSender{UdpAddr: netip.MustParseAddrPort("192.0.2.1:4242")}
hostinfo := &HostInfo{localIndexId: 77}
// A 2000 byte message is 2 symbols of 1000, so packets of 1024 bytes.
msg := make([]byte, header.Len, header.Len+2000)
header.Encode(msg, header.Version, header.Handshake, pqSubtype, 5, 2)
r := rand.New(rand.NewPCG(1, 1))
for range 2000 {
msg = append(msg, byte(r.Uint32()))
}
const pktLen = header.Len + fragment.HeaderLen + 1000
t.Run("first send", func(t *testing.T) {
require.NoError(t, hm.sendFragmentedResponse(via, msg, hostinfo, false, 0))
assert.Equal(t, []int{0, 1, 2, 3}, fragmentIndexes(t, conn.take(), 77, 5))
})
t.Run("resends are rate limited", func(t *testing.T) {
require.Error(t, hm.sendFragmentedResponse(via, msg, hostinfo, true, 10*pktLen))
assert.Empty(t, conn.take())
})
allowResend := func() {
hostinfo.handshakeResendAt.Store(time.Now().Add(-time.Second).UnixNano())
}
t.Run("resend offers fresh symbols", func(t *testing.T) {
allowResend()
require.NoError(t, hm.sendFragmentedResponse(via, msg, hostinfo, true, 10*pktLen))
assert.Equal(t, []int{4, 5, 6, 7}, fragmentIndexes(t, conn.take(), 77, 5))
})
t.Run("amplification budget trims repair", func(t *testing.T) {
allowResend()
// 3x of a little over one packet allows 3 packets: both data and 1 repair.
require.NoError(t, hm.sendFragmentedResponse(via, msg, hostinfo, true, pktLen+10))
assert.Equal(t, []int{8, 9, 10}, fragmentIndexes(t, conn.take(), 77, 5))
})
t.Run("amplification budget below the data refuses", func(t *testing.T) {
allowResend()
require.Error(t, hm.sendFragmentedResponse(via, msg, hostinfo, true, pktLen/2))
assert.Empty(t, conn.take())
})
}
func TestPQAnswered(t *testing.T) {
now := time.Unix(1000, 0)
a := newPQAnswered(5 * time.Second)
hi := &HostInfo{}
assert.Nil(t, a.lookup([]byte("msg1"), now))
a.add([]byte("msg1"), hi, now)
assert.Same(t, hi, a.lookup([]byte("msg1"), now.Add(time.Second)))
assert.Nil(t, a.lookup([]byte("other"), now))
assert.Nil(t, a.lookup([]byte("msg1"), now.Add(6*time.Second)), "stale entries are not answered")
a.expire(now.Add(6 * time.Second))
assert.Empty(t, a.entries)
for i := range pqAnsweredMax + 10 {
a.add([]byte{byte(i), byte(i >> 8)}, hi, now)
}
assert.Len(t, a.entries, pqAnsweredMax)
}
func TestPQFragKeyFor(t *testing.T) {
k1, s1 := pqFragKeyFor(ViaSender{UdpAddr: netip.MustParseAddrPort("192.0.2.1:4242")}, 9)
k2, s2 := pqFragKeyFor(ViaSender{UdpAddr: netip.MustParseAddrPort("192.0.2.1:4243")}, 9)
assert.NotEqual(t, k1, k2, "different ports are different messages")
assert.Equal(t, s1, s2, "but share an IPv4 /32 quota")
_, s3 := pqFragKeyFor(ViaSender{UdpAddr: netip.MustParseAddrPort("[2001:db8::1]:1")}, 9)
_, s4 := pqFragKeyFor(ViaSender{UdpAddr: netip.MustParseAddrPort("[2001:db8::ffff]:1")}, 9)
assert.Equal(t, s3, s4, "an IPv6 /64 shares a quota")
_, s5 := pqFragKeyFor(ViaSender{UdpAddr: netip.MustParseAddrPort("[::ffff:192.0.2.1]:1")}, 9)
assert.Equal(t, s1, s5, "v4-mapped addresses count as IPv4")
}
func TestPQCookies(t *testing.T) {
now := time.Unix(1000, 0)
c := newPQCookies(now)
key := pqFragKey{addr: netip.MustParseAddrPort("192.0.2.1:4242"), fragID: 9}
cookie := c.make(key, now)
assert.True(t, c.valid(key, cookie, now))
for _, other := range []pqFragKey{
{addr: netip.MustParseAddrPort("192.0.2.2:4242"), fragID: 9},
{addr: netip.MustParseAddrPort("192.0.2.1:4243"), fragID: 9},
{addr: netip.MustParseAddrPort("192.0.2.1:4242"), fragID: 10},
{addr: netip.MustParseAddrPort("192.0.2.1:4242"), relay: 1, fragID: 9},
} {
assert.False(t, c.valid(other, cookie, now), "%+v", other)
}
now = now.Add(pqCookieRotate)
assert.True(t, c.valid(key, cookie, now), "the previous secret is still accepted")
assert.NotEqual(t, cookie, c.make(key, now), "new cookies come from the new secret")
now = now.Add(pqCookieRotate)
assert.False(t, c.valid(key, cookie, now), "two rotations retire a cookie")
}
// pqMsg1Fragment builds the first fragment of a msg1, optionally with a cookie.
func pqMsg1Fragment(t *testing.T, fragID uint32, cookie *fragment.Cookie) ([]byte, *header.H) {
t.Helper()
s, err := fragment.NewSender(make([]byte, 2000), pqSubtype, 0, 1, fragID)
require.NoError(t, err)
if cookie != nil {
s.SetCookie(*cookie)
}
pkts, _, err := s.Attempt(0)
require.NoError(t, err)
var h header.H
require.NoError(t, h.Parse(pkts[0]))
return pkts[0], &h
}
// fillPQTable puts the responder's msg1 table at the cookie threshold.
func fillPQTable(t *testing.T, hm *HandshakeManager) {
t.Helper()
now := time.Now()
for i := 0; hm.pqTable.Load() < pqCookieLoad; i++ {
p, _ := pqMsg1Fragment(t, uint32(i+1), nil)
fh, sym, err := fragment.Parse(p[header.Len:])
require.NoError(t, err)
addr := netip.AddrPortFrom(netip.AddrFrom4([4]byte{198, 18, byte(i >> 8), byte(i)}), 4242)
key, src := pqFragKeyFor(ViaSender{UdpAddr: addr}, fh.FragID)
_, err = hm.pqTable.Add(key, src, now, fh, sym)
require.NoError(t, err)
}
}
func TestPQCookieUnderLoad(t *testing.T) {
hm, conn := newPQTestManager(t)
via := ViaSender{UdpAddr: netip.MustParseAddrPort("192.0.2.1:4242")}
t.Run("no cookie needed below the threshold", func(t *testing.T) {
p, h := pqMsg1Fragment(t, 5000, nil)
before := hm.pqTable.Len()
hm.handleFragment(via, p, h)
assert.Empty(t, conn.take())
assert.Equal(t, before+1, hm.pqTable.Len())
})
fillPQTable(t, hm)
var cookie fragment.Cookie
t.Run("a fragment without a cookie gets one and leaves no state", func(t *testing.T) {
p, h := pqMsg1Fragment(t, 7, nil)
before := hm.pqTable.Len()
hm.handleFragment(via, p, h)
assert.Equal(t, before, hm.pqTable.Len())
sent := conn.take()
require.Len(t, sent, 1)
reply := sent[0]
require.Len(t, reply, pqCookieReplyLen)
var rh header.H
require.NoError(t, rh.Parse(reply))
assert.Equal(t, header.Handshake, rh.Type)
assert.Equal(t, pqSubtype, rh.Subtype)
assert.Equal(t, uint32(7), rh.RemoteIndex, "the reply names the initiator's FragID")
assert.Equal(t, uint64(pqCookieCounter), rh.MessageCounter)
copy(cookie[:], reply[header.Len:])
})
t.Run("a forged cookie gets a fresh one", func(t *testing.T) {
bad := cookie
bad[0] ^= 1
p, h := pqMsg1Fragment(t, 7, &bad)
before := hm.pqTable.Len()
hm.handleFragment(via, p, h)
assert.Equal(t, before, hm.pqTable.Len())
assert.Len(t, conn.take(), 1)
})
t.Run("the cookie gets the fragment in", func(t *testing.T) {
p, h := pqMsg1Fragment(t, 7, &cookie)
before := hm.pqTable.Len()
hm.handleFragment(via, p, h)
assert.Equal(t, before+1, hm.pqTable.Len())
assert.Empty(t, conn.take())
})
t.Run("a cookie is bound to its address", func(t *testing.T) {
p, h := pqMsg1Fragment(t, 7, &cookie)
hm.handleFragment(ViaSender{UdpAddr: netip.MustParseAddrPort("192.0.2.99:4242")}, p, h)
assert.Len(t, conn.take(), 1, "another address gets its own cookie instead")
})
t.Run("a reply never outgrows what prompted it", func(t *testing.T) {
p, h := pqMsg1Fragment(t, 8, nil)
hm.sendCookie(via, fragment.Header{FragID: 8}, pqFragKey{}, pqCookieReplyLen-1, time.Now())
assert.Empty(t, conn.take())
hm.handleFragment(via, p, h)
assert.Len(t, conn.take(), 1)
})
}
func TestPQHandleCookie(t *testing.T) {
hm, conn := newPQTestManager(t)
via := ViaSender{UdpAddr: netip.MustParseAddrPort("192.0.2.1:4242")}
s, err := fragment.NewSender(make([]byte, 2000), pqSubtype, 0, 1, 99)
require.NoError(t, err)
_, _, err = s.Attempt(2)
require.NoError(t, err)
hh := &HandshakeHostInfo{hostinfo: &HostInfo{}, pq: &pqAttempt{index: 99, sender: s}}
hm.indexes[99] = hh
reply := make([]byte, header.Len, pqCookieReplyLen)
header.Encode(reply, header.Version, header.Handshake, pqSubtype, 99, pqCookieCounter)
reply = append(reply, bytes16(0xab)...)
var h header.H
require.NoError(t, h.Parse(reply))
hm.handleFragment(via, reply, &h)
assert.True(t, hh.pq.heard, "a cookie reply shows the peer speaks PQ")
sent := conn.take()
require.Len(t, sent, 4, "an immediate resend of fresh symbols")
for i, p := range sent {
fh, _, err := fragment.Parse(p[header.Len:])
require.NoError(t, err)
assert.True(t, fh.HasCookie())
assert.Equal(t, fragment.Cookie(bytes16(0xab)), fh.Cookie)
assert.Equal(t, uint8(4+i), fh.Index)
}
hm.handleFragment(via, reply, &h)
assert.Empty(t, conn.take(), "immediate resends are rate limited")
t.Run("replies for other indexes are ignored", func(t *testing.T) {
other := append([]byte(nil), reply...)
header.Encode(other, header.Version, header.Handshake, pqSubtype, 98, pqCookieCounter)
var oh header.H
require.NoError(t, oh.Parse(other))
hh.pq.cookieResendAt = time.Time{}
hm.handleFragment(via, other, &oh)
assert.Empty(t, conn.take())
})
}
func bytes16(b byte) []byte {
out := make([]byte, 16)
for i := range out {
out[i] = b
}
return out
}
func TestReloadPQ(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString(`handshakes: {pq: {initiate: false}}`))
cfg := defaultHandshakeConfig
cfg.pq = pqHandshakeConfigFrom(c)
hm := NewHandshakeManager(l, newHostMap(l), newTestLighthouse(), &udp.NoopConn{}, cfg)
c.RegisterReloadCallback(hm.reloadPQ)
assert.Equal(t, pqHandshakeConfig{repairDelay: DefaultPQRepairDelay}, *hm.pqConfig())
require.NoError(t, c.ReloadConfigString(`handshakes: {pq: {initiate: true, accept: true, sticky: true, repair_delay: 25ms}}`))
assert.Equal(t, pqHandshakeConfig{initiate: true, accept: true, sticky: true, repairDelay: 25 * time.Millisecond}, *hm.pqConfig())
before := hm.pqConfig()
require.NoError(t, c.ReloadConfigString(`handshakes: {try_interval: 1s, pq: {initiate: true, accept: true, sticky: true, repair_delay: 25ms}}`))
assert.Same(t, before, hm.pqConfig(), "an unrelated change leaves the settings alone")
require.NoError(t, c.ReloadConfigString(`handshakes: {}`))
assert.Equal(t, pqHandshakeConfig{repairDelay: DefaultPQRepairDelay}, *hm.pqConfig(), "removing the section restores the defaults")
require.NoError(t, c.ReloadConfigString(`handshakes: {pq: {require: true}}`))
assert.Equal(t, pqHandshakeConfig{initiate: true, accept: true, require: true, repairDelay: DefaultPQRepairDelay}, *hm.pqConfig(), "require implies initiate and accept")
}
func TestUsePQ(t *testing.T) {
hm, _ := newPQTestManager(t)
peer := netip.MustParseAddr("10.1.0.1")
set := func(cfg pqHandshakeConfig) { hm.pq.Store(&cfg) }
// attempts plays a run of attempts, given whether each was triggered early,
// and returns which were PQ.
attempts := func(triggered ...bool) []bool {
hh := &HandshakeHostInfo{}
var out []bool
for _, tr := range triggered {
hh.counter++
hh.lastPQ = hm.usePQ(hh, peer, tr)
out = append(out, hh.lastPQ)
}
return out
}
set(pqHandshakeConfig{})
assert.Equal(t, []bool{false, false, false}, attempts(false, false, false), "off")
set(pqHandshakeConfig{initiate: true})
assert.Equal(t, []bool{true, false, true, false}, attempts(false, false, false, false), "timer retries alternate, PQ first")
assert.Equal(t, []bool{true, true, false, true, false}, attempts(false, true, false, true, false), "an early trigger is a new path, which gets PQ first")
assert.Equal(t, []bool{true, true, false}, attempts(true, true, false), "a triggered first attempt is PQ")
t.Run("hearing back stops the classical turns", func(t *testing.T) {
hh := &HandshakeHostInfo{pq: &pqAttempt{}}
hh.counter, hh.lastPQ = 1, true
hh.counter++
assert.False(t, hm.usePQ(hh, peer, false), "unheard, the classical attempt gets its turn")
hh.pq.heard = true
assert.True(t, hm.usePQ(hh, peer, false))
})
set(pqHandshakeConfig{initiate: true, accept: true, require: true})
assert.Equal(t, []bool{true, true, true, true}, attempts(false, false, true, false), "require never offers classical")
set(pqHandshakeConfig{initiate: true, sticky: true})
assert.Equal(t, []bool{true, false}, attempts(false, false), "sticky alternates until the peer has done PQ")
hm.notePQPeer(pqSubtype, []netip.Addr{peer})
assert.Equal(t, []bool{true, true, true}, attempts(false, false, false), "then only PQ")
}
+2 -68
View File
@@ -11,7 +11,6 @@ import (
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func Test_NewHandshakeManagerVpnIp(t *testing.T) {
@@ -43,6 +42,8 @@ func Test_NewHandshakeManagerVpnIp(t *testing.T) {
i2 := blah.StartHandshake(ip, nil)
assert.Same(t, i, i2)
i.remotes = NewRemoteList([]netip.Addr{}, nil)
// Adding something to pending should not affect the main hostmap
assert.Empty(t, mainHM.Hosts)
@@ -234,70 +235,3 @@ func TestHandleIncomingDispatch(t *testing.T) {
assert.Empty(t, hm.indexes, "orphan stage-2 must not create state")
})
}
// Test_PendingHandshakeHasRemotes covers create-tunnel -address: a pending hostinfo must already
// carry its RemoteList when StartHandshake runs its callback, so SetRemote there, before the
// handshake loop has run, is safe and the address becomes one the handshake is sent to.
func Test_PendingHandshakeHasRemotes(t *testing.T) {
l := test.NewLogger()
mainHM := newHostMap(l)
preferredRanges := []netip.Prefix{}
mainHM.preferredRanges.Store(&preferredRanges)
hm := NewHandshakeManager(l, mainHM, newTestLighthouse(), &udp.NoopConn{}, defaultHandshakeConfig)
vpnIp := netip.MustParseAddr("172.1.1.2")
remote := netip.MustParseAddrPort("192.168.1.9:4242")
hi := hm.StartHandshake(vpnIp, func(hh *HandshakeHostInfo) {
require.NotNil(t, hh.hostinfo.remotes, "a pending handshake must never be without its RemoteList")
hh.hostinfo.SetRemote(remote)
})
assert.Contains(t, hi.remotes.CopyAddrs(preferredRanges), remote, "the handshake must be sent to the supplied remote")
assert.Equal(t, remote, hi.GetRemote())
// A RemoteList keeps one learned address per host, so a second remote replaces the
// first — the same behaviour change-remote has on an established tunnel.
other := netip.MustParseAddrPort("192.168.1.10:4242")
hi.SetRemote(other)
assert.Equal(t, []netip.AddrPort{other}, hi.remotes.CopyAddrs(preferredRanges))
assert.Equal(t, other, hi.GetRemote())
}
// newRemotesRaceFixture builds a handshake manager with an established tunnel to vpnAddr whose
// RemoteList is the lighthouse's cached list, the state every teardown-during-handshake race starts from.
func newRemotesRaceFixture(t *testing.T, vpnAddr netip.Addr) (*HandshakeManager, *Interface, *HostInfo) {
t.Helper()
l := test.NewLogger()
mainHM := newHostMap(l)
preferredRanges := []netip.Prefix{}
mainHM.preferredRanges.Store(&preferredRanges)
lh := newTestLighthouse()
hm := NewHandshakeManager(l, mainHM, lh, &udp.NoopConn{}, defaultHandshakeConfig)
f := &Interface{hostMap: mainHM, lightHouse: lh, handshakeManager: hm, l: l}
hm.f = f
established := &HostInfo{
vpnAddrs: []netip.Addr{vpnAddr},
localIndexId: 1,
remoteIndexId: 101,
remotes: lh.QueryCache([]netip.Addr{vpnAddr}),
HandshakePacket: map[uint8][]byte{},
}
mainHM.Lock()
mainHM.unlockedAddHostInfo(established, f)
mainHM.Unlock()
return hm, f, established
}
// With no handshake pending, tearing down the last tunnel still evicts the list.
func Test_CloseTunnelEvictsWithoutPendingHandshake(t *testing.T) {
vpnAddr := netip.MustParseAddr("172.1.1.2")
_, f, established := newRemotesRaceFixture(t, vpnAddr)
f.closeTunnel(established)
assert.NotContains(t, f.hostMap.Hosts, vpnAddr)
assert.NotContains(t, f.lightHouse.addrMap, vpnAddr)
}
+2 -19
View File
@@ -24,11 +24,6 @@ type m = map[string]any
const (
Version uint8 = 1
Len = 16
// MaxOverhead is the most a payload grows on its way to the underlay.
// A relayed packet has two headers and two AEAD tags, one for the inner tunnel and another for the relay tunnel.
// Every supported cipher (today) has a 16 byte tag.
MaxOverhead = 2 * (Len + 16)
)
type MessageType uint8
@@ -67,19 +62,8 @@ const (
const (
HandshakeIXPSK0 MessageSubType = 0
HandshakeXXPSK0 MessageSubType = 1
// HandshakeIXPSK2MLKEM1024 is IX with an ephemeral ML-KEM-1024 exchange mixed
// in through the psk2 slot. Its messages exceed the MTU and travel as
// fragments (see handshake/fragment).
HandshakeIXPSK2MLKEM1024 MessageSubType = 2
)
// IsFragmented reports whether handshake messages of this subtype travel as
// fragments.
func IsFragmented(st MessageSubType) bool {
return st == HandshakeIXPSK2MLKEM1024
}
var ErrHeaderTooShort = errors.New("header is too short")
var subTypeTestMap = map[MessageSubType]string{
@@ -99,8 +83,7 @@ var subTypeMap = map[MessageType]*map[MessageSubType]string{
Test: &subTypeTestMap,
CloseTunnel: &subTypeNoneMap,
Handshake: {
HandshakeIXPSK0: "ix_psk0",
HandshakeIXPSK2MLKEM1024: "ix_psk2_mlkem1024",
HandshakeIXPSK0: "ix_psk0",
},
Control: &subTypeNoneMap,
}
@@ -211,7 +194,7 @@ func IsValidSubType(t MessageType, s MessageSubType) bool {
case Message:
return s == MessageNone || s == MessageRelay
case Handshake:
return s == HandshakeIXPSK0 || s == HandshakeIXPSK2MLKEM1024
return s == HandshakeIXPSK0
case Test:
return s == TestReply || s == TestRequest
case Control, CloseTunnel, RecvError, LightHouse:
+1 -3
View File
@@ -96,8 +96,7 @@ func TestTypeMap(t *testing.T) {
Test: &subTypeTestMap,
CloseTunnel: &subTypeNoneMap,
Handshake: {
HandshakeIXPSK0: "ix_psk0",
HandshakeIXPSK2MLKEM1024: "ix_psk2_mlkem1024",
HandshakeIXPSK0: "ix_psk0",
},
Control: &subTypeNoneMap,
}, subTypeMap)
@@ -123,7 +122,6 @@ func TestIsValidSubType(t *testing.T) {
assert.False(t, IsValidSubType(Message, 2))
assert.True(t, IsValidSubType(Handshake, HandshakeIXPSK0))
assert.True(t, IsValidSubType(Handshake, HandshakeIXPSK2MLKEM1024))
// HandshakeXXPSK0 is defined but not a wire-valid subtype.
assert.False(t, IsValidSubType(Handshake, HandshakeXXPSK0))
+58 -5
View File
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"log/slog"
"net"
"net/netip"
"slices"
"sync"
@@ -17,6 +18,7 @@ import (
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/logging"
)
const defaultPromoteEvery = 1000 // Count of packets sent before we try moving a tunnel to a preferred underlay ip address
@@ -260,11 +262,6 @@ type HostInfo struct {
// We need these to avoid replayed handshake packets creating new hostinfos which causes churn
HandshakePacket map[uint8][]byte
// A responder resending a fragmented handshake response offers symbols it has
// not sent before, at most once per retry interval.
handshakeResendAt atomic.Int64 // unix nanos of the last send
handshakeNextSymbol atomic.Uint32 // next fragment symbol index to offer
// nextLHQuery is the earliest we can ask the lighthouse for new information.
// This is used to limit lighthouse re-queries in chatty clients
nextLHQuery atomic.Int64
@@ -933,3 +930,59 @@ func (i *HostInfo) logger(l *slog.Logger) *slog.Logger {
return li
}
// Utility functions
func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
//FIXME: This function is pretty garbage
var finalAddrs []netip.Addr
ifaces, _ := net.Interfaces()
for _, i := range ifaces {
allow := allowList.AllowName(i.Name)
if l.Enabled(context.Background(), logging.LevelTrace) {
l.Log(context.Background(), logging.LevelTrace, "localAllowList.AllowName",
"interfaceName", i.Name,
"allow", allow,
)
}
if !allow {
continue
}
addrs, _ := i.Addrs()
for _, rawAddr := range addrs {
var addr netip.Addr
switch v := rawAddr.(type) {
case *net.IPNet:
//continue
addr, _ = netip.AddrFromSlice(v.IP)
case *net.IPAddr:
addr, _ = netip.AddrFromSlice(v.IP)
}
if !addr.IsValid() {
if l.Enabled(context.Background(), slog.LevelDebug) {
l.Debug("addr was invalid", "localAddr", rawAddr)
}
continue
}
addr = addr.Unmap()
if addr.IsLoopback() == false && addr.IsLinkLocalUnicast() == false {
isAllowed := allowList.Allow(addr)
if l.Enabled(context.Background(), logging.LevelTrace) {
l.Log(context.Background(), logging.LevelTrace, "localAllowList.Allow",
"localAddr", addr,
"allowed", isAllowed,
)
}
if !isAllowed {
continue
}
finalAddrs = append(finalAddrs, addr)
}
}
}
return finalAddrs
}
+18 -35
View File
@@ -10,7 +10,6 @@ import (
"github.com/slackhq/nebula/firewall"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/iputil"
"github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/noiseutil"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio"
@@ -199,8 +198,8 @@ func (f *Interface) sendInsideMessage(hostinfo *HostInfo, pkt tio.Packet, nb []b
}
err = tio.SegmentSuperpacket(pkt, func(seg []byte) error {
//relay header + header + plaintext + AEAD tag + relay tag
scratch := sendBatch.Reserve(len(seg) + header.MaxOverhead)
//relay header + header + plaintext + AEAD tag (16 bytes for both AES-GCM and ChaCha20-Poly1305) + relay tag
scratch := sendBatch.Reserve(header.Len + header.Len + len(seg) + 16 + 16)
innerPacket := f.sendInsideEncrypt(hostinfo, ci, seg, scratch[header.Len:], nb)
if innerPacket == nil {
@@ -518,9 +517,7 @@ func (f *Interface) SendVia(via *HostInfo, relay *Relay, ad, nb, out []byte, noc
err = f.writers[q].WriteTo(toSend, via.GetRemote())
if err != nil {
if l, ok := logging.RateLimited(f.l, f.udpRelayTxErrLimiter); ok {
via.logger(l).Info("Failed to WriteTo in sendVia", "error", err)
}
via.logger(f.l).Info("Failed to WriteTo in sendVia", "error", err)
}
}
@@ -529,26 +526,16 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
return
}
useRelay := !remote.IsValid() && !hostinfo.GetRemote().IsValid()
// The encrypt appends to out in place and a relayed packet is then wrapped in place,
// so out has to fit the whole packet up front or the encrypt reallocates out from under SendVia.
// We cap the MTU of tun devices and routes we create ourselves, so only a host that set a bigger MTU than we allow gets here.
need := header.Len + len(p) + ci.eKey.Overhead()
if useRelay {
// The relay tunnel runs the same cipher, and adds its own header and tag
need += header.Len + ci.eKey.Overhead()
}
if cap(out) < need {
if l, ok := logging.RateLimited(f.l, f.txOversizeLimiter); ok {
hostinfo.logger(l).Error("Dropping outbound packet, out buffer not large enough", "outCap", cap(out), "payloadLen", len(p), "relayed", useRelay)
}
return
}
fullOut := out
if useRelay {
if len(out) < header.Len {
// out always has a capacity of mtu, but not always a length greater than the header.Len.
// Grow it to make sure the next operation works.
out = out[:header.Len]
}
// Save a header's worth of data at the front of the 'out' buffer.
out = out[header.Len:header.Len]
out = out[header.Len:]
}
if noiseutil.EncryptLockNeeded {
@@ -595,22 +582,18 @@ func (f *Interface) sendNoMetrics(t header.MessageType, st header.MessageSubType
if remote.IsValid() {
err = f.writers[q].WriteTo(out, remote)
if err != nil {
if l, ok := logging.RateLimited(f.l, f.udpTxErrLimiter); ok {
hostinfo.logger(l).Error("Failed to write outgoing packet",
"error", err,
"udpAddr", remote,
)
}
hostinfo.logger(f.l).Error("Failed to write outgoing packet",
"error", err,
"udpAddr", remote,
)
}
} else if hr := hostinfo.GetRemote(); hr.IsValid() {
err = f.writers[q].WriteTo(out, hr)
if err != nil {
if l, ok := logging.RateLimited(f.l, f.udpTxErrLimiter); ok {
hostinfo.logger(l).Error("Failed to write outgoing packet",
"error", err,
"udpAddr", hr,
)
}
hostinfo.logger(f.l).Error("Failed to write outgoing packet",
"error", err,
"udpAddr", hr,
)
}
} else {
// Try to send via a relay
+18 -24
View File
@@ -24,7 +24,6 @@ import (
"github.com/slackhq/nebula/overlay"
"github.com/slackhq/nebula/overlay/batch"
"github.com/slackhq/nebula/overlay/tio"
"github.com/slackhq/nebula/ratelimit"
"github.com/slackhq/nebula/udp"
)
@@ -125,10 +124,6 @@ type Interface struct {
batchers []*batch.MultiCoalescer
wg sync.WaitGroup
udpTxErrLimiter *ratelimit.Limiter // udp direct transmit error log limiter
udpRelayTxErrLimiter *ratelimit.Limiter // udp relay transmit error log limiter
txOversizeLimiter *ratelimit.Limiter // oversized outbound packet drop log limiter
// fatalErr holds the first unexpected reader error that caused shutdown.
// nil means "no fatal error" (yet)
fatalErr atomic.Pointer[error]
@@ -203,19 +198,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
return nil, errors.New("no connection manager")
}
udpTxErrLimiter, err := ratelimit.New(1, time.Second)
if err != nil {
return nil, err
}
udpRelayTxErrLimiter, err := ratelimit.New(1, time.Second)
if err != nil {
return nil, err
}
txOversizeLimiter, err := ratelimit.New(1, time.Second)
if err != nil {
return nil, err
}
if c.routines <= 1 {
c.PinThreads = false //pinning is not useful unless there's more than one tun reader
}
@@ -248,9 +230,6 @@ func NewInterface(ctx context.Context, c *InterfaceConfig) (*Interface, error) {
conntrackCacheTimeout: c.ConntrackCacheTimeout,
cpuAffinity: c.CpuAffinity,
pinThreads: c.PinThreads,
udpTxErrLimiter: udpTxErrLimiter,
udpRelayTxErrLimiter: udpRelayTxErrLimiter,
txOversizeLimiter: txOversizeLimiter,
metricHandshakes: metrics.GetOrRegisterHistogram("handshakes", nil, metrics.NewExpDecaySample(1028, 0.015)),
metricTxDropped: metrics.GetOrRegisterCounter("udp.tx.dropped", nil),
@@ -645,15 +624,17 @@ func (f *Interface) reloadMisc(c *config.C) {
}
}
// statsEmitter updates the interface's gauges, the stats server runs it on its capture loop.
func (f *Interface) statsEmitter() func() {
func (f *Interface) emitStats(ctx context.Context, i time.Duration) {
ticker := time.NewTicker(i)
defer ticker.Stop()
udpStats := udp.NewUDPStatsEmitter(f.writers)
certExpirationGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil)
certInitiatingVersion := metrics.GetOrRegisterGauge("certificate.initiating_version", nil)
certMaxVersion := metrics.GetOrRegisterGauge("certificate.max_version", nil)
return func() {
emit := func() {
f.firewall.EmitStats()
f.handshakeManager.EmitStats()
udpStats()
@@ -670,6 +651,19 @@ func (f *Interface) statsEmitter() func() {
certMaxVersion.Update(int64(certState.v1Cert.Version()))
}
}
// Prime gauges so a Prometheus scrape that lands before the first tick
// sees real values instead of the zero defaults (issue #907).
emit()
for {
select {
case <-ctx.Done():
return
case <-ticker.C:
emit()
}
}
}
func (f *Interface) GetHostInfo(vpnIp netip.Addr) *HostInfo {
+14 -5
View File
@@ -3,6 +3,7 @@
package nebula
import (
"context"
"net/netip"
"testing"
"time"
@@ -17,8 +18,11 @@ import (
"github.com/stretchr/testify/require"
)
// Priming before the first tick (issue #907) is Start's job, see TestStatsServer_Start_primes.
func Test_statsEmitter_setsGauges(t *testing.T) {
// Test_emitStats_primesGauges covers issue #907: a Prometheus scrape that
// landed before the first ticker fire used to read 0 for the cert gauges.
// emitStats now primes the gauges before entering the ticker loop. We assert
// the gauge is zero before the first call and non-zero after.
func Test_emitStats_primesGauges(t *testing.T) {
defer metrics.DefaultRegistry.UnregisterAll()
l := test.NewLogger()
@@ -52,12 +56,17 @@ func Test_statsEmitter_setsGauges(t *testing.T) {
ifce.pki.cs.Store(cs)
ttlGauge := metrics.GetOrRegisterGauge("certificate.ttl_seconds", nil)
require.Zero(t, ttlGauge.Value(), "gauge should be zero before the emitter runs")
require.Zero(t, ttlGauge.Value(), "gauge should be zero before emitStats runs")
ifce.statsEmitter()()
// Pre-cancel the context so emitStats returns after priming the gauges
// without ever reading from ticker.C. The one hour interval is just a
// belt-and-suspenders, the test does not expect the ticker to fire.
ctx, cancel := context.WithCancel(context.Background())
cancel()
ifce.emitStats(ctx, time.Hour)
ttl := ttlGauge.Value()
assert.Positive(t, ttl, "ttl gauge should be set by one run of the emitter")
assert.Positive(t, ttl, "ttl gauge should be primed by emitStats before its first tick")
assert.LessOrEqual(t, ttl, int64(3600))
assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.initiating_version", nil).Value())
assert.Equal(t, int64(cert.Version1), metrics.GetOrRegisterGauge("certificate.max_version", nil).Value())
+13 -3
View File
@@ -45,7 +45,8 @@ func setTransportChecksum4(packet []byte) {
if ihl < ipv4.HeaderLen || end < ihl || end > len(packet) {
return
}
// do not attempt to set checksums for fragments
// The checksum covers the whole datagram, which a fragment (MF set or a
// non-zero offset) does not carry.
if binary.BigEndian.Uint16(packet[6:8])&0x3fff != 0 {
return
}
@@ -67,7 +68,10 @@ func setTransportChecksum6(packet []byte) {
return
}
// we must find the transport header in order to write a checksum
// The checksum covers the whole datagram, which a fragment does not carry.
// An unknown extension header hides where the transport header starts. A
// chain longer than the walk's budget ends it early, at an offset that was
// never checked against the packet.
proto, offset, _, anyFragment, err := IPv6FindUpperProtocol(packet[:end])
if err != nil || anyFragment || offset >= end {
return
@@ -105,7 +109,8 @@ func transportExtent(transport []byte, proto uint8) ([]byte, bool) {
// writeTransportChecksum stores the checksum of transport, taken over the
// pseudo-header sum csum, in the header's checksum field. A UDP checksum that
// computes to zero goes on the wire as 0xffff: zero means no checksum was
// computed (RFC 768), and over IPv6 the checksum is mandatory (RFC 8200 section 8.1).
// computed (RFC 768), and over IPv6 the checksum is mandatory (RFC 8200
// section 8.1).
func writeTransportChecksum(transport []byte, proto uint8, csum uint32) {
var at, minLen int
switch proto {
@@ -128,6 +133,11 @@ func writeTransportChecksum(transport []byte, proto uint8, csum uint32) {
binary.BigEndian.PutUint16(transport[at:], sum)
}
// fold reduces a pseudo-header sum to the 16 bit seed Checksum takes. Carrying
// the high half back into the low half is what keeps the reduction lossless, so
// the seed sums exactly as the wider value would; 0xffff is its fixed point.
// Every term of that sum comes from a 16 bit field, so it stays far below the
// width at which the accumulator would wrap.
func fold(csum uint32) uint16 {
for csum > 0xffff {
csum = (csum >> 16) + (csum & 0xffff)
+16 -13
View File
@@ -345,11 +345,14 @@ func ipv6CreateRejectTCPPacket(packet []byte, out []byte, offset int) []byte {
return out
}
// IPv6FindUpperProtocol walks the ipv6 extension header chain and returns the upper layer protocol, the offset it begins at, and whether the packet is a non-first fragment.
// Only the RFC 8200 and IANA extension headers below are walked.
// Everything else, including Mobility (135), HIP (139), Shim6 (140), experimental 253/254, and real upper layer protocols like SCTP or GRE, is terminal.
// For a non-first fragment the returned protocol is the fragmented protocol and offset points at the fragment header, there is no transport header to locate.
// Returns ErrIPv6CouldNotFindPayload if packet is smaller than an ipv6 header, or the chain is truncated, or if more than 8 extension headers are present.
// IPv6FindUpperProtocol walks the ipv6 extension header chain and returns the upper layer protocol, the
// offset it begins at, and whether the packet is a non-first fragment. Only the RFC 8200 and IANA extension
// headers below are walked. Everything else, including Mobility (135), HIP (139), Shim6 (140), experimental
// 253/254, and real upper layer protocols like SCTP or GRE, is terminal. Walking those as extension headers
// is a firewall bypass, so they fail closed. For a non-first fragment the returned protocol is the fragmented
// protocol and offset points at the fragment header, there is no transport header to locate. Returns
// ErrIPv6CouldNotFindPayload if packet is smaller than an ipv6 header or the chain is truncated before a
// terminal protocol is reached.
func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragment bool, anyFragment bool, err error) {
const maxIPv6ExtHeaders = 8
if len(packet) < ipv6.HeaderLen {
@@ -362,14 +365,14 @@ func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragm
switch nextHeader {
case 0, 43, 60: // Hop-by-Hop, Routing, Destination
if len(packet) < offset+2 {
return 0, 0, false, false, ErrIPv6CouldNotFindPayload
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload
}
nextHeader = packet[offset]
offset += (int(packet[offset+1]) + 1) << 3
case 44: // Fragment
if len(packet) < offset+8 {
return 0, 0, false, false, ErrIPv6CouldNotFindPayload
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload
}
anyFragment = true
// Non-first fragments carry no transport header, report the fragmented protocol and stop
@@ -381,21 +384,21 @@ func IPv6FindUpperProtocol(packet []byte) (nextHeader uint8, offset int, isFragm
case 51: // AH
if len(packet) < offset+2 {
return 0, 0, false, false, ErrIPv6CouldNotFindPayload
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload
}
nextHeader = packet[offset]
offset += (int(packet[offset+1]) + 2) << 2
default:
// A prior extension header can declare a length that advances offset past the packet. The terminal
// protocol's header isn't actually here, so treat the chain as truncated rather than classifying it.
if offset > len(packet) {
// A prior extension header can declare a length that advances offset past the packet, reject this.
return 0, 0, false, false, ErrIPv6CouldNotFindPayload
return nextHeader, offset, isFragment, anyFragment, ErrIPv6CouldNotFindPayload
}
return nextHeader, offset, isFragment, anyFragment, nil //success!
return nextHeader, offset, isFragment, anyFragment, nil
}
}
// Fail closed
return 0, 0, false, false, ErrIPv6CouldNotFindPayload
return nextHeader, offset, isFragment, anyFragment, nil
}
func CreateICMPEchoResponse(packet, out []byte) []byte {
+5 -17
View File
@@ -529,17 +529,6 @@ func Test_IPv6FindUpperProtocol(t *testing.T) {
firstFragToUDP := []byte{17, 0, 0, 1, 0, 0, 0, 1} // frag offset 0, M=1, next = UDP
nonFirstFrag := []byte{17, 0, 0, 9, 0, 0, 0, 1} // frag offset non-zero, next = UDP
transport := []byte{0, 80, 1, 187, 0, 0, 0, 0} // stand-in bytes, IPv6FindUpperProtocol never reads ports
extToHbH := []byte{0, 0, 0, 0, 0, 0, 0, 0} // len 0 -> 8 bytes, next = Hop-by-Hop
// 8 walked headers spend the budget before the 9th slot's terminal is ever classified. This is the
// report 4040546 shape: without failing closed the walk hands back offset 104, and a caller reads ports
// from the attacker's bytes there.
var budgetChain []byte
for range 7 {
budgetChain = append(budgetChain, extToHbH...)
}
budgetChain = append(budgetChain, extToTCP...)
budgetChain = append(budgetChain, transport...)
tests := []struct {
name string
@@ -560,10 +549,9 @@ func Test_IPv6FindUpperProtocol(t *testing.T) {
{"first fragment walks to transport", 44, append(firstFragToUDP, transport...), 17, ipv6.HeaderLen + 8, false, true, nil},
{"non-first fragment stops", 44, append(nonFirstFrag, transport...), 17, ipv6.HeaderLen, true, true, nil},
{"unknown protocol is terminal", 132, transport, 132, ipv6.HeaderLen, false, false, nil}, // SCTP
{"truncated extension header", 0, nil, 0, 0, false, false, ErrIPv6CouldNotFindPayload},
{"truncated extension header", 0, nil, 0, ipv6.HeaderLen, false, false, ErrIPv6CouldNotFindPayload},
// Destination Options with a declared length (255+1)*8 = 2048 that runs past the 48 byte buffer, next = SCTP
{"extension length past buffer", 0, []byte{132, 255, 0, 0, 0, 0, 0, 0}, 0, 0, false, false, ErrIPv6CouldNotFindPayload},
{"budget exhausted fails closed", 0, budgetChain, 0, 0, false, false, ErrIPv6CouldNotFindPayload},
{"extension length past buffer", 60, []byte{132, 255, 0, 0, 0, 0, 0, 0}, 132, ipv6.HeaderLen + 2048, false, false, ErrIPv6CouldNotFindPayload},
}
for _, tt := range tests {
@@ -571,10 +559,10 @@ func Test_IPv6FindUpperProtocol(t *testing.T) {
packet := makeIPv6Packet(src, dst, tt.nextHeader, tt.payload)
proto, offset, isFragment, anyFragment, err := IPv6FindUpperProtocol(packet)
if tt.wantErr != nil {
require.ErrorIs(t, err, tt.wantErr)
} else {
require.NoError(t, err)
assert.ErrorIs(t, err, tt.wantErr)
return
}
require.NoError(t, err)
assert.Equal(t, tt.wantProto, proto)
assert.Equal(t, tt.wantOffset, offset)
assert.Equal(t, tt.wantFragment, isFragment)
+57 -88
View File
@@ -15,12 +15,10 @@ import (
"time"
"github.com/gaissmai/bart"
"github.com/rcrowley/go-metrics"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/logging"
"github.com/slackhq/nebula/nicfinder"
"github.com/slackhq/nebula/udp"
"github.com/slackhq/nebula/util"
)
@@ -36,18 +34,12 @@ type LightHouse struct {
myVpnNetworks []netip.Prefix
myVpnNetworksTable *bart.Lite
// myVpnAddrsTable contains our overlay host addrs, as opposed to the overlay networks
myVpnAddrsTable *bart.Lite
punchy *Punchy
punchy *Punchy
// localAddrsFn enumerates the underlay addresses we advertise. It is a field so tests can supply simulated
// addresses rather than whatever this machine's NICs happen to be. Set it before Start.
localAddrsFn func(*LocalAllowList) []netip.Addr
// lastLocalAddrsErr is the previous localAddrsFn failure. Enumeration runs on every update, so an
// unchanged failure is demoted to Debug rather than warning every lighthouse.interval forever.
lastLocalAddrsErr atomic.Pointer[string]
// Local cache of answers from light houses
// map of vpn addr to answers
addrMap map[netip.Addr]*RemoteList
@@ -82,8 +74,6 @@ type LightHouse struct {
updateTrigger chan struct{}
queryChan chan netip.Addr
// queryDropped counts queries QueryServer dropped because queryChan was full.
queryDropped metrics.Counter
calculatedRemotes atomic.Pointer[bart.Table[[]*calculatedRemote]] // Maps VpnAddr to []*calculatedRemote
@@ -114,19 +104,15 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
amLighthouse: amLighthouse,
myVpnNetworks: cs.myVpnNetworks,
myVpnNetworksTable: cs.myVpnNetworksTable,
myVpnAddrsTable: cs.myVpnAddrsTable,
addrMap: make(map[netip.Addr]*RemoteList),
nebulaPort: nebulaPort,
punchy: p,
updateTrigger: make(chan struct{}, 1),
queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 1024)),
queryDropped: metrics.GetOrRegisterCounter("lighthouse.query.dropped", nil),
queryChan: make(chan netip.Addr, c.GetUint32("handshakes.query_buffer", 64)),
l: l,
}
h.localAddrsFn = func(al *LocalAllowList) []netip.Addr {
addrs, err := nicfinder.Find(ctx, h.l, al)
h.logLocalAddrsErr(err)
return addrs
return localAddrs(h.l, al)
}
lighthouses := make([]netip.Addr, 0)
@@ -359,13 +345,7 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
}
}
// am_relay decides whether relay.relays applies to us at all
if initial || c.HasChanged("relay.relays") || c.HasChanged("relay.am_relay") {
var before []netip.Addr
if !initial {
before = lh.GetRelaysForMe()
}
if initial || c.HasChanged("relay.relays") {
switch c.GetBool("relay.am_relay", false) {
case true:
// Relays aren't allowed to specify other relays
@@ -390,11 +370,6 @@ func (lh *LightHouse) reload(c *config.C, initial bool) error {
}
lh.relaysForMe.Store(&relaysForMe)
}
// Tell the lighthouses now rather than at the next interval, but only when what we advertise changed
if !initial && !slices.Equal(before, lh.GetRelaysForMe()) {
lh.TriggerUpdate()
}
}
return nil
@@ -527,33 +502,26 @@ func (lh *LightHouse) Query(vpnAddr netip.Addr) *RemoteList {
return nil
}
// QueryServer is asynchronous and non-blocking, no reply should be expected.
// If the query channel is full, the query is dropped.
// QueryServer is asynchronous so no reply should be expected
func (lh *LightHouse) QueryServer(vpnAddr netip.Addr) {
// Don't put lighthouse addrs in the query channel because we can't query lighthouses about lighthouses
if lh.amLighthouse || lh.IsLighthouseAddr(vpnAddr) {
return
}
select {
case lh.queryChan <- vpnAddr:
default:
lh.queryDropped.Inc(1)
}
lh.queryChan <- vpnAddr
}
func (lh *LightHouse) QueryCache(vpnAddrs []netip.Addr) *RemoteList {
lh.RLock()
if v, ok := lh.addrMap[vpnAddrs[0]]; ok {
lh.RUnlock()
return v
rl, ok := lh.findRemoteList(vpnAddrs)
if ok {
return rl
}
lh.RUnlock()
lh.Lock()
defer lh.Unlock()
// Add an entry if we don't already have one
return lh.unlockedGetRemoteList(vpnAddrs) //todo CERT-V2 this contains addrmap lookups we could potentially skip
return lh.unlockedGetRemoteList(vpnAddrs) //todo this re-calls unlockedFindRemoteList
}
// queryAndPrepMessage is a lock helper on RemoteList, assisting the caller to build a lighthouse message containing
@@ -699,23 +667,58 @@ func (lh *LightHouse) addCalculatedRemotes(vpnAddr netip.Addr) bool {
return len(calculatedV4) > 0 || len(calculatedV6) > 0
}
func (lh *LightHouse) findRemoteList(vpnAddrs []netip.Addr) (*RemoteList, bool) {
lh.RLock()
defer lh.RUnlock()
return lh.unlockedFindRemoteList(vpnAddrs)
}
// unlockedFindRemoteList checks addrMap for each of vpnAddrs. It returns the first RemoteList found,
// and true if that RemoteList is present for all vpnAddrs.
// If false, it means the addrMap, and possibly the RemoteList, need to be corrected.
func (lh *LightHouse) unlockedFindRemoteList(vpnAddrs []netip.Addr) (*RemoteList, bool) {
var am *RemoteList
//todo: if a host with addresses A and B is "split", so it only has address A and a new host has only address B
//todo: we don't handly that correctly, I'm pretty sure.
missingOrDifferent := false
for _, addr := range vpnAddrs {
found, ok := lh.addrMap[addr]
if !ok {
missingOrDifferent = true
} else if am == nil {
am = found //the first list we find wins
} else if am != found {
missingOrDifferent = true
}
}
return am, !missingOrDifferent
}
// unlockedGetRemoteList assumes you have the lh lock
func (lh *LightHouse) unlockedGetRemoteList(allAddrs []netip.Addr) *RemoteList {
// before we go and make a new remotelist, we need to make sure we don't have one for any of this set of vpnaddrs yet
for i, addr := range allAddrs {
am, ok := lh.addrMap[addr]
if ok {
if i != 0 {
lh.addrMap[allAddrs[0]] = am
}
return am
am, ok := lh.unlockedFindRemoteList(allAddrs)
// we failed to find any RemoteLists: make a new one, fill out the addrMap
if am == nil {
am = NewRemoteList(allAddrs, lh.shouldAdd)
for _, addr := range allAddrs {
lh.addrMap[addr] = am
}
return am
}
// we found one! Do we need to fix it?
if !ok {
am.Lock()
am.vpnAddrs = make([]netip.Addr, len(allAddrs))
copy(am.vpnAddrs, allAddrs)
am.Unlock()
for _, addr := range allAddrs {
lh.addrMap[addr] = am
}
}
am := NewRemoteList(allAddrs, lh.shouldAdd)
for _, addr := range allAddrs {
lh.addrMap[addr] = am
}
return am
}
@@ -943,26 +946,6 @@ func (lh *LightHouse) TriggerUpdate() {
}
}
// logLocalAddrsErr reports a localAddrs failure at Warn the first time it is seen and at Debug while
// it persists unchanged, so a permanent failure does not warn on every update forever.
func (lh *LightHouse) logLocalAddrsErr(err error) {
if err == nil {
lh.lastLocalAddrsErr.Store(nil)
return
}
msg := err.Error()
prev := lh.lastLocalAddrsErr.Swap(&msg)
if prev != nil && *prev == msg {
if lh.l.Enabled(context.Background(), slog.LevelDebug) {
lh.l.Debug("Failed to collect local addresses to advertise", "error", err)
}
return
}
lh.l.Warn("Failed to collect local addresses to advertise", "error", err)
}
func (lh *LightHouse) SendUpdate() {
var v4 []*V4AddrPort
var v6 []*V6AddrPort
@@ -1208,17 +1191,6 @@ func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, fromVpnAddrs []neti
return
}
// Don't respond to requests for us.
if lhh.lh.myVpnAddrsTable.Contains(queryVpnAddr) {
if lhh.l.Enabled(context.Background(), slog.LevelDebug) {
lhh.l.Debug("Ignoring HostQuery for one of my own addresses",
"fromVpnAddrs", fromVpnAddrs,
"queryVpnAddr", queryVpnAddr,
)
}
return
}
found, ln, err := lhh.lh.queryAndPrepMessage(queryVpnAddr, func(c *cache) (int, error) {
n = lhh.resetMeta()
n.Type = NebulaMeta_HostQueryReply
@@ -1255,13 +1227,10 @@ func (lhh *LightHouseHandler) handleHostQuery(n *NebulaMeta, fromVpnAddrs []neti
// sendHostPunchNotification signals the other side to punch some zero byte udp packets
func (lhh *LightHouseHandler) sendHostPunchNotification(n *NebulaMeta, fromVpnAddrs []netip.Addr, punchNotifDest netip.Addr, w EncWriter) {
whereToPunch := fromVpnAddrs[0]
// Resolve the target before taking the querier's RemoteList lock below. GetHostInfo takes the
// hostmap lock, and the hostmap is taken before the lighthouse, which is taken before a
// RemoteList, so looking it up inside the callback would deadlock.
targetHI := lhh.lh.ifce.GetHostInfo(punchNotifDest)
found, ln, err := lhh.lh.queryAndPrepMessage(whereToPunch, func(c *cache) (int, error) {
n = lhh.resetMeta()
n.Type = NebulaMeta_HostPunchNotification
targetHI := lhh.lh.ifce.GetHostInfo(punchNotifDest)
var useVersion cert.Version
if targetHI == nil {
useVersion = lhh.lh.ifce.GetCertState().initiatingVersion
+147 -286
View File
@@ -1,21 +1,16 @@
package nebula
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"net/netip"
"sync"
"testing"
"time"
"github.com/gaissmai/bart"
"github.com/slackhq/nebula/cert"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.yaml.in/yaml/v3"
@@ -32,27 +27,15 @@ func TestOldIPv4Only(t *testing.T) {
assert.Equal(t, binary.BigEndian.Uint32(bp[:]), m.GetAddr())
}
func testCertState(networks ...netip.Prefix) *CertState {
cs := &CertState{
myVpnNetworks: networks,
myVpnNetworksTable: new(bart.Lite),
myVpnAddrs: make([]netip.Addr, 0, len(networks)),
myVpnAddrsTable: new(bart.Lite),
}
for _, n := range networks {
cs.myVpnNetworksTable.Insert(n)
cs.myVpnAddrs = append(cs.myVpnAddrs, n.Addr())
cs.myVpnAddrsTable.Insert(netip.PrefixFrom(n.Addr(), n.Addr().BitLen()))
}
return cs
}
func Test_lhStaticMapping(t *testing.T) {
l := test.NewLogger()
myVpnNet := netip.MustParsePrefix("10.128.0.1/16")
cs := testCertState(myVpnNet)
nt := new(bart.Lite)
nt.Insert(myVpnNet)
cs := &CertState{
myVpnNetworks: []netip.Prefix{myVpnNet},
myVpnNetworksTable: nt,
}
lh1 := "10.128.0.2"
c := config.NewC(l)
@@ -72,7 +55,12 @@ func Test_lhStaticMapping(t *testing.T) {
func TestReloadLighthouseInterval(t *testing.T) {
l := test.NewLogger()
myVpnNet := netip.MustParsePrefix("10.128.0.1/16")
cs := testCertState(myVpnNet)
nt := new(bart.Lite)
nt.Insert(myVpnNet)
cs := &CertState{
myVpnNetworks: []netip.Prefix{myVpnNet},
myVpnNetworksTable: nt,
}
lh1 := "10.128.0.2"
c := config.NewC(l)
@@ -102,7 +90,12 @@ func TestReloadLighthouseInterval(t *testing.T) {
func BenchmarkLighthouseHandleRequest(b *testing.B) {
l := test.NewLogger()
myVpnNet := netip.MustParsePrefix("10.128.0.1/0")
cs := testCertState(myVpnNet)
nt := new(bart.Lite)
nt.Insert(myVpnNet)
cs := &CertState{
myVpnNetworks: []netip.Prefix{myVpnNet},
myVpnNetworksTable: nt,
}
c := config.NewC(l)
lh, err := NewLightHouseFromConfig(b.Context(), l, c, cs, nil, nil)
@@ -202,7 +195,12 @@ func TestLighthouse_Memory(t *testing.T) {
c.Settings["listen"] = map[string]any{"port": 4242}
myVpnNet := netip.MustParsePrefix("10.128.0.1/24")
cs := testCertState(myVpnNet)
nt := new(bart.Lite)
nt.Insert(myVpnNet)
cs := &CertState{
myVpnNetworks: []netip.Prefix{myVpnNet},
myVpnNetworksTable: nt,
}
lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
lh.ifce = &mockEncWriter{}
require.NoError(t, err)
@@ -282,7 +280,12 @@ func TestLighthouse_reload(t *testing.T) {
c.Settings["listen"] = map[string]any{"port": 4242}
myVpnNet := netip.MustParsePrefix("10.128.0.1/24")
cs := testCertState(myVpnNet)
nt := new(bart.Lite)
nt.Insert(myVpnNet)
cs := &CertState{
myVpnNetworks: []netip.Prefix{myVpnNet},
myVpnNetworksTable: nt,
}
lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
require.NoError(t, err)
@@ -312,7 +315,12 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
}
myVpnNet := netip.MustParsePrefix("10.128.0.1/24")
cs := testCertState(myVpnNet)
nt := new(bart.Lite)
nt.Insert(myVpnNet)
cs := &CertState{
myVpnNetworks: []netip.Prefix{myVpnNet},
myVpnNetworksTable: nt,
}
lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
require.NoError(t, err)
@@ -421,9 +429,7 @@ func TestLighthouse_reloadStaticHostMap(t *testing.T) {
assert.Equal(t, []netip.AddrPort{netip.MustParseAddrPort("3.3.3.3:4242")}, rl.CopyAddrs([]netip.Prefix{}))
}
// sendLHHostRequest delivers a HostQuery to lhh and hands back the writer that
// captured what it emitted. Pass a nil filter to see every message.
func sendLHHostRequest(fromAddr netip.AddrPort, myVpnIp, queryVpnIp netip.Addr, lhh *LightHouseHandler, filter *NebulaMeta_MessageType) *testEncWriter {
func newLHHostRequest(fromAddr netip.AddrPort, myVpnIp, queryVpnIp netip.Addr, lhh *LightHouseHandler) testLhReply {
req := &NebulaMeta{
Type: NebulaMeta_HostQuery,
Details: &NebulaMetaDetails{},
@@ -441,59 +447,12 @@ func sendLHHostRequest(fromAddr netip.AddrPort, myVpnIp, queryVpnIp netip.Addr,
panic(err)
}
w := &testEncWriter{metaFilter: filter}
lhh.HandleRequest(fromAddr, []netip.Addr{myVpnIp}, b, w)
return w
}
func newLHHostRequest(fromAddr netip.AddrPort, myVpnIp, queryVpnIp netip.Addr, lhh *LightHouseHandler) testLhReply {
filter := NebulaMeta_HostQueryReply
return sendLHHostRequest(fromAddr, myVpnIp, queryVpnIp, lhh, &filter).lastReply
}
func TestLighthouse_IgnoresHostQueryForItself(t *testing.T) {
// Validate that we don't answer host queries for our own address.
l := test.NewLogger()
myVpnNet := netip.MustParsePrefix("10.128.0.1/24")
myVpnIp := myVpnNet.Addr()
c := config.NewC(l)
c.Settings["lighthouse"] = map[string]any{"am_lighthouse": true}
c.Settings["listen"] = map[string]any{"port": 4242}
// Add a static_host_map entry for ourselves, so our address
// is in the addrMap.
c.Settings["static_host_map"] = map[string]any{
myVpnIp.String(): []any{"192.168.100.1:4242"},
w := &testEncWriter{
metaFilter: &filter,
}
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(myVpnNet), nil, nil)
require.NoError(t, err)
lh.ifce = &mockEncWriter{}
lhh := lh.NewRequestHandler()
peerVpnIp := netip.MustParseAddr("10.128.0.2")
peerUdpAddr := netip.MustParseAddrPort("10.0.0.2:4242")
otherVpnIp := netip.MustParseAddr("10.128.0.3")
otherUdpAddr := netip.MustParseAddrPort("10.0.0.3:4242")
newLHHostUpdate(peerUdpAddr, peerVpnIp, []netip.AddrPort{peerUdpAddr}, lhh)
newLHHostUpdate(otherUdpAddr, otherVpnIp, []netip.AddrPort{otherUdpAddr}, lhh)
// Control: a query about a real peer is still answered, and still ends with
// the punch notification aimed at the host that was asked about.
w := sendLHHostRequest(peerUdpAddr, peerVpnIp, otherVpnIp, lhh, nil)
require.NotNil(t, w.lastReply.msg)
assert.Equal(t, NebulaMeta_HostPunchNotification, w.lastReply.msg.Type)
assert.Equal(t, otherVpnIp, w.lastReply.vpnIp)
// Now validate that we don't send to ourselves.
found, _, err := lh.queryAndPrepMessage(myVpnIp, func(*cache) (int, error) { return 0, nil })
require.NoError(t, err)
require.True(t, found, "the lighthouse should hold a cache entry for its own address")
w = sendLHHostRequest(peerUdpAddr, peerVpnIp, myVpnIp, lhh, nil)
assert.Nil(t, w.lastReply.msg, "a query about our own address must produce no reply and no punch notification")
lhh.HandleRequest(fromAddr, []netip.Addr{myVpnIp}, b, w)
return w.lastReply
}
func newLHHostUpdate(fromAddr netip.AddrPort, vpnIp netip.Addr, addrs []netip.AddrPort, lhh *LightHouseHandler) {
@@ -683,7 +642,12 @@ func TestLighthouse_Dont_Delete_Static_Hosts(t *testing.T) {
}
myVpnNet := netip.MustParsePrefix("10.128.0.1/24")
cs := testCertState(myVpnNet)
nt := new(bart.Lite)
nt.Insert(myVpnNet)
cs := &CertState{
myVpnNetworks: []netip.Prefix{myVpnNet},
myVpnNetworksTable: nt,
}
lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
require.NoError(t, err)
lh.ifce = &mockEncWriter{}
@@ -744,7 +708,12 @@ func TestLighthouse_DeletesWork(t *testing.T) {
}
myVpnNet := netip.MustParsePrefix("10.128.0.1/24")
cs := testCertState(myVpnNet)
nt := new(bart.Lite)
nt.Insert(myVpnNet)
cs := &CertState{
myVpnNetworks: []netip.Prefix{myVpnNet},
myVpnNetworksTable: nt,
}
lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
require.NoError(t, err)
lh.ifce = &mockEncWriter{}
@@ -770,230 +739,122 @@ func TestLighthouse_DeletesWork(t *testing.T) {
assert.Nil(t, out)
}
func TestLightHouse_logLocalAddrsErr(t *testing.T) {
out := &bytes.Buffer{}
lh := &LightHouse{l: test.NewLoggerWithOutput(out)}
// The first sighting of a failure warns.
lh.logLocalAddrsErr(errors.New("permission denied"))
assert.Contains(t, out.String(), "level=WARN")
assert.Contains(t, out.String(), "permission denied")
// Repeating unchanged does not warn again, which is what keeps a permanent failure from warning
// on every lighthouse.interval for the life of the process.
out.Reset()
lh.logLocalAddrsErr(errors.New("permission denied"))
assert.NotContains(t, out.String(), "level=WARN")
// A different failure is a new event and warns.
out.Reset()
lh.logLocalAddrsErr(errors.New("something else"))
assert.Contains(t, out.String(), "level=WARN")
assert.Contains(t, out.String(), "something else")
// Recovering resets, so the same failure returning later warns again.
out.Reset()
lh.logLocalAddrsErr(nil)
assert.Empty(t, out.String())
lh.logLocalAddrsErr(errors.New("something else"))
assert.Contains(t, out.String(), "level=WARN")
}
// heldEncWriter holds the query worker in its first GetHostInfo until release closes.
type heldEncWriter struct {
mockEncWriter
once sync.Once
held chan struct{}
release chan struct{}
}
func (w *heldEncWriter) GetHostInfo(netip.Addr) *HostInfo {
w.once.Do(func() { close(w.held) })
<-w.release
return nil
}
// QueryServer never waits on the query worker, since the packet readers call it and nothing drains the channel at
// shutdown; a query that finds the channel full is dropped and counted. The worker is held on its first query so the
// channel fills.
func TestLighthouse_QueryServerDoesNotBlock(t *testing.T) {
l := test.NewLogger()
cs := testCertState(netip.MustParsePrefix("10.128.0.1/16"))
lh1 := "10.128.0.2"
c := config.NewC(l)
c.Settings["lighthouse"] = map[string]any{"hosts": []any{lh1}}
c.Settings["static_host_map"] = map[string]any{lh1: []any{"1.1.1.1:4242"}}
c.Settings["handshakes"] = map[string]any{"query_buffer": 1}
lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
require.NoError(t, err)
w := &heldEncWriter{held: make(chan struct{}), release: make(chan struct{})}
lh.ifce = w
t.Cleanup(func() { close(w.release) })
lh.QueryServer(netip.MustParseAddr("10.128.0.10"))
select {
case <-w.held:
case <-time.After(5 * time.Second):
t.Fatal("the query worker never took the first query")
// newLHHostUpdateV2 sends a v2-style HostUpdateNotification where the sending tunnel carries
// multiple vpn addrs (a dual-stack v2 cert). Details.VpnAddr is left blank like SendUpdate does.
func newLHHostUpdateV2(fromAddr netip.AddrPort, vpnAddrs []netip.Addr, addrs []netip.AddrPort, lhh *LightHouseHandler) {
req := &NebulaMeta{
Type: NebulaMeta_HostUpdateNotification,
Details: &NebulaMetaDetails{},
}
dropped := lh.queryDropped.Count()
done := make(chan struct{})
go func() {
lh.QueryServer(netip.MustParseAddr("10.128.0.11"))
lh.QueryServer(netip.MustParseAddr("10.128.0.12"))
close(done)
}()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("QueryServer blocked on a full query channel")
for _, v := range addrs {
if v.Addr().Is4() {
req.Details.V4AddrPorts = append(req.Details.V4AddrPorts, netAddrToProtoV4AddrPort(v.Addr(), v.Port()))
} else {
req.Details.V6AddrPorts = append(req.Details.V6AddrPorts, netAddrToProtoV6AddrPort(v.Addr(), v.Port()))
}
}
assert.EqualValues(t, 1, lh.queryDropped.Count()-dropped)
b, err := req.Marshal()
if err != nil {
panic(err)
}
lhh.HandleRequest(fromAddr, vpnAddrs, b, &testEncWriter{})
}
// hostmapEncWriter answers GetHostInfo from a real hostmap, the way Interface does.
type hostmapEncWriter struct {
mockEncWriter
hm *HostMap
}
func (w *hostmapEncWriter) GetHostInfo(a netip.Addr) *HostInfo { return w.hm.QueryVpnAddr(a) }
func (w *hostmapEncWriter) GetCertState() *CertState {
return &CertState{initiatingVersion: cert.Version1}
}
// A lighthouse runs four paths that each hold one lock while waiting for the next: CheckAndComplete
// holds the hostmap and takes the handshake manager, StartHandshake holds the handshake manager and
// takes the lighthouse, a host update holds the lighthouse and takes the sender's RemoteList, and a
// punch notification holds the querier's RemoteList while it resolves the target. If that last step
// reaches the hostmap under the RemoteList lock, the four deadlock. Drive them all at once.
func TestLighthouse_PunchNotificationDoesNotCloseLockCycle(t *testing.T) {
func newIssue1868Lighthouse(t *testing.T) (*LightHouse, *LightHouseHandler) {
l := test.NewLogger()
myVpnNet := netip.MustParsePrefix("10.128.0.1/24")
c := config.NewC(l)
c.Settings["lighthouse"] = map[string]any{"am_lighthouse": true}
c.Settings["listen"] = map[string]any{"port": 4242}
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(myVpnNet), nil, nil)
require.NoError(t, err)
mainHM := newHostMap(l)
pr := []netip.Prefix{}
mainHM.preferredRanges.Store(&pr)
lh.ifce = &hostmapEncWriter{hm: mainHM}
hm := NewHandshakeManager(l, mainHM, lh, &udp.NoopConn{}, defaultHandshakeConfig)
f := &Interface{hostMap: mainHM, lightHouse: lh, handshakeManager: hm, l: l}
hm.f = f
peer := netip.MustParseAddr("10.128.0.2")
peerUdp := netip.MustParseAddrPort("10.0.0.2:4242")
target := netip.MustParseAddr("10.128.0.3")
targetUdp := netip.MustParseAddrPort("10.0.0.3:4242")
outbound := netip.MustParseAddr("10.128.0.4")
inbound := netip.MustParseAddr("10.128.0.5")
seed := lh.NewRequestHandler()
newLHHostUpdate(peerUdp, peer, []netip.AddrPort{peerUdp}, seed)
newLHHostUpdate(targetUdp, target, []netip.AddrPort{targetUdp}, seed)
stop := make(chan struct{})
var wg sync.WaitGroup
loop := func(body func()) {
defer wg.Done()
for {
select {
case <-stop:
return
default:
body()
}
}
myVpnNet4 := netip.MustParsePrefix("10.128.0.1/24")
myVpnNet6 := netip.MustParsePrefix("fd00::1/64")
nt := new(bart.Lite)
nt.Insert(myVpnNet4)
nt.Insert(myVpnNet6)
cs := &CertState{
myVpnNetworks: []netip.Prefix{myVpnNet4, myVpnNet6},
myVpnNetworksTable: nt,
}
wg.Add(4)
lh, err := NewLightHouseFromConfig(t.Context(), l, c, cs, nil, nil)
require.NoError(t, err)
lh.ifce = &mockEncWriter{}
return lh, lh.NewRequestHandler()
}
// Outbound handshakes keep starting and being dropped.
go loop(func() {
hm.DeleteHostInfo(hm.StartHandshake(outbound, nil))
})
// Scenario A: host registers via v2 (both addrs), then a v1 handshake with the same host completes on
// the lighthouse (rehandshake after cert renewal, relay-initiated handshake, traffic to the LH's v4
// addr...). handshake_manager does QueryCache(vpnAddrs) + RefreshFromHandshake(vpnAddrs) with the v1
// cert's single address, which truncates RemoteList.vpnAddrs.
func TestLighthouse_Issue1868_V1HandshakeTruncatesVpnAddrs(t *testing.T) {
lh, lhh := newIssue1868Lighthouse(t)
// Inbound handshakes keep completing and being torn down.
var idx uint32
go loop(func() {
idx++
hi := &HostInfo{
vpnAddrs: []netip.Addr{inbound},
localIndexId: idx,
remoteIndexId: idx,
remotes: lh.QueryCache([]netip.Addr{inbound}),
HandshakePacket: map[uint8][]byte{handshakePacketStage0: {byte(idx)}},
}
if _, err := hm.CheckAndComplete(hi, handshakePacketStage0, f); err == nil {
mainHM.DeleteHostInfo(hi)
}
})
hostV4 := netip.MustParseAddr("10.128.0.3")
hostV6 := netip.MustParseAddr("fd00::3")
hostUdp := netip.MustParseAddrPort("192.0.2.3:4242")
hostLan := netip.MustParseAddrPort("10.0.0.3:4242")
// The peer keeps sending host updates.
updates := lh.NewRequestHandler()
go loop(func() {
newLHHostUpdate(peerUdp, peer, []netip.AddrPort{peerUdp}, updates)
})
askerV4 := netip.MustParseAddr("10.128.0.2")
askerUdp := netip.MustParseAddrPort("192.0.2.2:4242")
// The same peer keeps querying for the target, which ends in a punch notification.
queries := lh.NewRequestHandler()
go loop(func() {
sendLHHostRequest(peerUdp, peer, target, queries, nil)
})
// Boot: host handshakes with the LH using its v2 cert and sends an update
newLHHostUpdateV2(hostUdp, []netip.Addr{hostV4, hostV6}, []netip.AddrPort{hostLan}, lhh)
time.Sleep(time.Second)
close(stop)
done := make(chan struct{})
go func() { wg.Wait(); close(done) }()
select {
case <-done:
case <-time.After(5 * time.Second):
t.Fatal("deadlock: the lighthouse paths never finished after stop was signalled")
// Both addresses resolve
r := newLHHostRequest(askerUdp, askerV4, hostV4, lhh)
require.NotNil(t, r.msg, "v4 query should be answered")
assertIp4InArray(t, r.msg.Details.V4AddrPorts, hostLan)
r = newLHHostRequest(askerUdp, askerV4, hostV6, lhh)
require.NotNil(t, r.msg, "v6 query should be answered before the v1 handshake")
assertIp4InArray(t, r.msg.Details.V4AddrPorts, hostLan)
// Later: a v1 handshake with the same host completes on the LH. This is exactly what
// handshake_manager.go does on completion, with the v1 cert's single vpn addr.
rl := lh.QueryCache([]netip.Addr{hostV4})
rl.RefreshFromHandshake([]netip.Addr{hostV4}, cert.Version1)
// The host keeps sending v2 updates over its v2 tunnel too
newLHHostUpdateV2(hostUdp, []netip.Addr{hostV4, hostV6}, []netip.AddrPort{hostLan}, lhh)
r = newLHHostRequest(askerUdp, askerV4, hostV4, lhh)
require.NotNil(t, r.msg, "v4 query should still be answered")
assertIp4InArray(t, r.msg.Details.V4AddrPorts, hostLan)
r = newLHHostRequest(askerUdp, askerV4, hostV6, lhh)
if assert.NotNil(t, r.msg, "BUG: v6 query is silently dropped after a v1 handshake truncated RemoteList.vpnAddrs") {
assertIp4InArray(t, r.msg.Details.V4AddrPorts, hostLan)
}
}
func TestLighthouse_reloadRelaysTriggersUpdate(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
base := "lighthouse:\n am_lighthouse: true\nlisten:\n port: 4242\n"
require.NoError(t, c.LoadString(base+"relay:\n relays: [10.128.0.5]\n"))
// Scenario B: the LH first creates a RemoteList for the host keyed only by its v4 addr (a pending
// LH-initiated v1 handshake does QueryCache([v4]) in handleOutbound), then the host arrives with v2.
// unlockedGetRemoteList/QueryCache hit on allAddrs[0] and never add the v6 key to addrMap.
func TestLighthouse_Issue1868_V4OnlyListNeverGainsV6Key(t *testing.T) {
lh, lhh := newIssue1868Lighthouse(t)
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
assert.Empty(t, lh.updateTrigger)
hostV4 := netip.MustParseAddr("10.128.0.3")
hostV6 := netip.MustParseAddr("fd00::3")
hostUdp := netip.MustParseAddrPort("192.0.2.3:4242")
hostLan := netip.MustParseAddrPort("10.0.0.3:4242")
require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6]\n"))
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.6")}, lh.GetRelaysForMe())
require.Len(t, lh.updateTrigger, 1)
<-lh.updateTrigger
askerV4 := netip.MustParseAddr("10.128.0.2")
askerUdp := netip.MustParseAddrPort("192.0.2.2:4242")
// The config changed but what we advertise did not, a bad entry is skipped
require.NoError(t, c.ReloadConfigString(base+"relay:\n relays: [10.128.0.6, nope]\n"))
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.6")}, lh.GetRelaysForMe())
assert.Empty(t, lh.updateTrigger)
}
// LH is a relay and someone asked it to relay to hostV4 while the host was offline:
// StartHandshake(hostV4) -> handleOutbound -> QueryCache([hostV4]) creates a v4-only list.
_ = lh.QueryCache([]netip.Addr{hostV4})
func TestLighthouse_reloadAmRelayAlone(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
base := "lighthouse:\n am_lighthouse: true\nlisten:\n port: 4242\nrelay:\n relays: [10.128.0.5]\n"
require.NoError(t, c.LoadString(base+" am_relay: false\n"))
// Host boots and handshakes v2 with the LH (responder path does QueryCache + RefreshFromHandshake)
rl := lh.QueryCache([]netip.Addr{hostV4, hostV6})
rl.RefreshFromHandshake([]netip.Addr{hostV4, hostV6}, cert.Version2)
newLHHostUpdateV2(hostUdp, []netip.Addr{hostV4, hostV6}, []netip.AddrPort{hostLan}, lhh)
lh, err := NewLightHouseFromConfig(t.Context(), l, c, testCertState(netip.MustParsePrefix("10.128.0.1/24")), nil, nil)
require.NoError(t, err)
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.5")}, lh.GetRelaysForMe())
r := newLHHostRequest(askerUdp, askerV4, hostV4, lhh)
require.NotNil(t, r.msg, "v4 query should be answered")
assertIp4InArray(t, r.msg.Details.V4AddrPorts, hostLan)
// Relays aren't allowed to have relays, flipping am_relay alone must drop them and tell the lighthouses
require.NoError(t, c.ReloadConfigString(base+" am_relay: true\n"))
assert.Empty(t, lh.GetRelaysForMe())
require.Len(t, lh.updateTrigger, 1)
<-lh.updateTrigger
require.NoError(t, c.ReloadConfigString(base+" am_relay: false\n"))
assert.Equal(t, []netip.Addr{netip.MustParseAddr("10.128.0.5")}, lh.GetRelaysForMe())
require.Len(t, lh.updateTrigger, 1)
r = newLHHostRequest(askerUdp, askerV4, hostV6, lhh)
if assert.NotNil(t, r.msg, "BUG: v6 query is silently dropped, addrMap never got the v6 key") {
assertIp4InArray(t, r.msg.Details.V4AddrPorts, hostLan)
}
}
-31
View File
@@ -1,31 +0,0 @@
package logging
import "log/slog"
// RateLimiter is what RateLimited needs from a rate limiter, such as a *ratelimit.Limiter.
// Allow reports whether this call may proceed, and how many calls were denied since the last one that did.
type RateLimiter interface {
Allow() (denied uint64, ok bool)
}
// RateLimited checks lim before anything is built.
// When allowed it returns l, with a "suppressed" count attached if lim denied calls since its last allowed one.
// A nil lim allows everything.
//
// if l, ok := logging.RateLimited(f.l, f.udpTxErrLimiter); ok {
// hostinfo.logger(l).Error("Failed to write outgoing packet", "error", err)
// }
func RateLimited(l *slog.Logger, lim RateLimiter) (*slog.Logger, bool) {
if lim == nil {
return l, true
}
denied, ok := lim.Allow()
if !ok {
return nil, false
}
if denied > 0 {
return l.With("suppressed", denied), true
}
return l, true
}
-106
View File
@@ -1,106 +0,0 @@
package logging
import (
"bytes"
"errors"
"io"
"log/slog"
"net/netip"
"strings"
"testing"
"time"
"github.com/slackhq/nebula/ratelimit"
)
type fakeLimiter struct {
denied uint64
ok bool
}
func (f fakeLimiter) Allow() (uint64, bool) { return f.denied, f.ok }
func TestRateLimited(t *testing.T) {
var buf bytes.Buffer
base := NewLogger(&buf)
if l, ok := RateLimited(base, fakeLimiter{ok: false}); ok || l != nil {
t.Fatalf("denied call: ok=%v logger=%v", ok, l)
}
if l, ok := RateLimited(base, fakeLimiter{ok: true}); !ok || l != base {
t.Fatal("nothing denied should hand back the same logger")
}
if l, ok := RateLimited(base, nil); !ok || l != base {
t.Fatal("a nil limiter should allow with the same logger")
}
l, ok := RateLimited(base, fakeLimiter{denied: 7, ok: true})
if !ok {
t.Fatal("allowed call denied")
}
l.Info("hello", "k", "v")
if !strings.Contains(buf.String(), "suppressed=7") {
t.Fatalf("want suppressed=7, got %q", buf.String())
}
}
// The benchmarks compare a log line through slog directly against the same line behind RateLimited: logged with
// nothing denied, denied, and logged after denials.
var (
benchErr = errors.New("sendto: no buffer space available")
benchAddr = netip.MustParseAddrPort("192.0.2.1:4242")
)
func benchLogger() *slog.Logger {
return NewLogger(io.Discard).With("localIndex", 1234, "remoteIndex", 5678)
}
func BenchmarkRateLimitedBaseline(b *testing.B) {
lg := benchLogger()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
lg.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr)
}
}
func BenchmarkRateLimitedAllowed(b *testing.B) {
lg, lim := benchLogger(), mustLimiter(b, 1, 0)
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if l, ok := RateLimited(lg, lim); ok {
l.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr)
}
}
}
func BenchmarkRateLimitedDenied(b *testing.B) {
lg, lim := benchLogger(), mustLimiter(b, 1, time.Hour)
lim.Allow()
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if l, ok := RateLimited(lg, lim); ok {
l.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr)
}
}
}
func BenchmarkRateLimitedAfterDenied(b *testing.B) {
lg, lim := benchLogger(), fakeLimiter{denied: 5, ok: true}
b.ReportAllocs()
for i := 0; i < b.N; i++ {
if l, ok := RateLimited(lg, lim); ok {
l.Error("Failed to write outgoing packet", "error", benchErr, "udpAddr", benchAddr)
}
}
}
func mustLimiter(b *testing.B, burst int, every time.Duration) *ratelimit.Limiter {
l, err := ratelimit.New(burst, every)
if err != nil {
b.Fatal(err)
}
return l
}
+5 -12
View File
@@ -32,12 +32,9 @@ const _ = noiseutil.RejectHeadroom - 4*maxRoutines
func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) {
ctx, cancel := context.WithCancel(context.Background())
// The goroutines started below stop only when this context does, and only a caller holding the
// Control can arrange that. Cancel whenever we are not handing one back, which covers an error
// and a config test alike: a config test used to leave the lighthouse query worker, and a
// hostname resolver per dns named static host, running for the life of the process.
// Automatically cancel the context if Main returns an error, to signal all created goroutines to quit.
defer func() {
if retcon == nil {
if reterr != nil {
cancel()
}
}()
@@ -232,11 +229,9 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
retries: int64(c.GetInt("handshakes.retries", DefaultHandshakeRetries)),
triggerBuffer: c.GetInt("handshakes.trigger_buffer", DefaultHandshakeTriggerBuffer),
messageMetrics: messageMetrics,
pq: pqHandshakeConfigFrom(c),
}
handshakeManager := NewHandshakeManager(l, hostMap, lightHouse, udpConns[0], handshakeConfig)
c.RegisterReloadCallback(handshakeManager.reloadPQ)
lightHouse.handshakeTrigger = handshakeManager.trigger
ds, err := newDnsServerFromConfig(ctx, l, pki, hostMap, c)
@@ -302,7 +297,6 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
}
var ifce *Interface
var emitStats func()
if !configTest {
ifce, err = NewInterface(ctx, ifConfig)
if err != nil {
@@ -321,12 +315,9 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
go handshakeManager.Run(ctx)
punchy.Start(ctx, ifce, hostMap, lightHouse)
// On the stats server's loop, so the gauges follow the interval it accepted
emitStats = ifce.statsEmitter()
}
stats, err := newStatsServerFromConfig(ctx, l, c, buildVersion, configTest, emitStats)
stats, err := newStatsServerFromConfig(ctx, l, c, buildVersion, configTest)
if err != nil {
return nil, util.ContextualizeIfNeeded("Failed to start stats emitter", err)
}
@@ -335,6 +326,8 @@ func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, dev
return nil, nil
}
go ifce.emitStats(ctx, c.GetDuration("stats.interval", time.Second*10))
attachCommands(l, c, ssh, ifce)
networkChanges := udp.NewNetworkChangeMonitor(ctx, l, c)
-74
View File
@@ -1,21 +1,12 @@
package nebula
import (
"fmt"
"net/netip"
"os"
"path/filepath"
"testing"
"time"
"github.com/slackhq/nebula/cert"
cert_test "github.com/slackhq/nebula/cert_test"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/util"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/goleak"
)
func TestParseCpuAffinity(t *testing.T) {
@@ -58,68 +49,3 @@ func TestParseCpuAffinity(t *testing.T) {
assert.Nil(t, parseCpuAffinity(newConfig([]any{1 << 20}), l, 1))
}
}
// TestMain_ConfigTestReleasesItsGoroutines pins the rule that Main only leaves goroutines running
// when it hands back a Control to stop them with.
//
// A config test gets no Control, so anything it started had nothing to stop it: the lighthouse
// query worker, and a hostname resolver per dns named static host, ran for the life of the
// process. That matters to every embedder that validates a config in process rather than by
// exec'ing, dnclient and the apple clients included, because they do it on each config load and
// the leak accumulates.
func TestMain_ConfigTestReleasesItsGoroutines(t *testing.T) {
defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
l := test.NewLogger()
dir := t.TempDir()
before := time.Now().Add(-time.Hour)
after := time.Now().Add(time.Hour)
ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, before, after, nil, nil, nil)
networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
_, _, keyPEM, certPEM := cert_test.NewTestCert(
cert.Version2, cert.Curve_CURVE25519, ca, caKey, "config-test", before, after, networks, nil, nil)
caPath := filepath.Join(dir, "ca.pem")
certPath := filepath.Join(dir, "cert.pem")
keyPath := filepath.Join(dir, "key.pem")
require.NoError(t, os.WriteFile(caPath, caPEM, 0o600))
require.NoError(t, os.WriteFile(certPath, certPEM, 0o600))
require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600))
// A static host by address, not by name: the query worker is the goroutine under test and a
// hostname would drag a real dns lookup into a unit test.
configBody := fmt.Sprintf(`
pki:
ca: %s
cert: %s
key: %s
static_host_map:
"10.0.0.2": ["192.0.2.1:4242"]
lighthouse:
hosts:
- "10.0.0.2"
listen:
host: 127.0.0.1
port: 0
tun:
disabled: true
firewall:
outbound:
- port: any
proto: any
host: any
inbound:
- port: any
proto: any
host: any
`, caPath, certPath, keyPath)
require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(configBody), 0o600))
c := config.NewC(l)
require.NoError(t, c.Load(dir))
ctrl, err := Main(c, true, "config-test", l, nil)
require.NoError(t, err)
require.Nil(t, ctrl, "a config test hands back nothing to stop, so it must stop itself")
}
-39
View File
@@ -1,39 +0,0 @@
//go:build !android
package nicfinder
import (
"context"
"errors"
"fmt"
"log/slog"
"net"
"net/netip"
)
func localAddrs(ctx context.Context, l *slog.Logger, filter Filter) ([]netip.Addr, error) {
ifaces, err := net.Interfaces()
if err != nil {
return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err)
}
var out []netip.Addr
var errs []error
for _, i := range ifaces {
if !allowName(ctx, l, filter, i.Name) {
continue
}
addrs, err := i.Addrs()
if err != nil {
errs = append(errs, fmt.Errorf("failed to get addresses for %s: %w", i.Name, err))
continue
}
for _, rawAddr := range addrs {
if addr, ok := allowedAddr(ctx, l, filter, rawAddr); ok {
out = append(out, addr)
}
}
}
return out, errors.Join(errs...)
}
-104
View File
@@ -1,104 +0,0 @@
//go:build android
package nicfinder
import (
"context"
"errors"
"fmt"
"log/slog"
"net/netip"
"os"
"github.com/DefinedNet/netlink"
"github.com/DefinedNet/netlink/nl"
"golang.org/x/sys/unix"
)
// Android denies bind() on netlink route sockets and RTM_GETLINK, so net.Interfaces fails with EACCES.
// Enumerate with an RTM_GETADDR dump on an unbound socket instead.
// localInterfaces returns every interface that has an address, with its addresses. It lists
// addresses over a netlink socket that is never bound and names each index with SIOCGIFNAME, the
// same operations Bionic's getifaddrs uses for app UIDs, which the sandbox permits.
func localAddrs(ctx context.Context, l *slog.Logger, filter Filter) ([]netip.Addr, error) {
var errs []error
addrs, err := netlinkAddrList()
if err != nil {
if !errors.Is(err, netlink.ErrDumpInterrupted) {
return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err)
}
// The address table kept changing under the dump and netlink gave up retrying. What it
// returned is still worth advertising.
errs = append(errs, fmt.Errorf("local addresses may be incomplete: %w", err))
}
// Any socket will do for SIOCGIFNAME.
sfd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0)
if err != nil {
return nil, os.NewSyscallError("socket", err)
}
defer unix.Close(sfd)
var out []netip.Addr
nameAllowed := make(map[int]bool) // by interface index, once named
for _, a := range addrs {
allow, named := nameAllowed[a.LinkIndex]
if (named && !allow) || a.IPNet == nil {
continue
}
addr, ok := allowedAddr(ctx, l, filter, a.IPNet)
if !ok {
continue
}
if !named {
name, err := interfaceNameByIndex(sfd, a.LinkIndex)
// ENODEV means the interface went away after the dump. Any other failure is reported
// and the interface skipped, so the rest are still returned.
if err != nil && !errors.Is(err, unix.ENODEV) {
errs = append(errs, fmt.Errorf("failed to resolve name of interface %d: %w", a.LinkIndex, err))
}
allow = err == nil && allowName(ctx, l, filter, name)
nameAllowed[a.LinkIndex] = allow
}
if allow {
out = append(out, addr)
}
}
return out, errors.Join(errs...)
}
// netlinkAddrList is netlink.AddrList over a socket that is never bound. The kernel autobinds it
// on the first send, which the sandbox permits.
func netlinkAddrList() ([]netlink.Addr, error) {
fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE)
if err != nil {
return nil, os.NewSyscallError("socket", err)
}
s, err := nl.NewNetlinkSocketFromFd(fd)
if err != nil {
unix.Close(fd)
return nil, err
}
h := netlink.NewHandleFromSockets(
map[int]*nl.SocketHandle{unix.NETLINK_ROUTE: {Socket: s}},
netlink.HandleOptions{RetryInterrupted: true},
)
defer h.Close()
if err := h.SetSocketTimeout(netlink.GetSocketTimeout()); err != nil {
return nil, err
}
return h.AddrList(nil, netlink.FAMILY_ALL)
}
// interfaceNameByIndex resolves an interface index with SIOCGIFNAME.
func interfaceNameByIndex(fd int, index int) (string, error) {
var ifr unix.Ifreq
ifr.SetUint32(uint32(index)) // ifr_ifindex is in the ifr_ifru union
if err := unix.IoctlIfreq(fd, unix.SIOCGIFNAME, &ifr); err != nil {
return "", err
}
return ifr.Name(), nil
}
-59
View File
@@ -1,59 +0,0 @@
package nicfinder
import (
"context"
"log/slog"
"net"
"net/netip"
"github.com/slackhq/nebula/logging"
)
// Filter decides which interfaces, by name, and which of their addresses Find returns.
type Filter interface {
AllowName(string) bool
Allow(netip.Addr) bool
}
// Find returns the addresses of this host's network interfaces that allowList accepts, with
// loopback and link-local addresses left out. When only some interfaces could be read it returns
// their addresses alongside an error for the rest. allowList must not be nil; it is called as
// given, so a nil pointer of a type whose methods accept a nil receiver is fine.
func Find(ctx context.Context, l *slog.Logger, allowList Filter) ([]netip.Addr, error) {
return localAddrs(ctx, l, allowList)
}
// allowName asks filter about an interface name and traces the answer.
func allowName(ctx context.Context, l *slog.Logger, filter Filter, name string) bool {
allow := filter.AllowName(name)
if l.Enabled(ctx, logging.LevelTrace) {
l.Log(ctx, logging.LevelTrace, "localAllowList.AllowName", "interfaceName", name, "allow", allow)
}
return allow
}
// allowedAddr converts one address a platform reports and says whether to advertise it: it must
// be a *net.IPNet or *net.IPAddr carrying an IP, not be loopback or link-local, and be accepted by
// filter, whose answer is traced. IPv4 comes back unmapped.
func allowedAddr(ctx context.Context, l *slog.Logger, filter Filter, rawAddr net.Addr) (netip.Addr, bool) {
var ip net.IP
switch v := rawAddr.(type) {
case *net.IPNet:
ip = v.IP
case *net.IPAddr:
ip = v.IP
}
addr, ok := netip.AddrFromSlice(ip)
if !ok {
return netip.Addr{}, false
}
addr = addr.Unmap()
if addr.IsLoopback() || addr.IsLinkLocalUnicast() {
return netip.Addr{}, false
}
isAllowed := filter.Allow(addr)
if l.Enabled(ctx, logging.LevelTrace) {
l.Log(ctx, logging.LevelTrace, "localAllowList.Allow", "localAddr", addr, "allowed", isAllowed)
}
return addr, isAllowed
}
-117
View File
@@ -1,117 +0,0 @@
package nicfinder
import (
"context"
"net"
"net/netip"
"regexp"
"testing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
)
type allowAll struct{}
func (allowAll) Allow(netip.Addr) bool { return true }
func (allowAll) AllowName(string) bool { return true }
type testAllowList struct {
bannedName *regexp.Regexp
bannedAddr netip.Addr
}
func (l *testAllowList) Allow(addr netip.Addr) bool {
return addr != l.bannedAddr
}
func (l *testAllowList) AllowName(n string) bool {
return !l.bannedName.MatchString(n)
}
func newTestAllowList(bannedName string) *testAllowList {
return &testAllowList{
bannedName: regexp.MustCompile(bannedName),
}
}
func TestAllowedAddr(t *testing.T) {
ctx, l := context.Background(), test.NewLogger()
allowed := func(filter Filter, addrs []net.Addr) []netip.Addr {
var out []netip.Addr
for _, rawAddr := range addrs {
if addr, ok := allowedAddr(ctx, l, filter, rawAddr); ok {
out = append(out, addr)
}
}
return out
}
addrs := []net.Addr{
&net.IPNet{IP: net.ParseIP("127.0.0.1"), Mask: net.CIDRMask(8, 32)},
&net.IPNet{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)},
&net.IPNet{IP: net.ParseIP("10.0.0.5"), Mask: net.CIDRMask(24, 32)},
&net.IPNet{IP: net.ParseIP("169.254.10.10"), Mask: net.CIDRMask(16, 32)},
&net.IPNet{IP: net.ParseIP("fe80::1"), Mask: net.CIDRMask(64, 128)},
&net.IPAddr{IP: net.ParseIP("fd00::5")},
&net.IPNet{},
otherAddr{},
nil,
}
// Loopback, link local and anything without an IP are dropped, and IPv4 comes back unmapped
// even though net.ParseIP hands over a 16 byte slice.
assert.Equal(t, []netip.Addr{
netip.MustParseAddr("10.0.0.5"),
netip.MustParseAddr("fd00::5"),
}, allowed(allowAll{}, addrs))
// The filter sees the unmapped form and can drop a single address.
al := newTestAllowList("none")
al.bannedAddr = netip.MustParseAddr("10.0.0.5")
assert.Equal(t, []netip.Addr{netip.MustParseAddr("fd00::5")}, allowed(al, addrs))
}
// otherAddr is a net.Addr that is neither *net.IPNet nor *net.IPAddr. netlink.Addr is one such
// type: it satisfies net.Addr through an embedded *net.IPNet, and passing it directly once
// silently produced no addresses.
type otherAddr struct{}
func (otherAddr) Network() string { return "other" }
func (otherAddr) String() string { return "other" }
// hostFilter counts AllowName calls and can reject every name or every address.
type hostFilter struct {
rejectNames bool
rejectAddrs bool
asked map[string]int
}
func (f *hostFilter) Allow(netip.Addr) bool { return !f.rejectAddrs }
func (f *hostFilter) AllowName(name string) bool {
f.asked[name]++
return !f.rejectNames
}
// Runs this platform's localAddrs against the host and checks it honors the whole filter.
func TestLocalAddrsAppliesFilter(t *testing.T) {
ctx, l := context.Background(), test.NewLogger()
f := &hostFilter{asked: map[string]int{}}
all, err := localAddrs(ctx, l, f)
if len(f.asked) == 0 {
t.Skipf("no interfaces to enumerate here: %v", err)
}
for name, n := range f.asked {
assert.Equal(t, 1, n, "AllowName asked %d times for %s", n, name)
}
// Loopback and link-local never come back, whatever the filter says.
for _, addr := range all {
assert.False(t, addr.IsLoopback() || addr.IsLinkLocalUnicast(), "%s", addr)
}
none, _ := localAddrs(ctx, l, &hostFilter{rejectNames: true, asked: map[string]int{}})
assert.Empty(t, none, "addresses got past a filter that rejects every interface name")
none, _ = localAddrs(ctx, l, &hostFilter{rejectAddrs: true, asked: map[string]int{}})
assert.Empty(t, none, "addresses got past a filter that rejects every address")
}
+1
View File
@@ -1,4 +1,5 @@
//go:build boringcrypto
// +build boringcrypto
package noiseutil
+1
View File
@@ -1,4 +1,5 @@
//go:build boringcrypto
// +build boringcrypto
package noiseutil
+8 -11
View File
@@ -160,7 +160,9 @@ func (f *Interface) readOutsidePackets(via ViaSender, packet []byte, rxc *rxCont
case header.TestReply:
// No-op, useful for the Roaming and connectionManager side-effects above
case header.TestRequest:
if header.MaxOverhead+len(out) > len(rxc.scratch) {
const maxCipherOverhead = 16 //todo we use this too often, needs a real importable const
const maxOverhead = header.Len + header.Len + maxCipherOverhead + maxCipherOverhead
if maxOverhead+len(out) > len(rxc.scratch) {
// A reply that cannot fit in scratch is dropped no matter the log level.
if f.l.Enabled(context.Background(), slog.LevelDebug) {
hostinfo.logger(f.l).Debug("dropping oversized test request", "payloadLen", len(out), "from", via)
@@ -261,16 +263,11 @@ func (f *Interface) handleOutsideRelayPacket(hostinfo *HostInfo, via ViaSender,
// closeTunnel closes a tunnel locally, it does not send a closeTunnel packet to the remote
func (f *Interface) closeTunnel(hostInfo *HostInfo) {
// Hold the hostmap lock across both the delete and the lighthouse eviction,
// to avoid a race with handshake completion.
f.hostMap.Lock()
defer f.hostMap.Unlock()
if !f.hostMap.unlockedDeleteHostInfo(hostInfo) {
return
final := f.hostMap.DeleteHostInfo(hostInfo)
if final {
// We no longer have any tunnels with this vpn addr, clear learned lighthouse state to lower memory usage
f.lightHouse.DeleteVpnAddrs(hostInfo.vpnAddrs)
}
// We no longer have any tunnels with this vpn addr, clear learned lighthouse state to lower memory usage
f.lightHouse.DeleteVpnAddrs(hostInfo.vpnAddrs)
}
// sendCloseTunnel is a helper function to send a proper close tunnel packet to a remote
@@ -357,7 +354,7 @@ func parseV6(data []byte, incoming bool, fp *firewall.ParsedPacket) error {
// and cannot drift into misreading an unknown protocol (SCTP, GRE, etc.) as a forged transport.
proto, offset, isFragment, anyFragment, err := iputil.IPv6FindUpperProtocol(data)
if err != nil {
return err
return ErrIPv6PacketTooShort
}
fp.Protocol = proto
+7 -7
View File
@@ -117,7 +117,7 @@ func Test_newPacket_v6(t *testing.T) {
require.NoError(t, err)
err = newPacket(buffer.Bytes(), true, p)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
// A v6 packet with a hop-by-hop extension
// ICMPv6 Payload (Echo Request)
@@ -151,12 +151,12 @@ func Test_newPacket_v6(t *testing.T) {
// A full IPv6 header and 1 byte in the first extension, but missing
// the length byte.
err = newPacket(buffer.Bytes()[:41], true, p)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
// A full IPv6 header plus 1 full extension, but only 1 byte of the
// next layer, missing length byte
err = newPacket(buffer.Bytes()[:49], true, p)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
err = nil
// A good ICMP packet
@@ -354,12 +354,12 @@ func Test_newPacket_v6(t *testing.T) {
// Ensure buffer bounds checking during processing, a truncated AH header can't reach the payload
err = newPacket(b[:41], true, p)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
// Invalid AH header
b = buffer.Bytes()
err = newPacket(b, true, p)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
}
func Test_newPacket_ipv6Fragment(t *testing.T) {
@@ -466,7 +466,7 @@ func Test_newPacket_ipv6Fragment(t *testing.T) {
// Too short of a fragment packet
err = newPacket(secondFrag[:len(secondFrag)-10], false, p)
require.ErrorIs(t, err, iputil.ErrIPv6CouldNotFindPayload)
require.ErrorIs(t, err, ErrIPv6PacketTooShort)
}
func BenchmarkParseV6(b *testing.B) {
@@ -707,7 +707,7 @@ func Test_newPacket_v6ExtHeaderPastBuffer(t *testing.T) {
pkt[40] = byte(layers.IPProtocolSCTP) // Dest Options next header = SCTP
pkt[41] = 255 // declared length (255+1)*8 = 2048, past the 48 byte buffer
require.ErrorIs(t, newPacket(pkt, true, p), iputil.ErrIPv6CouldNotFindPayload)
require.ErrorIs(t, newPacket(pkt, true, p), ErrIPv6PacketTooShort)
}
// Test_newPacket_v6ExtHeaderConfusion is a regression test for parseV6 walking any unrecognized
+2 -2
View File
@@ -127,7 +127,7 @@ func TestPseudoSumIPv6MatchesReference(t *testing.T) {
func TestIPv4HdrChecksumMatchesReference(t *testing.T) {
rng := rand.New(rand.NewSource(0x1791))
for _, hdrLen := range []int{20, 24, 40, 60} {
for trial := range 200 {
for trial := 0; trial < 200; trial++ {
hdr := make([]byte, hdrLen)
rng.Read(hdr)
hdr[0] = 0x40 | byte(hdrLen/4)
@@ -156,7 +156,7 @@ func TestIPv4HdrChecksumMatchesReference(t *testing.T) {
// the way a receiver does (pseudo-header + L4 must sum to all-ones).
func TestChecksumSeedReceiverAcceptance(t *testing.T) {
rng := rand.New(rand.NewSource(0x1826))
for trial := range 200 {
for trial := 0; trial < 200; trial++ {
src := [4]byte{byte(rng.Intn(256)), byte(rng.Intn(256)), byte(rng.Intn(256)), byte(rng.Intn(256))}
dst := [4]byte{byte(rng.Intn(256)), byte(rng.Intn(256)), byte(rng.Intn(256)), byte(rng.Intn(256))}
payLen := rng.Intn(1500)
+1 -1
View File
@@ -540,7 +540,7 @@ func TestCoalescerCapBySegments(t *testing.T) {
c := newTestTCPCoalescer(t, w)
pay := make([]byte, 512)
seq := uint32(1000)
for range tcpCoalesceMaxSegs + 5 {
for i := 0; i < tcpCoalesceMaxSegs+5; i++ {
if err := c.Commit(buildTCPv4(seq, tcpAck, pay)); err != nil {
t.Fatal(err)
}
+2 -2
View File
@@ -28,7 +28,7 @@ func TestSendBatchReserveCommitFlush(t *testing.T) {
b := NewSendBatch(fw, 4, 32)
ap := netip.MustParseAddrPort("10.0.0.1:4242")
for i := range 4 {
for i := 0; i < 4; i++ {
slot := b.Reserve(32)
if cap(slot) != 32 {
t.Fatalf("slot %d: cap=%d want 32", i, cap(slot))
@@ -72,7 +72,7 @@ func TestSendBatchSlotsDoNotOverlap(t *testing.T) {
b := NewSendBatch(fw, 3, 8)
ap := netip.MustParseAddrPort("10.0.0.1:80")
for i := range 3 {
for i := 0; i < 3; i++ {
s := b.Reserve(8)
pkt := append(s[:0], byte(0xA0+i), byte(0xB0+i))
b.Commit(pkt, ap)
+3 -3
View File
@@ -129,7 +129,7 @@ func TestUDPCoalescerCoalescesEqualSized(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true}
c := newTestUDPCoalescer(t, w)
pay := make([]byte, 1200)
for range 3 {
for i := 0; i < 3; i++ {
if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
t.Fatal(err)
}
@@ -259,7 +259,7 @@ func TestUDPCoalescerCapsAtMaxSegs(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true}
c := newTestUDPCoalescer(t, w)
pay := make([]byte, 100)
for range udpCoalesceMaxSegs + 5 {
for i := 0; i < udpCoalesceMaxSegs+5; i++ {
if err := c.Commit(buildUDPv4(1000, 53, pay)); err != nil {
t.Fatal(err)
}
@@ -317,7 +317,7 @@ func TestUDPCoalescerIPv6Coalesces(t *testing.T) {
w := &fakeTunWriter{gsoEnabled: true}
c := newTestUDPCoalescer(t, w)
pay := make([]byte, 1200)
for range 3 {
for i := 0; i < 3; i++ {
if err := c.Commit(buildUDPv6(1000, 53, pay)); err != nil {
t.Fatal(err)
}
+1 -1
View File
@@ -151,7 +151,7 @@ func TestChecksumTailPaths(t *testing.T) {
offsets := []int{0, 1, 3, 7, 15} // mix of aligned and odd starts
for k := 0; k <= maxK; k++ {
for tail := range 64 {
for tail := 0; tail < 64; tail++ {
length := 64*k + tail
for _, seed := range seeds {
for _, off := range offsets {
-57
View File
@@ -1,57 +0,0 @@
//go:build !e2e_testing
package overlay
import (
"fmt"
"log/slog"
"golang.org/x/sys/windows"
"golang.org/x/sys/windows/registry"
)
var tcpipInterfaceKeys = []string{
`SYSTEM\CurrentControlSet\Services\Tcpip\Parameters\Interfaces`,
`SYSTEM\CurrentControlSet\Services\Tcpip6\Parameters\Interfaces`,
}
// disableDNSRegistration prevents publishing overlay addresses in the host's
// DNS records, where clients outside the overlay cannot reach them.
// Failures are non-fatal so registry access cannot prevent tunnel creation.
func disableDNSRegistration(l *slog.Logger, guid windows.GUID) {
id := guid.String()
for _, base := range tcpipInterfaceKeys {
path := base + `\` + id
if err := setDNSRegistrationValues(path); err != nil {
l.Warn("Failed to disable DNS registration on the tun adapter", "error", err, "key", path)
continue
}
l.Debug("Disabled DNS registration on the tun adapter", "key", path)
}
}
func setDNSRegistrationValues(path string) error {
// The stack may not have created the adapter's interface key yet.
k, _, err := registry.CreateKey(registry.LOCAL_MACHINE, path, registry.SET_VALUE)
if err != nil {
return fmt.Errorf("open HKLM\\%s: %w", path, err)
}
defer k.Close()
// Neither RegistrationEnabled nor DisableDynamicUpdate alone reliably
// prevents registration, so all three settings are required.
values := []struct {
name string
data uint32
}{
{"RegistrationEnabled", 0},
{"DisableDynamicUpdate", 1},
{"MaxNumberOfAddressesToRegister", 0},
}
for _, v := range values {
if err := k.SetDWordValue(v.name, v.data); err != nil {
return fmt.Errorf("set %s: %w", v.name, err)
}
}
return nil
}
+1
View File
@@ -1,4 +1,5 @@
//go:build !e2e_testing
// +build !e2e_testing
package overlay
+1
View File
@@ -1,4 +1,5 @@
//go:build !e2e_testing
// +build !e2e_testing
package overlay
+4 -28
View File
@@ -50,13 +50,7 @@ func (r Route) String() string {
func makeRouteTree(l *slog.Logger, routes []Route, allowMTU bool) (*bart.Table[routing.Gateways], error) {
routeTree := new(bart.Table[routing.Gateways])
seen := make(map[netip.Prefix]bool, len(routes))
for _, r := range routes {
if seen[r.Cidr] {
l.Warn("route is listed more than once, only the last entry's via is used", "route", r)
}
seen[r.Cidr] = true
if !allowMTU && r.MTU > 0 {
l.Warn("route MTU is not supported on this platform",
"goos", runtime.GOOS,
@@ -104,11 +98,7 @@ func parseRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
mtu, ok := rMtu.(int)
if !ok {
s, ok := rMtu.(string)
if !ok {
return nil, fmt.Errorf("entry %v.mtu in tun.routes is not an integer: %v", i+1, rMtu)
}
mtu, err = strconv.Atoi(s)
mtu, err = strconv.Atoi(rMtu.(string))
if err != nil {
return nil, fmt.Errorf("entry %v.mtu in tun.routes is not an integer: %v", i+1, err)
}
@@ -184,11 +174,7 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
if rMtu, ok := m["mtu"]; ok {
mtu, ok = rMtu.(int)
if !ok {
s, ok := rMtu.(string)
if !ok {
return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, rMtu)
}
mtu, err = strconv.Atoi(s)
mtu, err = strconv.Atoi(rMtu.(string))
if err != nil {
return nil, fmt.Errorf("entry %v.mtu in tun.unsafe_routes is not an integer: %v", i+1, err)
}
@@ -206,15 +192,10 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
metric, ok := rMetric.(int)
if !ok {
s, ok := rMetric.(string)
if !ok {
return nil, fmt.Errorf("entry %v.metric in tun.unsafe_routes is not an integer: %v", i+1, rMetric)
}
n, err := strconv.ParseInt(s, 10, 32)
_, err = strconv.ParseInt(rMetric.(string), 10, 32)
if err != nil {
return nil, fmt.Errorf("entry %v.metric in tun.unsafe_routes is not an integer: %v", i+1, err)
}
metric = int(n)
}
if metric < 0 || metric > math.MaxInt32 {
@@ -267,15 +248,10 @@ func parseUnsafeRoutes(c *config.C, networks []netip.Prefix) ([]Route, error) {
gatewayWeight, ok := rGatewayWeight.(int)
if !ok {
s, ok := rGatewayWeight.(string)
if !ok {
return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1)
}
w, err := strconv.ParseInt(s, 10, 32)
_, err = strconv.ParseInt(rGatewayWeight.(string), 10, 32)
if err != nil {
return nil, fmt.Errorf("entry .weight in tun.unsafe_routes[%v].via[%v] is not an integer", i+1, ig+1)
}
gatewayWeight = int(w)
}
if gatewayWeight < 1 || gatewayWeight > math.MaxInt32 {
-116
View File
@@ -1,17 +1,13 @@
package overlay
import (
"bytes"
"fmt"
"net/netip"
"strings"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/header"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test"
"github.com/slackhq/nebula/udp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@@ -406,115 +402,3 @@ func Test_makeMultipathUnsafeRouteTree(t *testing.T) {
routing.CalculateBucketsForGateways(expectedGateways)
assert.ElementsMatch(t, expectedGateways, r)
}
func Test_getMTU(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
// unset
assert.Equal(t, DefaultMTU, getMTU(l, c))
// the largest a relayed packet can carry in one underlay buffer
require.Equal(t, udp.MTU, MaxMTU+header.MaxOverhead)
c.Settings["tun"] = map[string]any{"mtu": MaxMTU}
assert.Equal(t, MaxMTU, getMTU(l, c))
// too big is capped
c.Settings["tun"] = map[string]any{"mtu": 9000}
assert.Equal(t, MaxMTU, getMTU(l, c))
}
func Test_getAllRoutesFromConfigCapsMTU(t *testing.T) {
l := test.NewLogger()
c := config.NewC(l)
c.Settings["tun"] = map[string]any{
"routes": []any{
map[string]any{"mtu": "9000", "route": "10.0.0.0/29"},
map[string]any{"mtu": "1300", "route": "10.0.0.8/29"},
},
"unsafe_routes": []any{
map[string]any{"via": "127.0.0.1", "mtu": "9000", "route": "1.0.0.0/29"},
},
}
changed, routes, err := getAllRoutesFromConfig(l, c, []netip.Prefix{netip.MustParsePrefix("10.0.0.0/24")}, true)
require.NoError(t, err)
assert.True(t, changed)
mtus := map[string]int{}
for _, r := range routes {
mtus[r.Cidr.String()] = r.MTU
}
assert.Equal(t, map[string]int{"10.0.0.0/29": MaxMTU, "10.0.0.8/29": 1300, "1.0.0.0/29": MaxMTU}, mtus)
}
// The route tree keeps one entry per CIDR, so a CIDR listed twice only routes by the last one. Say so instead of
// letting the earlier via, or on Windows the earlier metric, quietly lose
func Test_makeRouteTree_WarnsOnDuplicate(t *testing.T) {
buf := &bytes.Buffer{}
l := test.NewLoggerWithOutput(buf)
via := func(s string) routing.Gateways {
return routing.Gateways{routing.NewGateway(netip.MustParseAddr(s), 1)}
}
cidr := netip.MustParsePrefix("10.251.1.0/24")
_, err := makeRouteTree(l, []Route{
{Cidr: cidr, Via: via("10.250.0.2"), Install: true, Metric: 100},
{Cidr: netip.MustParsePrefix("10.251.2.0/24"), Via: via("10.250.0.2"), Install: true},
}, false)
require.NoError(t, err)
assert.Empty(t, buf.String())
tree, err := makeRouteTree(l, []Route{
{Cidr: cidr, Via: via("10.250.0.2"), Install: true, Metric: 100},
{Cidr: cidr, Via: via("10.250.0.3"), Install: true, Metric: 200},
}, false)
require.NoError(t, err)
assert.Equal(t, 1, strings.Count(buf.String(), "route is listed more than once, only the last entry's via is used"))
gw, ok := tree.Lookup(netip.MustParseAddr("10.251.1.1"))
require.True(t, ok)
assert.Equal(t, netip.MustParseAddr("10.250.0.3"), gw[0].Addr(), "the last entry wins")
}
// A quoted metric or weight is a number written as a string, it means that number. Anything else is an error, not a
// panic
func Test_parseUnsafeRoutes_QuotedNumbers(t *testing.T) {
c := config.NewC(test.NewLogger())
n := netip.MustParsePrefix("10.0.0.1/24")
route := func(metric, weight any) {
c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{
"route": "1.0.0.0/8",
"metric": metric,
"via": []any{map[string]any{"gateway": "10.0.0.2", "weight": weight}},
}}}
}
route("100", "5")
routes, err := parseUnsafeRoutes(c, []netip.Prefix{n})
require.NoError(t, err)
require.Len(t, routes, 1)
assert.Equal(t, 100, routes[0].Metric)
assert.Equal(t, "{addr: 10.0.0.2, weight: 5}", routes[0].Via[0].String())
route(1.5, 1)
_, err = parseUnsafeRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry 1.metric in tun.unsafe_routes is not an integer: 1.5")
route(1, true)
_, err = parseUnsafeRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry .weight in tun.unsafe_routes[1].via[1] is not an integer")
}
// An mtu that is neither a number nor a quoted one is an error, not a panic
func Test_parseRoutes_MTUNotANumber(t *testing.T) {
c := config.NewC(test.NewLogger())
n := netip.MustParsePrefix("10.0.0.1/24")
c.Settings["tun"] = map[string]any{"routes": []any{map[string]any{"route": "1.0.0.0/8", "mtu": 1500.5}}}
_, err := parseRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry 1.mtu in tun.routes is not an integer: 1500.5")
c.Settings["tun"] = map[string]any{"unsafe_routes": []any{map[string]any{"route": "1.0.0.0/8", "via": "10.0.0.2", "mtu": 1500.5}}}
_, err = parseUnsafeRoutes(c, []netip.Prefix{n})
require.EqualError(t, err, "entry 1.mtu in tun.unsafe_routes is not an integer: 1500.5")
}
+1
View File
@@ -1,4 +1,5 @@
//go:build linux && !android
// +build linux,!android
package tio
+1
View File
@@ -1,4 +1,5 @@
//go:build linux && !android
// +build linux,!android
package tio
+1
View File
@@ -1,4 +1,5 @@
//go:build linux && !android
// +build linux,!android
package tio
+1
View File
@@ -1,4 +1,5 @@
//go:build linux && !android
// +build linux,!android
package tio
+1
View File
@@ -1,4 +1,5 @@
//go:build linux && !android
// +build linux,!android
package tio
+7 -4
View File
@@ -1,4 +1,5 @@
//go:build linux && !android && !e2e_testing
// +build linux,!android,!e2e_testing
package tio
@@ -116,15 +117,17 @@ func TestPoll_ConcurrentWrite_NoRace(t *testing.T) {
}()
var wg sync.WaitGroup
for range writers {
wg.Go(func() {
for range perWriter {
for w := 0; w < writers; w++ {
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < perWriter; i++ {
if _, werr := p.Write(payload); werr != nil {
t.Errorf("write: %v", werr)
return
}
}
})
}()
}
wg.Wait()
+1
View File
@@ -1,4 +1,5 @@
//go:build linux && !android
// +build linux,!android
package tio
+6 -5
View File
@@ -1,4 +1,5 @@
//go:build linux && !android && !e2e_testing
// +build linux,!android,!e2e_testing
package tio
@@ -136,7 +137,7 @@ func buildTSOv4(t *testing.T, payLen, mss int) ([]byte, virtio.Hdr) {
binary.BigEndian.PutUint16(pkt[34:36], 65535) // window
// payload
for i := range payLen {
for i := 0; i < payLen; i++ {
pkt[ipLen+tcpLen+i] = byte(i & 0xff)
}
return pkt, virtio.NewHeader(
@@ -256,7 +257,7 @@ func TestSegmentTCPv6(t *testing.T) {
pkt[53] = 0x19 // FIN | ACK | PSH — exercise FIN clearing too
binary.BigEndian.PutUint16(pkt[54:56], 65535)
for i := range payLen {
for i := 0; i < payLen; i++ {
pkt[ipLen+tcpLen+i] = byte(i)
}
@@ -360,7 +361,7 @@ func buildUSOv4(t *testing.T, payLen, gsoSize int) ([]byte, virtio.Hdr) {
binary.BigEndian.PutUint16(pkt[22:24], 53) // dport
binary.BigEndian.PutUint16(pkt[24:26], uint16(udpLen+payLen)) // superpacket length
for i := range payLen {
for i := 0; i < payLen; i++ {
pkt[ipLen+udpLen+i] = byte(i & 0xff)
}
@@ -472,7 +473,7 @@ func TestSegmentUDPv6(t *testing.T) {
// Superpacket-wide length, as the kernel supplies it; see buildUSOv4.
binary.BigEndian.PutUint16(pkt[44:46], uint16(udpLen+payLen))
for i := range payLen {
for i := 0; i < payLen; i++ {
pkt[ipLen+udpLen+i] = byte(i)
}
@@ -771,7 +772,7 @@ func buildTSOv6(payLen, gso int) []byte {
pkt[53] = 0x10 // ACK only
binary.BigEndian.PutUint16(pkt[54:56], 65535)
for i := range payLen {
for i := 0; i < payLen; i++ {
pkt[ipLen+tcpLen+i] = byte(i)
}
return pkt
+1
View File
@@ -1,4 +1,5 @@
//go:build linux && !android
// +build linux,!android
package virtio
+11 -4
View File
@@ -1,4 +1,5 @@
//go:build linux && !android
// +build linux,!android
// Package virtio implements the pure validation, header-correction, and
// per-segment slicing logic for kernel-supplied TSO/USO superpackets on
@@ -253,9 +254,12 @@ func SegmentTCP(pkt []byte, hdrLenU, csumStartU, gsoSizeU uint16, yield func(seg
var savedHdr [maxSegHdrLen]byte
copy(savedHdr[:headerLen], pkt[:headerLen])
for i := range numSeg {
for i := 0; i < numSeg; i++ {
segStart := i * gsoSize
segEnd := min(segStart+gsoSize, payLen)
segEnd := segStart + gsoSize
if segEnd > payLen {
segEnd = payLen
}
segPayLen := segEnd - segStart
segLen := headerLen + segPayLen
headerOff := i * gsoSize
@@ -355,9 +359,12 @@ func SegmentUDP(pkt []byte, hdrLenU, csumStartU, gsoSizeU uint16, yield func(seg
var savedHdr [maxSegHdrLen]byte
copy(savedHdr[:headerLen], pkt[:headerLen])
for i := range numSeg {
for i := 0; i < numSeg; i++ {
segStart := i * gsoSize
segEnd := min(segStart+gsoSize, payLen)
segEnd := segStart + gsoSize
if segEnd > payLen {
segEnd = payLen
}
segPayLen := segEnd - segStart
segLen := headerLen + segPayLen
headerOff := i * gsoSize
Loaded 100 of 144 files, more files were not shown because too many files have changed in this diff. Show more