Compare commits

..
21 changed files with 513 additions and 735 deletions

No files matched your search

+3 -3
View File
@@ -14,7 +14,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -40,7 +40,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -83,7 +83,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
# GitHub holds ARNs, not credentials, and ARNs outlive a rotation # GitHub holds ARNs, not credentials, and ARNs outlive a rotation
+3 -3
View File
@@ -34,7 +34,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: add hashicorp source - name: add hashicorp source
@@ -66,7 +66,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: add hashicorp source - name: add hashicorp source
@@ -92,7 +92,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
# WSL2 + Ubuntu so the smoke can run a real linux peer with its own # WSL2 + Ubuntu so the smoke can run a real linux peer with its own
+2 -2
View File
@@ -22,7 +22,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Smoke Docker - name: Smoke Docker
@@ -51,7 +51,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: build - name: build
+4 -4
View File
@@ -22,7 +22,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Install goimports - name: Install goimports
@@ -42,7 +42,7 @@ jobs:
- name: golangci-lint - name: golangci-lint
uses: golangci/golangci-lint-action@v9 uses: golangci/golangci-lint-action@v9
with: with:
version: v2.14 version: v2.12
test: test:
name: Test ${{ matrix.name }} name: Test ${{ matrix.name }}
@@ -87,7 +87,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -132,7 +132,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.27' go-version: '1.26'
check-latest: true check-latest: true
- name: Build ${{ matrix.name }} - name: Build ${{ matrix.name }}
+68
View File
@@ -0,0 +1,68 @@
# vpnemu
Reproduces, on macOS and without Tailscale, the way a VPN with an exit node takes nebula's vpn network route
away from its utun (#1931).
A macOS network extension never touches the routing table. Its routes are published in configd's dynamic store
under `State:/Network/Service/<id>/IPv4` and `/IPv6`, and configd's IPMonitor installs them. vpnemu creates a
utun and publishes those keys itself, so configd does exactly what it would do for the real app. It publishes
the routes Tailscale would with an exit node: every interface prefix that isn't a single IP, the rule in
Tailscale's `ipnlocal.internalAndExternalInterfacesFrom`.
| mode | Tailscale setting | published as | where nebula's prefix goes |
| --- | --- | --- | --- |
| `claim` | exit node, no LAN access | `AdditionalRoutes` | into vpnemu's utun |
| `exclude` | exit node, LAN access | `ExcludedRoutes`, service overrides primary | out the next interface (en0) |
configd answers `EEXIST` with a delete and an add, so whatever held the prefix loses it. When the service is
withdrawn, configd deletes its routes by prefix alone, whoever holds them by then.
## Build
It is its own module, so `purego` stays out of nebula's `go.mod`. purego reaches CoreFoundation and
SystemConfiguration without cgo, so it cross compiles and runs on a mac with no developer tools.
```sh
cd e2e/darwin/vpnemu
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o vpnemu .
# a cross compiled binary copied to a mac may need: codesign -s - -f vpnemu
```
## Run
```sh
sudo ./vpnemu -mode claim -keep en0 # runs until ctrl-c, then withdraws and closes its utun
sudo ./vpnemu -clear # withdraw a service a crashed run left behind
```
`-keep` names interfaces whose prefixes are never published. Keep the one your ssh session arrives on, or
`claim` takes your LAN into a utun nobody reads. `exclude` makes vpnemu primary, so anything off a kept
interface's subnet, internet included, goes into its utun and is dropped while it runs.
`-follow` (default on) republishes when interface prefixes change, as Tailscale does on link changes. With
`-follow=false` a route stays published after its interface is gone, which stages the route configd leaves
behind for a nebula that restarts before the VPN has caught up.
## A/B a nebula build
`repro.sh` starts nebula, then runs vpnemu, and reports egress and ping per peer before, during and after:
```sh
sudo ./repro.sh -n ./nebula -c config.yml -p 192.168.212.1 -p fd19:31::1 -m claim -k en0
```
Before #1931 (utun holds `/24` and `/64`):
```
during 192.168.212.1 egress=utun2 ping=FAIL
after 192.168.212.1 egress=en0 ping=FAIL
```
With #1931 (utun holds `/32` and `/128`, network routes added explicitly) every phase stays on nebula's utun.
### Leftover route
Run an old nebula, start `vpnemu -follow=false`, then restart onto a new nebula while vpnemu is still up. The
new nebula finds the vpn network routed to vpnemu's utun, replaces it and starts. Stop vpnemu and configd deletes
the prefix, taking nebula's route with it until nebula restarts. With real Tailscale this needs nebula to restart
before Tailscale notices the old utun is gone, a one time window when upgrading from a nebula without #1931.
+8
View File
@@ -0,0 +1,8 @@
module github.com/slackhq/nebula/e2e/darwin/vpnemu
go 1.26.0
require (
github.com/ebitengine/purego v0.9.1
golang.org/x/sys v0.48.0
)
+4
View File
@@ -0,0 +1,4 @@
github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+292
View File
@@ -0,0 +1,292 @@
//go:build darwin
// vpnemu stands in for a macOS VPN app with an exit node in use, so route hijacks like the one
// Tailscale causes can be reproduced without Tailscale.
//
// A network extension does not touch the routing table itself. It hands its routes to
// nesessionmanager, which publishes them in configd's dynamic store under
// State:/Network/Service/<id>/IPv4 and /IPv6, and configd's IPMonitor installs them. vpnemu
// creates a utun and publishes the same keys directly, so configd does exactly what it would do
// for the real thing.
//
// Which routes it publishes follows Tailscale's rule for exit nodes
// (ipnlocal.internalAndExternalInterfacesFrom): every interface address prefix that is not a
// single IP counts as an attached LAN. With LAN access off those prefixes are claimed into the
// tunnel (-mode claim, published as AdditionalRoutes). With LAN access on they are excluded and
// leave through the next interface (-mode exclude, published as ExcludedRoutes on a service that
// overrides the primary).
//
// It runs until interrupted, then withdraws the service and closes its utun. Needs root.
package main
import (
"flag"
"fmt"
"log"
"net"
"net/netip"
"os"
"os/exec"
"os/signal"
"slices"
"strings"
"syscall"
"time"
"github.com/ebitengine/purego"
"golang.org/x/sys/unix"
)
const (
serviceID = "VPNEMU"
// Tailscale's own addresses, single IPs, so the rule skips them like Tailscale skips its own
addr4 = "100.100.0.1"
addr6 = "fd7a:115c:a1e0::1"
utunControlName = "com.apple.net.utun_control"
utunOptIfname = 2
cfStringUTF8 = 0x08000100
)
var (
cfStringCreateWithCString func(alloc uintptr, s *byte, enc uint32) uintptr
cfDataCreate func(alloc uintptr, b *byte, n int) uintptr
cfPropertyListCreateWithData func(alloc, data uintptr, opts uint, format *int, err *uintptr) uintptr
cfRelease func(ref uintptr)
scDynamicStoreCreate func(alloc, name, callout, ctx uintptr) uintptr
scDynamicStoreSetValue func(store, key, value uintptr) bool
scDynamicStoreRemoveValue func(store, key uintptr) bool
scError func() int32
)
// loadFrameworks binds the few CoreFoundation and SystemConfiguration calls we need. purego does
// it without cgo, so this cross compiles and runs on a mac with no developer tools. JXA can read
// the dynamic store but its writes fail, and scutil cannot build the arrays of dictionaries the
// route keys hold.
func loadFrameworks() {
cf, err := purego.Dlopen("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation", purego.RTLD_NOW|purego.RTLD_GLOBAL)
if err != nil {
log.Fatal(err)
}
sc, err := purego.Dlopen("/System/Library/Frameworks/SystemConfiguration.framework/SystemConfiguration", purego.RTLD_NOW|purego.RTLD_GLOBAL)
if err != nil {
log.Fatal(err)
}
purego.RegisterLibFunc(&cfStringCreateWithCString, cf, "CFStringCreateWithCString")
purego.RegisterLibFunc(&cfDataCreate, cf, "CFDataCreate")
purego.RegisterLibFunc(&cfPropertyListCreateWithData, cf, "CFPropertyListCreateWithData")
purego.RegisterLibFunc(&cfRelease, cf, "CFRelease")
purego.RegisterLibFunc(&scDynamicStoreCreate, sc, "SCDynamicStoreCreate")
purego.RegisterLibFunc(&scDynamicStoreSetValue, sc, "SCDynamicStoreSetValue")
purego.RegisterLibFunc(&scDynamicStoreRemoveValue, sc, "SCDynamicStoreRemoveValue")
purego.RegisterLibFunc(&scError, sc, "SCError")
}
func cfString(s string) uintptr {
b := append([]byte(s), 0)
return cfStringCreateWithCString(0, &b[0], cfStringUTF8)
}
// cfPlist builds a property list from its xml form, the least code way to make nested CF values
func cfPlist(xml string) uintptr {
b := []byte(xml)
d := cfDataCreate(0, &b[0], len(b))
defer cfRelease(d)
var errRef uintptr
p := cfPropertyListCreateWithData(0, d, 0, nil, &errRef)
if p == 0 {
log.Fatalf("invalid property list:\n%s", xml)
}
return p
}
type store struct {
ref uintptr
k4, k6 uintptr
}
func openStore() *store {
return &store{
ref: scDynamicStoreCreate(0, cfString("vpnemu"), 0, 0),
k4: cfString("State:/Network/Service/" + serviceID + "/IPv4"),
k6: cfString("State:/Network/Service/" + serviceID + "/IPv6"),
}
}
func (s *store) set(key uintptr, xml string) {
v := cfPlist(xml)
defer cfRelease(v)
if !scDynamicStoreSetValue(s.ref, key, v) {
log.Fatalf("SCDynamicStoreSetValue failed: SCError %d", scError())
}
}
func (s *store) withdraw() {
scDynamicStoreRemoveValue(s.ref, s.k4)
scDynamicStoreRemoveValue(s.ref, s.k6)
}
// createUtun opens a utun the kernel numbers for us and gives it Tailscale-like single addresses.
// The interface lives as long as the returned fd.
func createUtun() (int, string) {
fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL)
if err != nil {
log.Fatalf("system socket: %v", err)
}
info := &unix.CtlInfo{}
copy(info.Name[:], utunControlName)
if err = unix.IoctlCtlInfo(fd, info); err != nil {
log.Fatalf("CTLIOCGINFO: %v", err)
}
if err = unix.Connect(fd, &unix.SockaddrCtl{ID: info.Id, Unit: 0}); err != nil {
log.Fatalf("connect utun control: %v", err)
}
name, err := unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, utunOptIfname)
if err != nil {
log.Fatalf("utun name: %v", err)
}
for _, args := range [][]string{
{name, "inet", addr4, addr4, "netmask", "255.255.255.255", "up"},
{name, "inet6", addr6, "prefixlen", "128"},
} {
if out, err := exec.Command("ifconfig", args...).CombinedOutput(); err != nil {
log.Fatalf("ifconfig %s: %v: %s", strings.Join(args, " "), err, out)
}
}
return fd, name
}
// lanPrefixes applies Tailscale's attached-LAN rule to every interface but those in skip
func lanPrefixes(skip []string) (v4, v6 []netip.Prefix) {
ifs, err := net.Interfaces()
if err != nil {
log.Fatal(err)
}
for _, i := range ifs {
if slices.Contains(skip, i.Name) || i.Flags&net.FlagLoopback != 0 {
continue
}
addrs, _ := i.Addrs()
for _, a := range addrs {
n, ok := a.(*net.IPNet)
if !ok {
continue
}
ip, _ := netip.AddrFromSlice(n.IP)
ip = ip.Unmap()
ones, _ := n.Mask.Size()
pfx := netip.PrefixFrom(ip, ones).Masked()
// Tailscale skips single IPs. Link-local is our addition, every interface has the
// same fe80::/64 and claiming it would only break the harness's own host.
if pfx.IsSingleIP() || ip.IsLinkLocalUnicast() {
continue
}
if ip.Is4() {
v4 = append(v4, pfx)
} else {
v6 = append(v6, pfx)
}
}
}
slices.SortFunc(v4, func(a, b netip.Prefix) int { return a.Addr().Compare(b.Addr()) })
slices.SortFunc(v6, func(a, b netip.Prefix) int { return a.Addr().Compare(b.Addr()) })
return v4, v6
}
func routes4(ps []netip.Prefix) string {
var b strings.Builder
for _, p := range ps {
mask := net.IP(net.CIDRMask(p.Bits(), 32)).String()
fmt.Fprintf(&b, "<dict><key>DestinationAddress</key><string>%s</string><key>SubnetMask</key><string>%s</string></dict>", p.Addr(), mask)
}
return b.String()
}
func routes6(ps []netip.Prefix) string {
var b strings.Builder
for _, p := range ps {
fmt.Fprintf(&b, "<dict><key>DestinationAddress</key><string>%s</string><key>PrefixLength</key><integer>%d</integer></dict>", p.Addr(), p.Bits())
}
return b.String()
}
func (s *store) publish(ifName string, exclude bool, v4, v6 []netip.Prefix) {
routesKey, primary := "AdditionalRoutes", ""
if exclude {
// configd only steers excluded routes for a service with a default route that is primary.
// While this runs, everything not on a kept interface's subnet goes into the utun and dies.
routesKey = "ExcludedRoutes"
primary = "<key>Router</key><string>" + addr4 + "</string><key>OverridePrimary</key><integer>1</integer>"
}
const head = `<?xml version="1.0" encoding="UTF-8"?><plist version="1.0"><dict>`
s.set(s.k4, head+
"<key>InterfaceName</key><string>"+ifName+"</string>"+
"<key>Addresses</key><array><string>"+addr4+"</string></array>"+
"<key>SubnetMasks</key><array><string>255.255.255.255</string></array>"+primary+
"<key>"+routesKey+"</key><array>"+routes4(v4)+"</array></dict></plist>")
s.set(s.k6, head+
"<key>InterfaceName</key><string>"+ifName+"</string>"+
"<key>Addresses</key><array><string>"+addr6+"</string></array>"+
"<key>PrefixLength</key><array><integer>128</integer></array>"+
"<key>"+routesKey+"</key><array>"+routes6(v6)+"</array></dict></plist>")
log.Printf("published %s on %s, %s: %v %v", serviceID, ifName, routesKey, v4, v6)
}
func main() {
mode := flag.String("mode", "claim", "claim: exit node without LAN access, the prefixes are routed into the vpn\n"+
"exclude: exit node with LAN access, the prefixes leave via the next interface and the vpn becomes primary")
keep := flag.String("keep", "en0", "comma separated interfaces whose prefixes are never published, keep the one your ssh session uses")
follow := flag.Bool("follow", true, "republish when interface prefixes change, as Tailscale does on link changes.\n"+
"Off, a route stays published after its interface is gone, which is how a leftover route is staged")
clear := flag.Bool("clear", false, "withdraw a service a crashed run left behind, then exit")
flag.Parse()
if *mode != "claim" && *mode != "exclude" {
log.Fatalf("unknown -mode %q", *mode)
}
if os.Geteuid() != 0 {
log.Fatal("needs root, to create the utun and write configd's dynamic store")
}
loadFrameworks()
s := openStore()
if *clear {
s.withdraw()
log.Print("withdrawn")
return
}
fd, ifName := createUtun()
exclude := *mode == "exclude"
skip := append(strings.Split(*keep, ","), ifName)
v4, v6 := lanPrefixes(skip)
s.publish(ifName, exclude, v4, v6)
sig := make(chan os.Signal, 1)
signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
tick := time.NewTicker(time.Second)
defer tick.Stop()
for {
select {
case <-sig:
s.withdraw()
_ = unix.Close(fd)
log.Print("withdrawn")
return
case <-tick.C:
if !*follow {
continue
}
n4, n6 := lanPrefixes(skip)
if !slices.Equal(n4, v4) || !slices.Equal(n6, v6) {
v4, v6 = n4, n6
s.publish(ifName, exclude, v4, v6)
}
}
}
}
+76
View File
@@ -0,0 +1,76 @@
#!/bin/sh
# Reports where traffic to nebula peers goes before, during and after a vpnemu exit node.
#
# usage: sudo ./repro.sh -n <nebula binary> -c <nebula config> -p <peer vpn addr> [-p <peer>...]
# [-m claim|exclude] [-k <keep interfaces>]
#
# Each peer should answer pings over the overlay. A healthy run shows the nebula utun as egress
# and the ping working in all three phases.
set -u
MODE=claim
KEEP=en0
PEERS=""
NEBULA=""
CONFIG=""
while getopts n:c:p:m:k: o; do
case $o in
n) NEBULA=$OPTARG ;;
c) CONFIG=$OPTARG ;;
p) PEERS="$PEERS $OPTARG" ;;
m) MODE=$OPTARG ;;
k) KEEP=$OPTARG ;;
*) exit 2 ;;
esac
done
if [ -z "$NEBULA" ] || [ -z "$CONFIG" ] || [ -z "$PEERS" ]; then
sed -n '4,5p' "$0"
exit 2
fi
HERE=$(cd "$(dirname "$0")" && pwd)
VPNEMU=${VPNEMU:-$HERE/vpnemu}
LOG=$(mktemp -t nebula-repro)
report() {
for p in $PEERS; do
case $p in
*:*) fam=-inet6 ping=ping6 ;;
*) fam=-inet ping="ping -t 3" ;;
esac
egress=$(route -n get $fam "$p" 2>/dev/null | awk '/interface:/{print $2}')
if $ping -c 2 "$p" >/dev/null 2>&1; then ok=ok; else ok=FAIL; fi
printf '%-8s %-24s egress=%-8s ping=%s\n' "$1" "$p" "${egress:-none}" "$ok"
done
}
cleanup() {
[ -n "${EMU:-}" ] && kill "$EMU" 2>/dev/null
[ -n "${NEB:-}" ] && kill "$NEB" 2>/dev/null
wait 2>/dev/null
}
trap cleanup EXIT INT TERM
"$NEBULA" -config "$CONFIG" >"$LOG" 2>&1 &
NEB=$!
sleep 3
if ! kill -0 $NEB 2>/dev/null; then
echo "nebula exited:"
tail -5 "$LOG"
exit 1
fi
echo "nebula on $(grep -o 'interface=utun[0-9]*' "$LOG" | head -1 | cut -d= -f2), log in $LOG"
report before
"$VPNEMU" -mode "$MODE" -keep "$KEEP" &
EMU=$!
sleep 3
report during
kill $EMU
wait $EMU 2>/dev/null
EMU=
sleep 2
report after
grep -E 'level=(WARN|ERROR).*(route|Route)' "$LOG"
+3 -4
View File
@@ -1,6 +1,6 @@
module github.com/slackhq/nebula module github.com/slackhq/nebula
go 1.27.2 go 1.26.0
require ( require (
dario.cat/mergo v1.0.2 dario.cat/mergo v1.0.2
@@ -33,7 +33,7 @@ require (
golang.zx2c4.com/wireguard/windows v1.0.1 golang.zx2c4.com/wireguard/windows v1.0.1
google.golang.org/protobuf v1.36.12 google.golang.org/protobuf v1.36.12
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980 gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
) )
require ( require (
@@ -45,6 +45,5 @@ require (
github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect
github.com/vishvananda/netns v0.0.5 // indirect github.com/vishvananda/netns v0.0.5 // indirect
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc // indirect golang.org/x/time v0.5.0 // indirect
golang.org/x/time v0.15.0 // indirect
) )
+4 -6
View File
@@ -157,8 +157,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc h1:TS73t7x3KarrNd5qAipmspBDS1rkMcgVG/fS1aRb4Rc=
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -205,8 +203,8 @@ golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -245,5 +243,5 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980 h1:ribyrTYQ/QYeUIs8x8bVLYwGyepa7RnHLqJzegT5YYk= gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe h1:fre4i6mv4iBuz5lCMOzHD1rH1ljqHWSICFmZRbbgp3g=
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980/go.mod h1:pK/dapF5Bf3KGW7/kd5CI5IC4lYh9FMFY1qETAIGLXk= gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU=
+14 -98
View File
@@ -6,10 +6,8 @@ import (
"errors" "errors"
"fmt" "fmt"
"log/slog" "log/slog"
"math/bits"
"net/netip" "net/netip"
"os" "os"
"sync"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -30,7 +28,7 @@ type tun struct {
DefaultMTU int DefaultMTU int
Routes atomic.Pointer[[]Route] Routes atomic.Pointer[[]Route]
routeTree atomic.Pointer[bart.Table[routing.Gateways]] routeTree atomic.Pointer[bart.Table[routing.Gateways]]
linkAddr atomic.Pointer[netroute.LinkAddr] linkAddr *netroute.LinkAddr
// hostOwned means the fd arrived from the OS, which has already configured addressing, mtu // hostOwned means the fd arrived from the OS, which has already configured addressing, mtu
// and routes for it. NEPacketTunnelProvider on darwin does this. // and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool hostOwned bool
@@ -245,7 +243,7 @@ func (t *tun) Activate() error {
if linkAddr == nil { if linkAddr == nil {
return fmt.Errorf("unable to discover link_addr for tun interface") return fmt.Errorf("unable to discover link_addr for tun interface")
} }
t.linkAddr.Store(linkAddr) t.linkAddr = linkAddr
for _, network := range t.vpnNetworks { for _, network := range t.vpnNetworks {
if network.Addr().Is4() { if network.Addr().Is4() {
@@ -271,7 +269,6 @@ func (t *tun) Activate() error {
return t.addRoutes(false) return t.addRoutes(false)
} }
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
func (t *tun) activate4(network netip.Prefix) error { func (t *tun) activate4(network netip.Prefix) error {
s, err := unix.Socket( s, err := unix.Socket(
unix.AF_INET, unix.AF_INET,
@@ -298,32 +295,22 @@ func (t *tun) activate4(network netip.Prefix) error {
MaskAddr: unix.RawSockaddrInet4{ MaskAddr: unix.RawSockaddrInet4{
Len: unix.SizeofSockaddrInet4, Len: unix.SizeofSockaddrInet4,
Family: unix.AF_INET, Family: unix.AF_INET,
Addr: [4]byte{255, 255, 255, 255}, //always make the address a /32 Addr: prefixToMask(network).As4(),
}, },
} }
// On Darwin, VPNs don't edit the routing table directly.
// A network extension hands its routes to the system, and configd installs them.
// configd replaces any route already holding the same prefix, including ours.
// As an example, Tailscale with an exit node treats any interface prefix wider than one address as a LAN.
// With LAN access off, it routes each LAN into Tailscale's utun.
// With LAN access on, it excludes each LAN from its tunnel.
// configd routes an excluded prefix like its own routes would, ignoring ours, so it goes out the default route.
// Either way our route is replaced, and traffic for the vpn network no longer reaches nebula.
// A single address isn't a LAN to Tailscale, so our route is left alone.
// We route the vpn network to the tun explicitly instead.
if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil { if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun v4 address: %s", err) return fmt.Errorf("failed to set tun v4 address: %s", err)
} }
if err := addRoute(network, t.linkAddr.Load()); err != nil { err = addRoute(network, t.linkAddr)
return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err) if err != nil {
return err
} }
return nil return nil
} }
// activate6 is activate4 for IPv6
func (t *tun) activate6(network netip.Prefix) error { func (t *tun) activate6(network netip.Prefix) error {
s, err := unix.Socket( s, err := unix.Socket(
unix.AF_INET6, unix.AF_INET6,
@@ -345,24 +332,20 @@ func (t *tun) activate6(network netip.Prefix) error {
PrefixMask: unix.RawSockaddrInet6{ PrefixMask: unix.RawSockaddrInet6{
Len: unix.SizeofSockaddrInet6, Len: unix.SizeofSockaddrInet6,
Family: unix.AF_INET6, Family: unix.AF_INET6,
// Always set the netmask to be a /128. See activate4 for why we do this. Addr: prefixToMask(network).As16(),
Addr: [16]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
}, },
Lifetime: addrLifetime{ Lifetime: addrLifetime{
// never expires // never expires
Vltime: 0xffffffff, Vltime: 0xffffffff,
Pltime: 0xffffffff, Pltime: 0xffffffff,
}, },
Flags: _IN6_IFF_NODAD, // No DAD, a utun has no neighbors. Flags: _IN6_IFF_NODAD,
} }
if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil { if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun address: %s", err) return fmt.Errorf("failed to set tun address: %s", err)
} }
if err := addRoute(network, t.linkAddr.Load()); err != nil {
return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err)
}
return nil return nil
} }
@@ -411,56 +394,6 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
return routing.Gateways{} return routing.Gateways{}
} }
func fetchRoutes() ([]netroute.Message, error) {
rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_DUMP, 0)
if err != nil {
return nil, err
}
return netroute.ParseRIB(unix.NET_RT_DUMP, rib)
}
// routeInterfaceIn returns the index of the interface prefix is routed through, or 0 if msgs has no route for exactly
// that prefix. Routes scoped to an interface are skipped, they don't stop us adding ours.
func routeInterfaceIn(msgs []netroute.Message, prefix netip.Prefix) int {
for _, msg := range msgs {
m, ok := msg.(*netroute.RouteMessage)
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK || m.Flags&unix.RTF_IFSCOPE != 0 {
continue
}
if p, ok := routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK]); ok && p == prefix.Masked() {
return m.Index
}
}
return 0
}
// routePrefix returns the prefix a route covers. Host routes have no netmask.
func routePrefix(dst, mask netroute.Addr) (netip.Prefix, bool) {
switch d := dst.(type) {
case *netroute.Inet4Addr:
bits := 32
if m, ok := mask.(*netroute.Inet4Addr); ok {
bits = maskBits(m.IP[:])
}
return netip.PrefixFrom(netip.AddrFrom4(d.IP), bits), true
case *netroute.Inet6Addr:
bits := 128
if m, ok := mask.(*netroute.Inet6Addr); ok {
bits = maskBits(m.IP[:])
}
return netip.PrefixFrom(netip.AddrFrom16(d.IP), bits), true
}
return netip.Prefix{}, false
}
func maskBits(mask []byte) int {
n := 0
for _, b := range mask {
n += bits.OnesCount8(b)
}
return n
}
// Get the LinkAddr for the interface of the given name // Get the LinkAddr for the interface of the given name
// Is there an easier way to fetch this when we create the interface? // Is there an easier way to fetch this when we create the interface?
// Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers. // Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers.
@@ -495,14 +428,7 @@ func (t *tun) addRoutes(logErrors bool) error {
return nil return nil
} }
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil {
return nil
}
routes := *t.Routes.Load() routes := *t.Routes.Load()
systemRoutes := sync.OnceValues(fetchRoutes)
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if len(r.Via) == 0 || !r.Install {
@@ -510,16 +436,10 @@ func (t *tun) addRoutes(logErrors bool) error {
continue continue
} }
err := addRoute(r.Cidr, linkAddr) err := addRoute(r.Cidr, t.linkAddr)
if err != nil { if err != nil {
if errors.Is(err, unix.EEXIST) { if errors.Is(err, unix.EEXIST) {
// Every route we added already exists on a reload, only warn about someone else's t.l.Warn("unable to add unsafe_route, identical route already exists", "route", r.Cidr)
msgs, err := systemRoutes()
if err != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
}
} else { } else {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
@@ -530,8 +450,6 @@ func (t *tun) addRoutes(logErrors bool) error {
} }
} else { } else {
t.l.Info("Added route", "route", r) t.l.Info("Added route", "route", r)
// A cidr listed twice would otherwise find our own route missing from an older fetch
systemRoutes = sync.OnceValues(fetchRoutes)
} }
} }
@@ -539,18 +457,16 @@ func (t *tun) addRoutes(logErrors bool) error {
} }
func (t *tun) removeRoutes(routes []Route) error { func (t *tun) removeRoutes(routes []Route) error {
linkAddr := t.linkAddr.Load() if t.hostOwned {
if t.hostOwned || linkAddr == nil {
return nil return nil
} }
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if !r.Install {
// addRoutes never installs these
continue continue
} }
err := delRoute(r.Cidr, linkAddr) err := delRoute(r.Cidr, t.linkAddr)
if err != nil { if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
} else { } else {
-103
View File
@@ -1,103 +0,0 @@
//go:build darwin && !ios && !e2e_testing
package overlay
import (
"net"
"net/netip"
"os"
"os/exec"
"strings"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
netroute "golang.org/x/net/route"
"golang.org/x/sys/unix"
)
// Every mac routes loopback through lo0
func TestRouteInterfaceIn(t *testing.T) {
msgs, err := fetchRoutes()
require.NoError(t, err)
routeInterface := func(s string) int { return routeInterfaceIn(msgs, netip.MustParsePrefix(s)) }
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
assert.Equal(t, lo0.Index, routeInterface("127.0.0.0/8"))
assert.Equal(t, lo0.Index, routeInterface("127.0.0.1/32"), "a host route has no netmask")
assert.Equal(t, lo0.Index, routeInterface("::1/128"))
assert.Equal(t, 0, routeInterface("127.0.0.0/16"), "only an exact prefix counts")
// Most macs also list the default route scoped to their other interfaces
out, err := exec.Command("route", "-n", "get", "default").Output()
require.NoError(t, err)
var name string
for _, line := range strings.Split(string(out), "\n") {
if v, ok := strings.CutPrefix(strings.TrimSpace(line), "interface: "); ok {
name = v
}
}
if name == "" {
t.Skip("no default route")
}
def, err := net.InterfaceByName(name)
require.NoError(t, err)
assert.Equal(t, def.Index, routeInterface("0.0.0.0/0"))
}
func TestRouteInterfaceIn_SkipsScopedCopies(t *testing.T) {
route := func(index, flags int) netroute.Message {
addrs := make([]netroute.Addr, unix.RTAX_MAX)
addrs[unix.RTAX_DST] = &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}}
addrs[unix.RTAX_NETMASK] = &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}}
return &netroute.RouteMessage{Index: index, Flags: flags, Addrs: addrs}
}
p := netip.MustParsePrefix("192.168.1.0/24")
assert.Equal(t, 7, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE), route(7, unix.RTF_UP)}, p))
assert.Equal(t, 0, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE)}, p))
}
// A reload can happen before Activate
func TestTunRoutesBeforeActivate(t *testing.T) {
tn := &tun{l: test.NewLogger()}
routes := []Route{{
Cidr: netip.MustParsePrefix("10.253.1.0/24"),
Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)},
Install: true,
}}
tn.Routes.Store(&routes)
assert.NoError(t, tn.addRoutes(true))
assert.NoError(t, tn.removeRoutes(routes))
}
// A tun.routes entry with no via is never installed, so removing it must not delete a route for the same cidr that
// belongs to someone else
func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating a utun and adding routes needs root")
}
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
cidr := "10.250.0.128/25"
require.NoError(t, exec.Command("route", "-q", "-n", "add", "-net", cidr, "-interface", "lo0").Run())
t.Cleanup(func() { _ = exec.Command("route", "-q", "-n", "delete", "-net", cidr).Run() })
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("tun:\n routes:\n - route: "+cidr+"\n mtu: 1300\n"))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
require.NoError(t, c.ReloadConfigString("tun: {}\n"))
msgs, err := fetchRoutes()
require.NoError(t, err)
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
}
+1 -2
View File
@@ -589,8 +589,7 @@ func (t *tun) addRoutes(logErrors bool) error {
func (t *tun) removeRoutes(routes []Route) error { func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if !r.Install {
// addRoutes never installs these
continue continue
} }
+3 -96
View File
@@ -552,17 +552,13 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
if cidr.Addr().Is6() { if cidr.Addr().Is6() {
nr.Priority = 256 nr.Priority = 256
} }
held, err := t.otherLinkRoutes() err := netlink.RouteReplace(&nr)
if err != nil {
return err
}
err = t.replaceRoute(&nr, held)
if err != nil { if err != nil {
t.l.Warn("Failed to set default route MTU, retrying", "error", err, "cidr", cidr) t.l.Warn("Failed to set default route MTU, retrying", "error", err, "cidr", cidr)
//retry twice more -- on some systems there appears to be a race condition where if we set routes too soon, netlink says `invalid argument` //retry twice more -- on some systems there appears to be a race condition where if we set routes too soon, netlink says `invalid argument`
for range 2 { for range 2 {
time.Sleep(100 * time.Millisecond) time.Sleep(100 * time.Millisecond)
err = t.replaceRoute(&nr, held) err = netlink.RouteReplace(&nr)
if err == nil { if err == nil {
break break
} else { } else {
@@ -582,15 +578,6 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
} }
func (t *tun) addRoutes(logErrors bool) error { func (t *tun) addRoutes(logErrors bool) error {
held, err := t.otherLinkRoutes()
if err != nil {
if logErrors {
t.l.Error("Failed to add routes", "error", err)
return nil
}
return err
}
// Path routes // Path routes
routes := *t.Routes.Load() routes := *t.Routes.Load()
for _, r := range routes { for _, r := range routes {
@@ -615,7 +602,7 @@ func (t *tun) addRoutes(logErrors bool) error {
nr.Priority = r.Metric nr.Priority = r.Metric
} }
err := t.replaceRoute(&nr, held) err := netlink.RouteReplace(&nr)
if err != nil { if err != nil {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
@@ -631,86 +618,6 @@ func (t *tun) addRoutes(logErrors bool) error {
return nil return nil
} }
// ip6RoutePrioUser is the metric the kernel gives an IPv6 route added without one
const ip6RoutePrioUser = 1024
// routeKey is where the main table files a route. Replacing a route replaces the first one with the same key, whatever
// device it is on
type routeKey struct {
dst netip.Prefix
priority int
}
func routeKeyOf(dst *net.IPNet, family, priority int) (routeKey, bool) {
var p netip.Prefix
if dst == nil {
// A default route
if family == unix.AF_INET6 {
p = netip.PrefixFrom(netip.IPv6Unspecified(), 0)
} else {
p = netip.PrefixFrom(netip.IPv4Unspecified(), 0)
}
} else {
addr, ok := netip.AddrFromSlice(dst.IP)
if !ok {
return routeKey{}, false
}
ones, _ := dst.Mask.Size()
p = netip.PrefixFrom(addr.Unmap(), ones).Masked()
}
if priority == 0 && p.Addr().Is6() {
priority = ip6RoutePrioUser
}
return routeKey{dst: p, priority: priority}, true
}
// otherLinkRoutes returns the keys of the main table routes that are not on our device
func (t *tun) otherLinkRoutes() (map[routeKey]struct{}, error) {
routes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, &netlink.Route{Table: unix.RT_TABLE_MAIN}, netlink.RT_FILTER_TABLE)
// An interrupted dump may miss a route, which then gets replaced like it always used to be
if err != nil && !errors.Is(err, netlink.ErrDumpInterrupted) {
return nil, fmt.Errorf("failed to list the system routes: %w", err)
}
held := make(map[routeKey]struct{}, len(routes))
for _, r := range routes {
if r.LinkIndex == t.deviceIndex {
continue
}
if k, ok := routeKeyOf(r.Dst, r.Family, r.Priority); ok {
held[k] = struct{}{}
}
}
return held, nil
}
// replaceRoute puts nr in the main table in place of our own route with its key. Replacing would take the route of
// another device holding that key, so ours goes in behind it instead. Theirs keeps routing and the kernel falls back
// to ours once theirs is removed, like when leaving the lan that had it. A different metric decides which one wins.
func (t *tun) replaceRoute(nr *netlink.Route, held map[routeKey]struct{}) error {
family := unix.AF_INET6
if nr.Dst.IP.To4() != nil {
family = unix.AF_INET
}
k, ok := routeKeyOf(nr.Dst, family, nr.Priority)
if _, taken := held[k]; !ok || !taken {
return netlink.RouteReplace(nr)
}
// Appending doesn't update a route that is already ours, it adds a second one
ours := netlink.Route{
LinkIndex: nr.LinkIndex,
Dst: nr.Dst,
Scope: nr.Scope,
Table: nr.Table,
Priority: k.priority,
}
if err := netlink.RouteDel(&ours); err != nil && !errors.Is(err, unix.ESRCH) {
return err
}
t.l.Warn("Another interface already routes this destination at the same metric, our route is used once theirs is removed", "destination", k.dst, "metric", k.priority)
return netlink.RouteAppend(nr)
}
func (t *tun) removeRoutes(routes []Route) { func (t *tun) removeRoutes(routes []Route) {
for _, r := range routes { for _, r := range routes {
if !r.Install { if !r.Install {
-133
View File
@@ -1,133 +0,0 @@
//go:build linux && !android && !e2e_testing
package overlay
import (
"net"
"net/netip"
"os"
"testing"
"github.com/DefinedNet/netlink"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sys/unix"
)
// An unsafe_route with the same destination and metric as a lan's route must not replace it. Ours waits behind theirs
// and takes over when the lan goes away. Run as root, or unprivileged with `unshare -rn`.
func TestTunRouteLeavesAnotherDevicesRoute(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating links and adding routes needs root")
}
lan := &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: "nebrt0"}, PeerName: "nebrt1"}
require.NoError(t, netlink.LinkAdd(lan))
t.Cleanup(func() { _ = netlink.LinkDel(lan) })
peer, err := netlink.LinkByName("nebrt1")
require.NoError(t, err)
require.NoError(t, netlink.LinkSetUp(lan))
require.NoError(t, netlink.LinkSetUp(peer))
lanAddrs := []*netlink.Addr{
{IPNet: &net.IPNet{IP: net.ParseIP("198.51.100.5"), Mask: net.CIDRMask(24, 32)}},
{IPNet: &net.IPNet{IP: net.ParseIP("2001:db8:1::5"), Mask: net.CIDRMask(64, 128)}, Flags: unix.IFA_F_NODAD},
}
for _, a := range lanAddrs {
require.NoError(t, netlink.AddrAdd(lan, a))
}
l := test.NewLogger()
c := config.NewC(l)
// The kernel puts a v4 lan route at metric 0, our default, and a v6 one at 256
require.NoError(t, c.LoadString(`
tun:
dev: nebrt2
unsafe_routes:
- route: 198.51.100.0/24
via: 10.250.0.2
- route: 2001:db8:1::/64
via: fd00:250::2
metric: 256
`))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
via := func(dst string) int {
routes, err := netlink.RouteGet(net.ParseIP(dst))
require.NoError(t, err)
require.NotEmpty(t, routes)
return routes[0].LinkIndex
}
devices := func(dst string) []int {
_, n, err := net.ParseCIDR(dst)
require.NoError(t, err)
routes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, &netlink.Route{Dst: n}, netlink.RT_FILTER_DST)
require.NoError(t, err)
var out []int
for _, r := range routes {
out = append(out, r.LinkIndex)
}
return out
}
for _, dst := range []string{"198.51.100.0/24", "2001:db8:1::/64"} {
assert.ElementsMatch(t, []int{lan.Index, tn.deviceIndex}, devices(dst), dst)
}
assert.Equal(t, lan.Index, via("198.51.100.9"))
assert.Equal(t, lan.Index, via("2001:db8:1::9"))
// Leaving the lan
for _, a := range lanAddrs {
require.NoError(t, netlink.AddrDel(lan, a))
}
assert.Equal(t, tn.deviceIndex, via("198.51.100.9"))
assert.Equal(t, tn.deviceIndex, via("2001:db8:1::9"))
// Dropping the unsafe_routes removes only ours
for _, a := range lanAddrs {
require.NoError(t, netlink.AddrAdd(lan, a))
}
require.NoError(t, c.ReloadConfigString("tun:\n dev: nebrt2\n"))
for _, dst := range []string{"198.51.100.0/24", "2001:db8:1::/64"} {
assert.Equal(t, []int{lan.Index}, devices(dst), dst)
}
}
// The route for our own vpn network is replaced to set its mtu, which must not take a lan's route for the same network
func TestTunVpnNetworkLeavesAnotherDevicesRoute(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating links and adding routes needs root")
}
lan := &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: "nebrt0"}, PeerName: "nebrt1"}
require.NoError(t, netlink.LinkAdd(lan))
t.Cleanup(func() { _ = netlink.LinkDel(lan) })
require.NoError(t, netlink.LinkSetUp(lan))
require.NoError(t, netlink.AddrAdd(lan, &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP("10.250.0.5"), Mask: net.CIDRMask(24, 32)}}))
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("tun:\n dev: nebrt2\n"))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
_, n, err := net.ParseCIDR("10.250.0.0/24")
require.NoError(t, err)
routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, &netlink.Route{Dst: n}, netlink.RT_FILTER_DST)
require.NoError(t, err)
var devices []int
for _, r := range routes {
devices = append(devices, r.LinkIndex)
if r.LinkIndex == tn.deviceIndex {
assert.Equal(t, tn.DefaultMTU, r.MTU, "ours still carries the mtu")
}
}
assert.ElementsMatch(t, []int{lan.Index, tn.deviceIndex}, devices)
}
+1 -2
View File
@@ -420,8 +420,7 @@ func (t *tun) addRoutes(logErrors bool) error {
func (t *tun) removeRoutes(routes []Route) error { func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if !r.Install {
// addRoutes never installs these
continue continue
} }
+1 -2
View File
@@ -399,8 +399,7 @@ func (t *tun) addRoutes(logErrors bool) error {
func (t *tun) removeRoutes(routes []Route) error { func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if !r.Install {
// addRoutes never installs these
continue continue
} }
+13 -33
View File
@@ -60,20 +60,6 @@ type sshDeviceInfoFlags struct {
func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) { func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C) {
if c.GetBool("sshd.enabled", false) { if c.GetBool("sshd.enabled", false) {
// Restarting the server drops every open session, the one that asked for this reload included. Only restart
// when it isn't running or its config changed. The host key file can change under the same config, new
// handshakes use the new key without a restart
if ssh.Running() && !c.HasChanged("sshd") {
key, err := loadSSHHostKey(c)
if err == nil {
err = ssh.SetHostKey(key)
}
if err != nil {
l.Error("Failed to reconfigure the sshd", "error", err)
ssh.Stop()
}
return
}
sshRun, err := configSSH(l, ssh, c) sshRun, err := configSSH(l, ssh, c)
if err != nil { if err != nil {
l.Error("Failed to reconfigure the sshd", "error", err) l.Error("Failed to reconfigure the sshd", "error", err)
@@ -88,22 +74,6 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
}) })
} }
// loadSSHHostKey returns sshd.host_key, read from its file when it names one.
func loadSSHHostKey(c *config.C) ([]byte, error) {
hostKeyPathOrKey := c.GetString("sshd.host_key", "")
if hostKeyPathOrKey == "" {
return nil, fmt.Errorf("sshd.host_key must be provided")
}
if strings.Contains(hostKeyPathOrKey, "-----BEGIN") {
return []byte(hostKeyPathOrKey), nil
}
b, err := os.ReadFile(hostKeyPathOrKey)
if err != nil {
return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err)
}
return b, nil
}
// configSSH reads the ssh info out of the passed-in Config and // configSSH reads the ssh info out of the passed-in Config and
// updates the passed-in SSHServer. On success, it returns a function // updates the passed-in SSHServer. On success, it returns a function
// that callers may invoke to run the configured ssh server. On // that callers may invoke to run the configured ssh server. On
@@ -122,9 +92,19 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
return nil, fmt.Errorf("sshd.listen can not use port 22") return nil, fmt.Errorf("sshd.listen can not use port 22")
} }
hostKeyBytes, err := loadSSHHostKey(c) hostKeyPathOrKey := c.GetString("sshd.host_key", "")
if err != nil { if hostKeyPathOrKey == "" {
return nil, err return nil, fmt.Errorf("sshd.host_key must be provided")
}
var hostKeyBytes []byte
if strings.Contains(hostKeyPathOrKey, "-----BEGIN") {
hostKeyBytes = []byte(hostKeyPathOrKey)
} else {
hostKeyBytes, err = os.ReadFile(hostKeyPathOrKey)
if err != nil {
return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err)
}
} }
err = ssh.SetHostKey(hostKeyBytes) err = ssh.SetHostKey(hostKeyBytes)
+13 -48
View File
@@ -8,18 +8,14 @@ import (
"log/slog" "log/slog"
"net" "net"
"sync" "sync"
"sync/atomic"
"github.com/armon/go-radix" "github.com/armon/go-radix"
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
) )
type SSHServer struct { type SSHServer struct {
// config is replaced whole when the host key changes, ssh.ServerConfig.AddHostKey only replaces a key of the same config *ssh.ServerConfig
// algorithm and appends any other l *slog.Logger
config atomic.Pointer[ssh.ServerConfig]
authenticate func(ssh.ConnMetadata, ssh.PublicKey) (*ssh.Permissions, error)
l *slog.Logger
certChecker *ssh.CertChecker certChecker *ssh.CertChecker
@@ -32,12 +28,7 @@ type SSHServer struct {
// List of available commands // List of available commands
helpCommand *Command helpCommand *Command
commands *radix.Tree commands *radix.Tree
listener atomic.Pointer[net.Listener] listener net.Listener
// runs counts the Runs that are listening. A reload's Run can start before the one it replaced has returned
runs atomic.Int32
// addr is where the newest Run listens, nil when none is
addr atomic.Pointer[net.Addr]
// ctx parents per-Run contexts. Cancelling it (e.g. via Control.Stop) tears the server down even // ctx parents per-Run contexts. Cancelling it (e.g. via Control.Stop) tears the server down even
// across reloads, since each Run derives a fresh child rather than reusing this one directly. // across reloads, since each Run derives a fresh child rather than reusing this one directly.
@@ -94,8 +85,10 @@ func NewSSHServer(ctx context.Context, l *slog.Logger) (*SSHServer, error) {
}, },
} }
s.authenticate = cc.Authenticate s.config = &ssh.ServerConfig{
s.config.Store(s.newConfig()) PublicKeyCallback: cc.Authenticate,
ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"),
}
s.RegisterCommand(&Command{ s.RegisterCommand(&Command{
Name: "help", Name: "help",
@@ -114,32 +107,10 @@ func (s *SSHServer) SetHostKey(hostPrivateKey []byte) error {
return fmt.Errorf("failed to parse private key: %s", err) return fmt.Errorf("failed to parse private key: %s", err)
} }
config := s.newConfig() s.config.AddHostKey(private)
config.AddHostKey(private)
s.config.Store(config)
return nil return nil
} }
func (s *SSHServer) newConfig() *ssh.ServerConfig {
return &ssh.ServerConfig{
PublicKeyCallback: s.authenticate,
ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"),
}
}
// Addr is the address the newest Run listens on, nil when none is.
func (s *SSHServer) Addr() net.Addr {
if a := s.addr.Load(); a != nil {
return *a
}
return nil
}
// Running is true while a Run is listening.
func (s *SSHServer) Running() bool {
return s.runs.Load() > 0
}
func (s *SSHServer) ClearTrustedCAs() { func (s *SSHServer) ClearTrustedCAs() {
s.authLock.Lock() s.authLock.Lock()
s.trustedCAs = []ssh.PublicKey{} s.trustedCAs = []ssh.PublicKey{}
@@ -210,16 +181,10 @@ func (s *SSHServer) Run(addr string) error {
// this run owns. They start equal but a fast reload may overwrite s.listener with the next run's // this run owns. They start equal but a fast reload may overwrite s.listener with the next run's
// listener before this run's watcher fires, so each run must close its own listener via the local // listener before this run's watcher fires, so each run must close its own listener via the local
// reference. // reference.
s.listener.Store(&listener) s.listener = listener
s.runs.Add(1)
bound := listener.Addr()
s.addr.Store(&bound)
runCtx, cancel := context.WithCancel(s.ctx) runCtx, cancel := context.WithCancel(s.ctx)
defer cancel() defer cancel()
// Deferred after cancel so they run first, a run on its way out no longer counts as listening
defer s.runs.Add(-1)
defer s.addr.CompareAndSwap(&bound, nil)
// Close the listener when this run's context is cancelled. That can come from the parent // Close the listener when this run's context is cancelled. That can come from the parent
// (Control.Stop), from Run returning normally (defer cancel above), or transitively when a sibling // (Control.Stop), from Run returning normally (defer cancel above), or transitively when a sibling
@@ -232,7 +197,7 @@ func (s *SSHServer) Run(addr string) error {
} }
}() }()
s.l.Info("SSH server is listening", "sshListener", bound) s.l.Info("SSH server is listening", "sshListener", addr)
// Run loops until there is an error // Run loops until there is an error
s.run(runCtx, listener) s.run(runCtx, listener)
@@ -261,7 +226,7 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
<-sessionContext.Done() <-sessionContext.Done()
c.Close() c.Close()
}() }()
conn, chans, reqs, err := ssh.NewServerConn(c, s.config.Load()) conn, chans, reqs, err := ssh.NewServerConn(c, s.config)
fp := "" fp := ""
if conn != nil { if conn != nil {
fp = conn.Permissions.Extensions["fp"] fp = conn.Permissions.Extensions["fp"]
@@ -299,8 +264,8 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
} }
func (s *SSHServer) Stop() { func (s *SSHServer) Stop() {
if l := s.listener.Load(); l != nil { if s.listener != nil {
if err := (*l).Close(); err != nil && !errors.Is(err, net.ErrClosed) { if err := s.listener.Close(); err != nil {
s.l.Warn("Failed to close the sshd listener", "error", err) s.l.Warn("Failed to close the sshd listener", "error", err)
} }
} }
-196
View File
@@ -1,196 +0,0 @@
package nebula
import (
"crypto"
"crypto/ed25519"
"crypto/rand"
"crypto/rsa"
"encoding/pem"
"fmt"
"net"
"os"
"path/filepath"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/sshd"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/crypto/ssh"
)
type sshdReloadHarness struct {
c *config.C
server *sshd.SSHServer
keyFile string
signer ssh.Signer
extra string
}
func newSSHDReloadHarness(t *testing.T) *sshdReloadHarness {
t.Helper()
l := test.NewLogger()
h := &sshdReloadHarness{keyFile: filepath.Join(t.TempDir(), "host_key")}
h.writeHostKey(t, newEd25519Key(t))
_, userKey, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
h.signer, err = ssh.NewSignerFromKey(userKey)
require.NoError(t, err)
h.c = config.NewC(l)
require.NoError(t, h.c.LoadString(h.config()))
h.server, err = sshd.NewSSHServer(t.Context(), l)
require.NoError(t, err)
wireSSHReload(l, h.server, h.c)
run, err := configSSH(l, h.server, h.c)
require.NoError(t, err)
go run()
t.Cleanup(h.server.Stop)
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
return h
}
func newEd25519Key(t *testing.T) crypto.Signer {
_, key, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
return key
}
// writeHostKey replaces the host key file and returns the public key a client should now see
func (h *sshdReloadHarness) writeHostKey(t *testing.T, key crypto.Signer) ssh.PublicKey {
t.Helper()
block, err := ssh.MarshalPrivateKey(key, "")
require.NoError(t, err)
require.NoError(t, os.WriteFile(h.keyFile, pem.EncodeToMemory(block), 0o600))
pub, err := ssh.NewPublicKey(key.Public())
require.NoError(t, err)
return pub
}
func (h *sshdReloadHarness) config() string {
return fmt.Sprintf("sshd:\n enabled: true\n listen: 127.0.0.1:0\n host_key: %s\n authorized_users:\n - user: test\n keys: [%q]\n%s",
h.keyFile, string(ssh.MarshalAuthorizedKey(h.signer.PublicKey())), h.extra)
}
// dial connects to wherever the server listens now, a restart on :0 moves it. hostKeyAlgorithms limits which host keys
// the client accepts, none means any
func (h *sshdReloadHarness) dial(hostKeyAlgorithms ...string) (*ssh.Client, ssh.PublicKey, error) {
addr := h.server.Addr()
if addr == nil {
return nil, nil, fmt.Errorf("not listening")
}
var hostKey ssh.PublicKey
c, err := ssh.Dial("tcp", addr.String(), &ssh.ClientConfig{
User: "test",
Auth: []ssh.AuthMethod{ssh.PublicKeys(h.signer)},
HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error {
hostKey = key
return nil
},
HostKeyAlgorithms: hostKeyAlgorithms,
Timeout: time.Second,
})
return c, hostKey, err
}
func (h *sshdReloadHarness) connect(t *testing.T) (*ssh.Client, ssh.PublicKey) {
t.Helper()
var client *ssh.Client
var hostKey ssh.PublicKey
require.Eventually(t, func() bool {
var err error
client, hostKey, err = h.dial()
return err == nil
}, 5*time.Second, 20*time.Millisecond)
t.Cleanup(func() { _ = client.Close() })
return client, hostKey
}
// dropped is true when the server closed the client's connection within wait
func dropped(client *ssh.Client, wait time.Duration) bool {
done := make(chan struct{})
go func() {
_ = client.Wait()
close(done)
}()
select {
case <-done:
return true
case <-time.After(wait):
return false
}
}
// Restarting sshd drops every open session, the one that asked for the reload included. A reload that touches nothing
// sshd reads leaves it alone
func TestSSHDReload_UnchangedKeepsSessions(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
require.NoError(t, h.c.ReloadConfigString(h.config()+"punchy:\n punch: true\n"))
assert.False(t, dropped(client, 500*time.Millisecond), "an unchanged sshd was restarted")
}
func TestSSHDReload_ChangedConfigRestarts(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
h.extra = " trusted_cas: []\n"
require.NoError(t, h.c.ReloadConfigString(h.config()))
assert.True(t, dropped(client, 5*time.Second), "a changed sshd config was not applied")
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t)
}
// The config can stay the same while the host key file it names changes. New connections get the new key, open
// sessions are left alone
func TestSSHDReload_ChangedHostKeyFile(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
want := h.writeHostKey(t, newEd25519Key(t))
require.NoError(t, h.c.ReloadConfigString(h.config()))
_, got := h.connect(t)
assert.Equal(t, want.Marshal(), got.Marshal(), "a new host key was not applied")
assert.False(t, dropped(client, 500*time.Millisecond), "a new host key restarted sshd")
}
// A host key of another algorithm replaces the old one rather than being offered next to it
func TestSSHDReload_HostKeyAlgorithmChange(t *testing.T) {
h := newSSHDReloadHarness(t)
h.connect(t)
rsaKey, err := rsa.GenerateKey(rand.Reader, 2048)
require.NoError(t, err)
want := h.writeHostKey(t, rsaKey)
require.NoError(t, h.c.ReloadConfigString(h.config()))
_, got := h.connect(t)
assert.Equal(t, want.Marshal(), got.Marshal())
_, _, err = h.dial(ssh.KeyAlgoED25519)
assert.Error(t, err, "the old ed25519 host key is still offered")
}
// A server that isn't running, say its port was busy, is tried again on every reload as before
func TestSSHDReload_NotRunningRestarts(t *testing.T) {
h := newSSHDReloadHarness(t)
h.server.Stop()
require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond)
require.NoError(t, h.c.ReloadConfigString(h.config()))
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t)
}
// A host key that no longer loads stops sshd, the same as a config that doesn't
func TestSSHDReload_BadHostKeyFileStops(t *testing.T) {
h := newSSHDReloadHarness(t)
h.connect(t)
require.NoError(t, os.WriteFile(h.keyFile, []byte("not a key"), 0o600))
require.NoError(t, h.c.ReloadConfigString(h.config()))
require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond)
}