From 781a590b8a1d056848f1f1fd0435c65405ce69df Mon Sep 17 00:00:00 2001 From: JackDoan Date: Mon, 12 Oct 2026 12:49:28 -0500 Subject: [PATCH] e2e/darwin: Add vpnemu, a configd harness that reproduces exit node route hijacks --- e2e/darwin/vpnemu/README.md | 68 +++++++++ e2e/darwin/vpnemu/go.mod | 8 + e2e/darwin/vpnemu/go.sum | 4 + e2e/darwin/vpnemu/main.go | 292 ++++++++++++++++++++++++++++++++++++ e2e/darwin/vpnemu/repro.sh | 76 ++++++++++ 5 files changed, 448 insertions(+) create mode 100644 e2e/darwin/vpnemu/README.md create mode 100644 e2e/darwin/vpnemu/go.mod create mode 100644 e2e/darwin/vpnemu/go.sum create mode 100644 e2e/darwin/vpnemu/main.go create mode 100755 e2e/darwin/vpnemu/repro.sh diff --git a/e2e/darwin/vpnemu/README.md b/e2e/darwin/vpnemu/README.md new file mode 100644 index 00000000..dab3f230 --- /dev/null +++ b/e2e/darwin/vpnemu/README.md @@ -0,0 +1,68 @@ +# vpnemu + +Reproduces, on macOS and without Tailscale, the way a VPN with an exit node takes nebula's vpn network route +away from its utun (#1931). + +A macOS network extension never touches the routing table. Its routes are published in configd's dynamic store +under `State:/Network/Service//IPv4` and `/IPv6`, and configd's IPMonitor installs them. vpnemu creates a +utun and publishes those keys itself, so configd does exactly what it would do for the real app. It publishes +the routes Tailscale would with an exit node: every interface prefix that isn't a single IP, the rule in +Tailscale's `ipnlocal.internalAndExternalInterfacesFrom`. + +| mode | Tailscale setting | published as | where nebula's prefix goes | +| --- | --- | --- | --- | +| `claim` | exit node, no LAN access | `AdditionalRoutes` | into vpnemu's utun | +| `exclude` | exit node, LAN access | `ExcludedRoutes`, service overrides primary | out the next interface (en0) | + +configd answers `EEXIST` with a delete and an add, so whatever held the prefix loses it. When the service is +withdrawn, configd deletes its routes by prefix alone, whoever holds them by then. + +## Build + +It is its own module, so `purego` stays out of nebula's `go.mod`. purego reaches CoreFoundation and +SystemConfiguration without cgo, so it cross compiles and runs on a mac with no developer tools. + +```sh +cd e2e/darwin/vpnemu +CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o vpnemu . +# a cross compiled binary copied to a mac may need: codesign -s - -f vpnemu +``` + +## Run + +```sh +sudo ./vpnemu -mode claim -keep en0 # runs until ctrl-c, then withdraws and closes its utun +sudo ./vpnemu -clear # withdraw a service a crashed run left behind +``` + +`-keep` names interfaces whose prefixes are never published. Keep the one your ssh session arrives on, or +`claim` takes your LAN into a utun nobody reads. `exclude` makes vpnemu primary, so anything off a kept +interface's subnet, internet included, goes into its utun and is dropped while it runs. + +`-follow` (default on) republishes when interface prefixes change, as Tailscale does on link changes. With +`-follow=false` a route stays published after its interface is gone, which stages the route configd leaves +behind for a nebula that restarts before the VPN has caught up. + +## A/B a nebula build + +`repro.sh` starts nebula, then runs vpnemu, and reports egress and ping per peer before, during and after: + +```sh +sudo ./repro.sh -n ./nebula -c config.yml -p 192.168.212.1 -p fd19:31::1 -m claim -k en0 +``` + +Before #1931 (utun holds `/24` and `/64`): + +``` +during 192.168.212.1 egress=utun2 ping=FAIL +after 192.168.212.1 egress=en0 ping=FAIL +``` + +With #1931 (utun holds `/32` and `/128`, network routes added explicitly) every phase stays on nebula's utun. + +### Leftover route + +Run an old nebula, start `vpnemu -follow=false`, then restart onto a new nebula while vpnemu is still up. The +new nebula finds the vpn network routed to vpnemu's utun, replaces it and starts. Stop vpnemu and configd deletes +the prefix, taking nebula's route with it until nebula restarts. With real Tailscale this needs nebula to restart +before Tailscale notices the old utun is gone, a one time window when upgrading from a nebula without #1931. diff --git a/e2e/darwin/vpnemu/go.mod b/e2e/darwin/vpnemu/go.mod new file mode 100644 index 00000000..2f0d62a2 --- /dev/null +++ b/e2e/darwin/vpnemu/go.mod @@ -0,0 +1,8 @@ +module github.com/slackhq/nebula/e2e/darwin/vpnemu + +go 1.26.0 + +require ( + github.com/ebitengine/purego v0.9.1 + golang.org/x/sys v0.48.0 +) diff --git a/e2e/darwin/vpnemu/go.sum b/e2e/darwin/vpnemu/go.sum new file mode 100644 index 00000000..86373a1f --- /dev/null +++ b/e2e/darwin/vpnemu/go.sum @@ -0,0 +1,4 @@ +github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A= +github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= +golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/e2e/darwin/vpnemu/main.go b/e2e/darwin/vpnemu/main.go new file mode 100644 index 00000000..85cc693a --- /dev/null +++ b/e2e/darwin/vpnemu/main.go @@ -0,0 +1,292 @@ +//go:build darwin + +// vpnemu stands in for a macOS VPN app with an exit node in use, so route hijacks like the one +// Tailscale causes can be reproduced without Tailscale. +// +// A network extension does not touch the routing table itself. It hands its routes to +// nesessionmanager, which publishes them in configd's dynamic store under +// State:/Network/Service//IPv4 and /IPv6, and configd's IPMonitor installs them. vpnemu +// creates a utun and publishes the same keys directly, so configd does exactly what it would do +// for the real thing. +// +// Which routes it publishes follows Tailscale's rule for exit nodes +// (ipnlocal.internalAndExternalInterfacesFrom): every interface address prefix that is not a +// single IP counts as an attached LAN. With LAN access off those prefixes are claimed into the +// tunnel (-mode claim, published as AdditionalRoutes). With LAN access on they are excluded and +// leave through the next interface (-mode exclude, published as ExcludedRoutes on a service that +// overrides the primary). +// +// It runs until interrupted, then withdraws the service and closes its utun. Needs root. +package main + +import ( + "flag" + "fmt" + "log" + "net" + "net/netip" + "os" + "os/exec" + "os/signal" + "slices" + "strings" + "syscall" + "time" + + "github.com/ebitengine/purego" + "golang.org/x/sys/unix" +) + +const ( + serviceID = "VPNEMU" + // Tailscale's own addresses, single IPs, so the rule skips them like Tailscale skips its own + addr4 = "100.100.0.1" + addr6 = "fd7a:115c:a1e0::1" + + utunControlName = "com.apple.net.utun_control" + utunOptIfname = 2 + cfStringUTF8 = 0x08000100 +) + +var ( + cfStringCreateWithCString func(alloc uintptr, s *byte, enc uint32) uintptr + cfDataCreate func(alloc uintptr, b *byte, n int) uintptr + cfPropertyListCreateWithData func(alloc, data uintptr, opts uint, format *int, err *uintptr) uintptr + cfRelease func(ref uintptr) + scDynamicStoreCreate func(alloc, name, callout, ctx uintptr) uintptr + scDynamicStoreSetValue func(store, key, value uintptr) bool + scDynamicStoreRemoveValue func(store, key uintptr) bool + scError func() int32 +) + +// loadFrameworks binds the few CoreFoundation and SystemConfiguration calls we need. purego does +// it without cgo, so this cross compiles and runs on a mac with no developer tools. JXA can read +// the dynamic store but its writes fail, and scutil cannot build the arrays of dictionaries the +// route keys hold. +func loadFrameworks() { + cf, err := purego.Dlopen("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation", purego.RTLD_NOW|purego.RTLD_GLOBAL) + if err != nil { + log.Fatal(err) + } + sc, err := purego.Dlopen("/System/Library/Frameworks/SystemConfiguration.framework/SystemConfiguration", purego.RTLD_NOW|purego.RTLD_GLOBAL) + if err != nil { + log.Fatal(err) + } + purego.RegisterLibFunc(&cfStringCreateWithCString, cf, "CFStringCreateWithCString") + purego.RegisterLibFunc(&cfDataCreate, cf, "CFDataCreate") + purego.RegisterLibFunc(&cfPropertyListCreateWithData, cf, "CFPropertyListCreateWithData") + purego.RegisterLibFunc(&cfRelease, cf, "CFRelease") + purego.RegisterLibFunc(&scDynamicStoreCreate, sc, "SCDynamicStoreCreate") + purego.RegisterLibFunc(&scDynamicStoreSetValue, sc, "SCDynamicStoreSetValue") + purego.RegisterLibFunc(&scDynamicStoreRemoveValue, sc, "SCDynamicStoreRemoveValue") + purego.RegisterLibFunc(&scError, sc, "SCError") +} + +func cfString(s string) uintptr { + b := append([]byte(s), 0) + return cfStringCreateWithCString(0, &b[0], cfStringUTF8) +} + +// cfPlist builds a property list from its xml form, the least code way to make nested CF values +func cfPlist(xml string) uintptr { + b := []byte(xml) + d := cfDataCreate(0, &b[0], len(b)) + defer cfRelease(d) + var errRef uintptr + p := cfPropertyListCreateWithData(0, d, 0, nil, &errRef) + if p == 0 { + log.Fatalf("invalid property list:\n%s", xml) + } + return p +} + +type store struct { + ref uintptr + k4, k6 uintptr +} + +func openStore() *store { + return &store{ + ref: scDynamicStoreCreate(0, cfString("vpnemu"), 0, 0), + k4: cfString("State:/Network/Service/" + serviceID + "/IPv4"), + k6: cfString("State:/Network/Service/" + serviceID + "/IPv6"), + } +} + +func (s *store) set(key uintptr, xml string) { + v := cfPlist(xml) + defer cfRelease(v) + if !scDynamicStoreSetValue(s.ref, key, v) { + log.Fatalf("SCDynamicStoreSetValue failed: SCError %d", scError()) + } +} + +func (s *store) withdraw() { + scDynamicStoreRemoveValue(s.ref, s.k4) + scDynamicStoreRemoveValue(s.ref, s.k6) +} + +// createUtun opens a utun the kernel numbers for us and gives it Tailscale-like single addresses. +// The interface lives as long as the returned fd. +func createUtun() (int, string) { + fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL) + if err != nil { + log.Fatalf("system socket: %v", err) + } + info := &unix.CtlInfo{} + copy(info.Name[:], utunControlName) + if err = unix.IoctlCtlInfo(fd, info); err != nil { + log.Fatalf("CTLIOCGINFO: %v", err) + } + if err = unix.Connect(fd, &unix.SockaddrCtl{ID: info.Id, Unit: 0}); err != nil { + log.Fatalf("connect utun control: %v", err) + } + name, err := unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, utunOptIfname) + if err != nil { + log.Fatalf("utun name: %v", err) + } + + for _, args := range [][]string{ + {name, "inet", addr4, addr4, "netmask", "255.255.255.255", "up"}, + {name, "inet6", addr6, "prefixlen", "128"}, + } { + if out, err := exec.Command("ifconfig", args...).CombinedOutput(); err != nil { + log.Fatalf("ifconfig %s: %v: %s", strings.Join(args, " "), err, out) + } + } + return fd, name +} + +// lanPrefixes applies Tailscale's attached-LAN rule to every interface but those in skip +func lanPrefixes(skip []string) (v4, v6 []netip.Prefix) { + ifs, err := net.Interfaces() + if err != nil { + log.Fatal(err) + } + for _, i := range ifs { + if slices.Contains(skip, i.Name) || i.Flags&net.FlagLoopback != 0 { + continue + } + addrs, _ := i.Addrs() + for _, a := range addrs { + n, ok := a.(*net.IPNet) + if !ok { + continue + } + ip, _ := netip.AddrFromSlice(n.IP) + ip = ip.Unmap() + ones, _ := n.Mask.Size() + pfx := netip.PrefixFrom(ip, ones).Masked() + // Tailscale skips single IPs. Link-local is our addition, every interface has the + // same fe80::/64 and claiming it would only break the harness's own host. + if pfx.IsSingleIP() || ip.IsLinkLocalUnicast() { + continue + } + if ip.Is4() { + v4 = append(v4, pfx) + } else { + v6 = append(v6, pfx) + } + } + } + slices.SortFunc(v4, func(a, b netip.Prefix) int { return a.Addr().Compare(b.Addr()) }) + slices.SortFunc(v6, func(a, b netip.Prefix) int { return a.Addr().Compare(b.Addr()) }) + return v4, v6 +} + +func routes4(ps []netip.Prefix) string { + var b strings.Builder + for _, p := range ps { + mask := net.IP(net.CIDRMask(p.Bits(), 32)).String() + fmt.Fprintf(&b, "DestinationAddress%sSubnetMask%s", p.Addr(), mask) + } + return b.String() +} + +func routes6(ps []netip.Prefix) string { + var b strings.Builder + for _, p := range ps { + fmt.Fprintf(&b, "DestinationAddress%sPrefixLength%d", p.Addr(), p.Bits()) + } + return b.String() +} + +func (s *store) publish(ifName string, exclude bool, v4, v6 []netip.Prefix) { + routesKey, primary := "AdditionalRoutes", "" + if exclude { + // configd only steers excluded routes for a service with a default route that is primary. + // While this runs, everything not on a kept interface's subnet goes into the utun and dies. + routesKey = "ExcludedRoutes" + primary = "Router" + addr4 + "OverridePrimary1" + } + + const head = `` + s.set(s.k4, head+ + "InterfaceName"+ifName+""+ + "Addresses"+addr4+""+ + "SubnetMasks255.255.255.255"+primary+ + ""+routesKey+""+routes4(v4)+"") + s.set(s.k6, head+ + "InterfaceName"+ifName+""+ + "Addresses"+addr6+""+ + "PrefixLength128"+ + ""+routesKey+""+routes6(v6)+"") + + log.Printf("published %s on %s, %s: %v %v", serviceID, ifName, routesKey, v4, v6) +} + +func main() { + mode := flag.String("mode", "claim", "claim: exit node without LAN access, the prefixes are routed into the vpn\n"+ + "exclude: exit node with LAN access, the prefixes leave via the next interface and the vpn becomes primary") + keep := flag.String("keep", "en0", "comma separated interfaces whose prefixes are never published, keep the one your ssh session uses") + follow := flag.Bool("follow", true, "republish when interface prefixes change, as Tailscale does on link changes.\n"+ + "Off, a route stays published after its interface is gone, which is how a leftover route is staged") + clear := flag.Bool("clear", false, "withdraw a service a crashed run left behind, then exit") + flag.Parse() + + if *mode != "claim" && *mode != "exclude" { + log.Fatalf("unknown -mode %q", *mode) + } + if os.Geteuid() != 0 { + log.Fatal("needs root, to create the utun and write configd's dynamic store") + } + + loadFrameworks() + s := openStore() + if *clear { + s.withdraw() + log.Print("withdrawn") + return + } + + fd, ifName := createUtun() + exclude := *mode == "exclude" + skip := append(strings.Split(*keep, ","), ifName) + + v4, v6 := lanPrefixes(skip) + s.publish(ifName, exclude, v4, v6) + + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) + tick := time.NewTicker(time.Second) + defer tick.Stop() + + for { + select { + case <-sig: + s.withdraw() + _ = unix.Close(fd) + log.Print("withdrawn") + return + case <-tick.C: + if !*follow { + continue + } + n4, n6 := lanPrefixes(skip) + if !slices.Equal(n4, v4) || !slices.Equal(n6, v6) { + v4, v6 = n4, n6 + s.publish(ifName, exclude, v4, v6) + } + } + } +} diff --git a/e2e/darwin/vpnemu/repro.sh b/e2e/darwin/vpnemu/repro.sh new file mode 100755 index 00000000..454022b4 --- /dev/null +++ b/e2e/darwin/vpnemu/repro.sh @@ -0,0 +1,76 @@ +#!/bin/sh +# Reports where traffic to nebula peers goes before, during and after a vpnemu exit node. +# +# usage: sudo ./repro.sh -n -c -p [-p ...] +# [-m claim|exclude] [-k ] +# +# Each peer should answer pings over the overlay. A healthy run shows the nebula utun as egress +# and the ping working in all three phases. +set -u + +MODE=claim +KEEP=en0 +PEERS="" +NEBULA="" +CONFIG="" +while getopts n:c:p:m:k: o; do + case $o in + n) NEBULA=$OPTARG ;; + c) CONFIG=$OPTARG ;; + p) PEERS="$PEERS $OPTARG" ;; + m) MODE=$OPTARG ;; + k) KEEP=$OPTARG ;; + *) exit 2 ;; + esac +done +if [ -z "$NEBULA" ] || [ -z "$CONFIG" ] || [ -z "$PEERS" ]; then + sed -n '4,5p' "$0" + exit 2 +fi + +HERE=$(cd "$(dirname "$0")" && pwd) +VPNEMU=${VPNEMU:-$HERE/vpnemu} +LOG=$(mktemp -t nebula-repro) + +report() { + for p in $PEERS; do + case $p in + *:*) fam=-inet6 ping=ping6 ;; + *) fam=-inet ping="ping -t 3" ;; + esac + egress=$(route -n get $fam "$p" 2>/dev/null | awk '/interface:/{print $2}') + if $ping -c 2 "$p" >/dev/null 2>&1; then ok=ok; else ok=FAIL; fi + printf '%-8s %-24s egress=%-8s ping=%s\n' "$1" "$p" "${egress:-none}" "$ok" + done +} + +cleanup() { + [ -n "${EMU:-}" ] && kill "$EMU" 2>/dev/null + [ -n "${NEB:-}" ] && kill "$NEB" 2>/dev/null + wait 2>/dev/null +} +trap cleanup EXIT INT TERM + +"$NEBULA" -config "$CONFIG" >"$LOG" 2>&1 & +NEB=$! +sleep 3 +if ! kill -0 $NEB 2>/dev/null; then + echo "nebula exited:" + tail -5 "$LOG" + exit 1 +fi +echo "nebula on $(grep -o 'interface=utun[0-9]*' "$LOG" | head -1 | cut -d= -f2), log in $LOG" +report before + +"$VPNEMU" -mode "$MODE" -keep "$KEEP" & +EMU=$! +sleep 3 +report during + +kill $EMU +wait $EMU 2>/dev/null +EMU= +sleep 2 +report after + +grep -E 'level=(WARN|ERROR).*(route|Route)' "$LOG"