Compare commits

..
15 changed files with 505 additions and 498 deletions

No files matched your search

+68
View File
@@ -0,0 +1,68 @@
# vpnemu
Reproduces, on macOS and without Tailscale, the way a VPN with an exit node takes nebula's vpn network route
away from its utun (#1931).
A macOS network extension never touches the routing table. Its routes are published in configd's dynamic store
under `State:/Network/Service/<id>/IPv4` and `/IPv6`, and configd's IPMonitor installs them. vpnemu creates a
utun and publishes those keys itself, so configd does exactly what it would do for the real app. It publishes
the routes Tailscale would with an exit node: every interface prefix that isn't a single IP, the rule in
Tailscale's `ipnlocal.internalAndExternalInterfacesFrom`.
| mode | Tailscale setting | published as | where nebula's prefix goes |
| --- | --- | --- | --- |
| `claim` | exit node, no LAN access | `AdditionalRoutes` | into vpnemu's utun |
| `exclude` | exit node, LAN access | `ExcludedRoutes`, service overrides primary | out the next interface (en0) |
configd answers `EEXIST` with a delete and an add, so whatever held the prefix loses it. When the service is
withdrawn, configd deletes its routes by prefix alone, whoever holds them by then.
## Build
It is its own module, so `purego` stays out of nebula's `go.mod`. purego reaches CoreFoundation and
SystemConfiguration without cgo, so it cross compiles and runs on a mac with no developer tools.
```sh
cd e2e/darwin/vpnemu
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o vpnemu .
# a cross compiled binary copied to a mac may need: codesign -s - -f vpnemu
```
## Run
```sh
sudo ./vpnemu -mode claim -keep en0 # runs until ctrl-c, then withdraws and closes its utun
sudo ./vpnemu -clear # withdraw a service a crashed run left behind
```
`-keep` names interfaces whose prefixes are never published. Keep the one your ssh session arrives on, or
`claim` takes your LAN into a utun nobody reads. `exclude` makes vpnemu primary, so anything off a kept
interface's subnet, internet included, goes into its utun and is dropped while it runs.
`-follow` (default on) republishes when interface prefixes change, as Tailscale does on link changes. With
`-follow=false` a route stays published after its interface is gone, which stages the route configd leaves
behind for a nebula that restarts before the VPN has caught up.
## A/B a nebula build
`repro.sh` starts nebula, then runs vpnemu, and reports egress and ping per peer before, during and after:
```sh
sudo ./repro.sh -n ./nebula -c config.yml -p 192.168.212.1 -p fd19:31::1 -m claim -k en0
```
Before #1931 (utun holds `/24` and `/64`):
```
during 192.168.212.1 egress=utun2 ping=FAIL
after 192.168.212.1 egress=en0 ping=FAIL
```
With #1931 (utun holds `/32` and `/128`, network routes added explicitly) every phase stays on nebula's utun.
### Leftover route
Run an old nebula, start `vpnemu -follow=false`, then restart onto a new nebula while vpnemu is still up. The
new nebula finds the vpn network routed to vpnemu's utun, replaces it and starts. Stop vpnemu and configd deletes
the prefix, taking nebula's route with it until nebula restarts. With real Tailscale this needs nebula to restart
before Tailscale notices the old utun is gone, a one time window when upgrading from a nebula without #1931.
+8
View File
@@ -0,0 +1,8 @@
module github.com/slackhq/nebula/e2e/darwin/vpnemu
go 1.26.0
require (
github.com/ebitengine/purego v0.9.1
golang.org/x/sys v0.48.0
)
+4
View File
@@ -0,0 +1,4 @@
github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
+292
View File
@@ -0,0 +1,292 @@
//go:build darwin
// vpnemu stands in for a macOS VPN app with an exit node in use, so route hijacks like the one
// Tailscale causes can be reproduced without Tailscale.
//
// A network extension does not touch the routing table itself. It hands its routes to
// nesessionmanager, which publishes them in configd's dynamic store under
// State:/Network/Service/<id>/IPv4 and /IPv6, and configd's IPMonitor installs them. vpnemu
// creates a utun and publishes the same keys directly, so configd does exactly what it would do
// for the real thing.
//
// Which routes it publishes follows Tailscale's rule for exit nodes
// (ipnlocal.internalAndExternalInterfacesFrom): every interface address prefix that is not a
// single IP counts as an attached LAN. With LAN access off those prefixes are claimed into the
// tunnel (-mode claim, published as AdditionalRoutes). With LAN access on they are excluded and
// leave through the next interface (-mode exclude, published as ExcludedRoutes on a service that
// overrides the primary).
//
// It runs until interrupted, then withdraws the service and closes its utun. Needs root.
package main
import (
"flag"
"fmt"
"log"
"net"
"net/netip"
"os"
"os/exec"
"os/signal"
"slices"
"strings"
"syscall"
"time"
"github.com/ebitengine/purego"
"golang.org/x/sys/unix"
)
const (
serviceID = "VPNEMU"
// Tailscale's own addresses, single IPs, so the rule skips them like Tailscale skips its own
addr4 = "100.100.0.1"
addr6 = "fd7a:115c:a1e0::1"
utunControlName = "com.apple.net.utun_control"
utunOptIfname = 2
cfStringUTF8 = 0x08000100
)
var (
cfStringCreateWithCString func(alloc uintptr, s *byte, enc uint32) uintptr
cfDataCreate func(alloc uintptr, b *byte, n int) uintptr
cfPropertyListCreateWithData func(alloc, data uintptr, opts uint, format *int, err *uintptr) uintptr
cfRelease func(ref uintptr)
scDynamicStoreCreate func(alloc, name, callout, ctx uintptr) uintptr
scDynamicStoreSetValue func(store, key, value uintptr) bool
scDynamicStoreRemoveValue func(store, key uintptr) bool
scError func() int32
)
// loadFrameworks binds the few CoreFoundation and SystemConfiguration calls we need. purego does
// it without cgo, so this cross compiles and runs on a mac with no developer tools. JXA can read
// the dynamic store but its writes fail, and scutil cannot build the arrays of dictionaries the
// route keys hold.
func loadFrameworks() {
cf, err := purego.Dlopen("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation", purego.RTLD_NOW|purego.RTLD_GLOBAL)
if err != nil {
log.Fatal(err)
}
sc, err := purego.Dlopen("/System/Library/Frameworks/SystemConfiguration.framework/SystemConfiguration", purego.RTLD_NOW|purego.RTLD_GLOBAL)
if err != nil {
log.Fatal(err)
}
purego.RegisterLibFunc(&cfStringCreateWithCString, cf, "CFStringCreateWithCString")
purego.RegisterLibFunc(&cfDataCreate, cf, "CFDataCreate")
purego.RegisterLibFunc(&cfPropertyListCreateWithData, cf, "CFPropertyListCreateWithData")
purego.RegisterLibFunc(&cfRelease, cf, "CFRelease")
purego.RegisterLibFunc(&scDynamicStoreCreate, sc, "SCDynamicStoreCreate")
purego.RegisterLibFunc(&scDynamicStoreSetValue, sc, "SCDynamicStoreSetValue")
purego.RegisterLibFunc(&scDynamicStoreRemoveValue, sc, "SCDynamicStoreRemoveValue")
purego.RegisterLibFunc(&scError, sc, "SCError")
}
func cfString(s string) uintptr {
b := append([]byte(s), 0)
return cfStringCreateWithCString(0, &b[0], cfStringUTF8)
}
// cfPlist builds a property list from its xml form, the least code way to make nested CF values
func cfPlist(xml string) uintptr {
b := []byte(xml)
d := cfDataCreate(0, &b[0], len(b))
defer cfRelease(d)
var errRef uintptr
p := cfPropertyListCreateWithData(0, d, 0, nil, &errRef)
if p == 0 {
log.Fatalf("invalid property list:\n%s", xml)
}
return p
}
type store struct {
ref uintptr
k4, k6 uintptr
}
func openStore() *store {
return &store{
ref: scDynamicStoreCreate(0, cfString("vpnemu"), 0, 0),
k4: cfString("State:/Network/Service/" + serviceID + "/IPv4"),
k6: cfString("State:/Network/Service/" + serviceID + "/IPv6"),
}
}
func (s *store) set(key uintptr, xml string) {
v := cfPlist(xml)
defer cfRelease(v)
if !scDynamicStoreSetValue(s.ref, key, v) {
log.Fatalf("SCDynamicStoreSetValue failed: SCError %d", scError())
}
}
func (s *store) withdraw() {
scDynamicStoreRemoveValue(s.ref, s.k4)
scDynamicStoreRemoveValue(s.ref, s.k6)
}
// createUtun opens a utun the kernel numbers for us and gives it Tailscale-like single addresses.
// The interface lives as long as the returned fd.
func createUtun() (int, string) {
fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL)
if err != nil {
log.Fatalf("system socket: %v", err)
}
info := &unix.CtlInfo{}
copy(info.Name[:], utunControlName)
if err = unix.IoctlCtlInfo(fd, info); err != nil {
log.Fatalf("CTLIOCGINFO: %v", err)
}
if err = unix.Connect(fd, &unix.SockaddrCtl{ID: info.Id, Unit: 0}); err != nil {
log.Fatalf("connect utun control: %v", err)
}
name, err := unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, utunOptIfname)
if err != nil {
log.Fatalf("utun name: %v", err)
}
for _, args := range [][]string{
{name, "inet", addr4, addr4, "netmask", "255.255.255.255", "up"},
{name, "inet6", addr6, "prefixlen", "128"},
} {
if out, err := exec.Command("ifconfig", args...).CombinedOutput(); err != nil {
log.Fatalf("ifconfig %s: %v: %s", strings.Join(args, " "), err, out)
}
}
return fd, name
}
// lanPrefixes applies Tailscale's attached-LAN rule to every interface but those in skip
func lanPrefixes(skip []string) (v4, v6 []netip.Prefix) {
ifs, err := net.Interfaces()
if err != nil {
log.Fatal(err)
}
for _, i := range ifs {
if slices.Contains(skip, i.Name) || i.Flags&net.FlagLoopback != 0 {
continue
}
addrs, _ := i.Addrs()
for _, a := range addrs {
n, ok := a.(*net.IPNet)
if !ok {
continue
}
ip, _ := netip.AddrFromSlice(n.IP)
ip = ip.Unmap()
ones, _ := n.Mask.Size()
pfx := netip.PrefixFrom(ip, ones).Masked()
// Tailscale skips single IPs. Link-local is our addition, every interface has the
// same fe80::/64 and claiming it would only break the harness's own host.
if pfx.IsSingleIP() || ip.IsLinkLocalUnicast() {
continue
}
if ip.Is4() {
v4 = append(v4, pfx)
} else {
v6 = append(v6, pfx)
}
}
}
slices.SortFunc(v4, func(a, b netip.Prefix) int { return a.Addr().Compare(b.Addr()) })
slices.SortFunc(v6, func(a, b netip.Prefix) int { return a.Addr().Compare(b.Addr()) })
return v4, v6
}
func routes4(ps []netip.Prefix) string {
var b strings.Builder
for _, p := range ps {
mask := net.IP(net.CIDRMask(p.Bits(), 32)).String()
fmt.Fprintf(&b, "<dict><key>DestinationAddress</key><string>%s</string><key>SubnetMask</key><string>%s</string></dict>", p.Addr(), mask)
}
return b.String()
}
func routes6(ps []netip.Prefix) string {
var b strings.Builder
for _, p := range ps {
fmt.Fprintf(&b, "<dict><key>DestinationAddress</key><string>%s</string><key>PrefixLength</key><integer>%d</integer></dict>", p.Addr(), p.Bits())
}
return b.String()
}
func (s *store) publish(ifName string, exclude bool, v4, v6 []netip.Prefix) {
routesKey, primary := "AdditionalRoutes", ""
if exclude {
// configd only steers excluded routes for a service with a default route that is primary.
// While this runs, everything not on a kept interface's subnet goes into the utun and dies.
routesKey = "ExcludedRoutes"
primary = "<key>Router</key><string>" + addr4 + "</string><key>OverridePrimary</key><integer>1</integer>"
}
const head = `<?xml version="1.0" encoding="UTF-8"?><plist version="1.0"><dict>`
s.set(s.k4, head+
"<key>InterfaceName</key><string>"+ifName+"</string>"+
"<key>Addresses</key><array><string>"+addr4+"</string></array>"+
"<key>SubnetMasks</key><array><string>255.255.255.255</string></array>"+primary+
"<key>"+routesKey+"</key><array>"+routes4(v4)+"</array></dict></plist>")
s.set(s.k6, head+
"<key>InterfaceName</key><string>"+ifName+"</string>"+
"<key>Addresses</key><array><string>"+addr6+"</string></array>"+
"<key>PrefixLength</key><array><integer>128</integer></array>"+
"<key>"+routesKey+"</key><array>"+routes6(v6)+"</array></dict></plist>")
log.Printf("published %s on %s, %s: %v %v", serviceID, ifName, routesKey, v4, v6)
}
func main() {
mode := flag.String("mode", "claim", "claim: exit node without LAN access, the prefixes are routed into the vpn\n"+
"exclude: exit node with LAN access, the prefixes leave via the next interface and the vpn becomes primary")
keep := flag.String("keep", "en0", "comma separated interfaces whose prefixes are never published, keep the one your ssh session uses")
follow := flag.Bool("follow", true, "republish when interface prefixes change, as Tailscale does on link changes.\n"+
"Off, a route stays published after its interface is gone, which is how a leftover route is staged")
clear := flag.Bool("clear", false, "withdraw a service a crashed run left behind, then exit")
flag.Parse()
if *mode != "claim" && *mode != "exclude" {
log.Fatalf("unknown -mode %q", *mode)
}
if os.Geteuid() != 0 {
log.Fatal("needs root, to create the utun and write configd's dynamic store")
}
loadFrameworks()
s := openStore()
if *clear {
s.withdraw()
log.Print("withdrawn")
return
}
fd, ifName := createUtun()
exclude := *mode == "exclude"
skip := append(strings.Split(*keep, ","), ifName)
v4, v6 := lanPrefixes(skip)
s.publish(ifName, exclude, v4, v6)
sig := make(chan os.Signal, 1)
signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
tick := time.NewTicker(time.Second)
defer tick.Stop()
for {
select {
case <-sig:
s.withdraw()
_ = unix.Close(fd)
log.Print("withdrawn")
return
case <-tick.C:
if !*follow {
continue
}
n4, n6 := lanPrefixes(skip)
if !slices.Equal(n4, v4) || !slices.Equal(n6, v6) {
v4, v6 = n4, n6
s.publish(ifName, exclude, v4, v6)
}
}
}
}
+76
View File
@@ -0,0 +1,76 @@
#!/bin/sh
# Reports where traffic to nebula peers goes before, during and after a vpnemu exit node.
#
# usage: sudo ./repro.sh -n <nebula binary> -c <nebula config> -p <peer vpn addr> [-p <peer>...]
# [-m claim|exclude] [-k <keep interfaces>]
#
# Each peer should answer pings over the overlay. A healthy run shows the nebula utun as egress
# and the ping working in all three phases.
set -u
MODE=claim
KEEP=en0
PEERS=""
NEBULA=""
CONFIG=""
while getopts n:c:p:m:k: o; do
case $o in
n) NEBULA=$OPTARG ;;
c) CONFIG=$OPTARG ;;
p) PEERS="$PEERS $OPTARG" ;;
m) MODE=$OPTARG ;;
k) KEEP=$OPTARG ;;
*) exit 2 ;;
esac
done
if [ -z "$NEBULA" ] || [ -z "$CONFIG" ] || [ -z "$PEERS" ]; then
sed -n '4,5p' "$0"
exit 2
fi
HERE=$(cd "$(dirname "$0")" && pwd)
VPNEMU=${VPNEMU:-$HERE/vpnemu}
LOG=$(mktemp -t nebula-repro)
report() {
for p in $PEERS; do
case $p in
*:*) fam=-inet6 ping=ping6 ;;
*) fam=-inet ping="ping -t 3" ;;
esac
egress=$(route -n get $fam "$p" 2>/dev/null | awk '/interface:/{print $2}')
if $ping -c 2 "$p" >/dev/null 2>&1; then ok=ok; else ok=FAIL; fi
printf '%-8s %-24s egress=%-8s ping=%s\n' "$1" "$p" "${egress:-none}" "$ok"
done
}
cleanup() {
[ -n "${EMU:-}" ] && kill "$EMU" 2>/dev/null
[ -n "${NEB:-}" ] && kill "$NEB" 2>/dev/null
wait 2>/dev/null
}
trap cleanup EXIT INT TERM
"$NEBULA" -config "$CONFIG" >"$LOG" 2>&1 &
NEB=$!
sleep 3
if ! kill -0 $NEB 2>/dev/null; then
echo "nebula exited:"
tail -5 "$LOG"
exit 1
fi
echo "nebula on $(grep -o 'interface=utun[0-9]*' "$LOG" | head -1 | cut -d= -f2), log in $LOG"
report before
"$VPNEMU" -mode "$MODE" -keep "$KEEP" &
EMU=$!
sleep 3
report during
kill $EMU
wait $EMU 2>/dev/null
EMU=
sleep 2
report after
grep -E 'level=(WARN|ERROR).*(route|Route)' "$LOG"
+4 -4
View File
@@ -16,7 +16,7 @@ require (
github.com/miekg/dns v1.1.73
github.com/miekg/pkcs11 v1.1.2
github.com/nbrownus/go-metrics-prometheus v0.0.0-20210712211119-974a6260965f
github.com/prometheus/client_golang v1.25.0
github.com/prometheus/client_golang v1.24.1
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
@@ -41,9 +41,9 @@ require (
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/google/btree v1.1.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/prometheus/client_model v0.6.3 // indirect
github.com/prometheus/common v0.72.0 // indirect
github.com/prometheus/procfs v0.22.0 // indirect
github.com/prometheus/client_model v0.6.2 // indirect
github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect
github.com/vishvananda/netns v0.0.5 // indirect
golang.org/x/time v0.5.0 // indirect
)
+10 -10
View File
@@ -69,8 +69,8 @@ github.com/kardianos/service v1.3.0 h1:/LGy+xPP2TM+GLTiCZ2di7cy0Jd/qrawlTUfqKYFd
github.com/kardianos/service v1.3.0/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.20.1 h1:T7kKElXUMXrUJ2E9QhQhxFtcK5rPyLdsGZvdbLMPdiQ=
github.com/klauspost/compress v1.20.1/go.mod h1:LUdAzn7YLVvxLpc7y3V1m40wESHTgc1422pwwBSKYuI=
github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk=
github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
@@ -105,24 +105,24 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.25.0 h1:cDtW1TcFsMi1RJc8khwCKB4VhzwV04aKqbYtNOXuiK0=
github.com/prometheus/client_golang v1.25.0/go.mod h1:j2X/4/kaQHLPnmyEU8zjmkndo1T317VpCIQEml8LTKk=
github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU=
github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/prometheus/client_model v0.6.3 h1:O0jaTVAYNxTHYInEPFJt5I3+sN8zqBtVMPTB1qyxiEo=
github.com/prometheus/client_model v0.6.3/go.mod h1:gpN5P9S7Rr6Yr92PiQ+Ixvhf6JZEkF1dnxsYL2aPBEM=
github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc=
github.com/prometheus/common v0.72.0 h1:tAYsE+sPJxIncDAobm4H5aQjmox9ZxEIIqPbiffa8G4=
github.com/prometheus/common v0.72.0/go.mod h1:77NWqAQ2tXT7BIK40qjJdw5Acrsrg1TlHAnsQi3i6mk=
github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY=
github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc=
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
github.com/prometheus/procfs v0.22.0 h1:6q9+/JL9IKAPbCmBrv9n5O5Ty3NKnciV5X7YGw0oics=
github.com/prometheus/procfs v0.22.0/go.mod h1:CvmFr/GVhIjIvWJZW3tgkODBQMRIf0EyWMQLHCHab58=
github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI=
github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY=
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5XpJzTSTfLsJV/mx9Q9g7kxmchpfZyxgzM=
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+14 -98
View File
@@ -6,10 +6,8 @@ import (
"errors"
"fmt"
"log/slog"
"math/bits"
"net/netip"
"os"
"sync"
"sync/atomic"
"syscall"
"unsafe"
@@ -30,7 +28,7 @@ type tun struct {
DefaultMTU int
Routes atomic.Pointer[[]Route]
routeTree atomic.Pointer[bart.Table[routing.Gateways]]
linkAddr atomic.Pointer[netroute.LinkAddr]
linkAddr *netroute.LinkAddr
// hostOwned means the fd arrived from the OS, which has already configured addressing, mtu
// and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool
@@ -245,7 +243,7 @@ func (t *tun) Activate() error {
if linkAddr == nil {
return fmt.Errorf("unable to discover link_addr for tun interface")
}
t.linkAddr.Store(linkAddr)
t.linkAddr = linkAddr
for _, network := range t.vpnNetworks {
if network.Addr().Is4() {
@@ -271,7 +269,6 @@ func (t *tun) Activate() error {
return t.addRoutes(false)
}
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
func (t *tun) activate4(network netip.Prefix) error {
s, err := unix.Socket(
unix.AF_INET,
@@ -298,32 +295,22 @@ func (t *tun) activate4(network netip.Prefix) error {
MaskAddr: unix.RawSockaddrInet4{
Len: unix.SizeofSockaddrInet4,
Family: unix.AF_INET,
Addr: [4]byte{255, 255, 255, 255}, //always make the address a /32
Addr: prefixToMask(network).As4(),
},
}
// On Darwin, VPNs don't edit the routing table directly.
// A network extension hands its routes to the system, and configd installs them.
// configd replaces any route already holding the same prefix, including ours.
// As an example, Tailscale with an exit node treats any interface prefix wider than one address as a LAN.
// With LAN access off, it routes each LAN into Tailscale's utun.
// With LAN access on, it excludes each LAN from its tunnel.
// configd routes an excluded prefix like its own routes would, ignoring ours, so it goes out the default route.
// Either way our route is replaced, and traffic for the vpn network no longer reaches nebula.
// A single address isn't a LAN to Tailscale, so our route is left alone.
// We route the vpn network to the tun explicitly instead.
if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun v4 address: %s", err)
}
if err := addRoute(network, t.linkAddr.Load()); err != nil {
return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err)
err = addRoute(network, t.linkAddr)
if err != nil {
return err
}
return nil
}
// activate6 is activate4 for IPv6
func (t *tun) activate6(network netip.Prefix) error {
s, err := unix.Socket(
unix.AF_INET6,
@@ -345,24 +332,20 @@ func (t *tun) activate6(network netip.Prefix) error {
PrefixMask: unix.RawSockaddrInet6{
Len: unix.SizeofSockaddrInet6,
Family: unix.AF_INET6,
// Always set the netmask to be a /128. See activate4 for why we do this.
Addr: [16]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
Addr: prefixToMask(network).As16(),
},
Lifetime: addrLifetime{
// never expires
Vltime: 0xffffffff,
Pltime: 0xffffffff,
},
Flags: _IN6_IFF_NODAD, // No DAD, a utun has no neighbors.
Flags: _IN6_IFF_NODAD,
}
if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun address: %s", err)
}
if err := addRoute(network, t.linkAddr.Load()); err != nil {
return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err)
}
return nil
}
@@ -411,56 +394,6 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
return routing.Gateways{}
}
func fetchRoutes() ([]netroute.Message, error) {
rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_DUMP, 0)
if err != nil {
return nil, err
}
return netroute.ParseRIB(unix.NET_RT_DUMP, rib)
}
// routeInterfaceIn returns the index of the interface prefix is routed through, or 0 if msgs has no route for exactly
// that prefix. Routes scoped to an interface are skipped, they don't stop us adding ours.
func routeInterfaceIn(msgs []netroute.Message, prefix netip.Prefix) int {
for _, msg := range msgs {
m, ok := msg.(*netroute.RouteMessage)
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK || m.Flags&unix.RTF_IFSCOPE != 0 {
continue
}
if p, ok := routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK]); ok && p == prefix.Masked() {
return m.Index
}
}
return 0
}
// routePrefix returns the prefix a route covers. Host routes have no netmask.
func routePrefix(dst, mask netroute.Addr) (netip.Prefix, bool) {
switch d := dst.(type) {
case *netroute.Inet4Addr:
bits := 32
if m, ok := mask.(*netroute.Inet4Addr); ok {
bits = maskBits(m.IP[:])
}
return netip.PrefixFrom(netip.AddrFrom4(d.IP), bits), true
case *netroute.Inet6Addr:
bits := 128
if m, ok := mask.(*netroute.Inet6Addr); ok {
bits = maskBits(m.IP[:])
}
return netip.PrefixFrom(netip.AddrFrom16(d.IP), bits), true
}
return netip.Prefix{}, false
}
func maskBits(mask []byte) int {
n := 0
for _, b := range mask {
n += bits.OnesCount8(b)
}
return n
}
// Get the LinkAddr for the interface of the given name
// Is there an easier way to fetch this when we create the interface?
// Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers.
@@ -495,14 +428,7 @@ func (t *tun) addRoutes(logErrors bool) error {
return nil
}
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil {
return nil
}
routes := *t.Routes.Load()
systemRoutes := sync.OnceValues(fetchRoutes)
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
@@ -510,16 +436,10 @@ func (t *tun) addRoutes(logErrors bool) error {
continue
}
err := addRoute(r.Cidr, linkAddr)
err := addRoute(r.Cidr, t.linkAddr)
if err != nil {
if errors.Is(err, unix.EEXIST) {
// Every route we added already exists on a reload, only warn about someone else's
msgs, err := systemRoutes()
if err != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
}
t.l.Warn("unable to add unsafe_route, identical route already exists", "route", r.Cidr)
} else {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors {
@@ -530,8 +450,6 @@ func (t *tun) addRoutes(logErrors bool) error {
}
} else {
t.l.Info("Added route", "route", r)
// A cidr listed twice would otherwise find our own route missing from an older fetch
systemRoutes = sync.OnceValues(fetchRoutes)
}
}
@@ -539,18 +457,16 @@ func (t *tun) addRoutes(logErrors bool) error {
}
func (t *tun) removeRoutes(routes []Route) error {
linkAddr := t.linkAddr.Load()
if t.hostOwned || linkAddr == nil {
if t.hostOwned {
return nil
}
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
if !r.Install {
continue
}
err := delRoute(r.Cidr, linkAddr)
err := delRoute(r.Cidr, t.linkAddr)
if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r)
} else {
-103
View File
@@ -1,103 +0,0 @@
//go:build darwin && !ios && !e2e_testing
package overlay
import (
"net"
"net/netip"
"os"
"os/exec"
"strings"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
netroute "golang.org/x/net/route"
"golang.org/x/sys/unix"
)
// Every mac routes loopback through lo0
func TestRouteInterfaceIn(t *testing.T) {
msgs, err := fetchRoutes()
require.NoError(t, err)
routeInterface := func(s string) int { return routeInterfaceIn(msgs, netip.MustParsePrefix(s)) }
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
assert.Equal(t, lo0.Index, routeInterface("127.0.0.0/8"))
assert.Equal(t, lo0.Index, routeInterface("127.0.0.1/32"), "a host route has no netmask")
assert.Equal(t, lo0.Index, routeInterface("::1/128"))
assert.Equal(t, 0, routeInterface("127.0.0.0/16"), "only an exact prefix counts")
// Most macs also list the default route scoped to their other interfaces
out, err := exec.Command("route", "-n", "get", "default").Output()
require.NoError(t, err)
var name string
for _, line := range strings.Split(string(out), "\n") {
if v, ok := strings.CutPrefix(strings.TrimSpace(line), "interface: "); ok {
name = v
}
}
if name == "" {
t.Skip("no default route")
}
def, err := net.InterfaceByName(name)
require.NoError(t, err)
assert.Equal(t, def.Index, routeInterface("0.0.0.0/0"))
}
func TestRouteInterfaceIn_SkipsScopedCopies(t *testing.T) {
route := func(index, flags int) netroute.Message {
addrs := make([]netroute.Addr, unix.RTAX_MAX)
addrs[unix.RTAX_DST] = &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}}
addrs[unix.RTAX_NETMASK] = &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}}
return &netroute.RouteMessage{Index: index, Flags: flags, Addrs: addrs}
}
p := netip.MustParsePrefix("192.168.1.0/24")
assert.Equal(t, 7, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE), route(7, unix.RTF_UP)}, p))
assert.Equal(t, 0, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE)}, p))
}
// A reload can happen before Activate
func TestTunRoutesBeforeActivate(t *testing.T) {
tn := &tun{l: test.NewLogger()}
routes := []Route{{
Cidr: netip.MustParsePrefix("10.253.1.0/24"),
Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)},
Install: true,
}}
tn.Routes.Store(&routes)
assert.NoError(t, tn.addRoutes(true))
assert.NoError(t, tn.removeRoutes(routes))
}
// A tun.routes entry with no via is never installed, so removing it must not delete a route for the same cidr that
// belongs to someone else
func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating a utun and adding routes needs root")
}
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
cidr := "10.250.0.128/25"
require.NoError(t, exec.Command("route", "-q", "-n", "add", "-net", cidr, "-interface", "lo0").Run())
t.Cleanup(func() { _ = exec.Command("route", "-q", "-n", "delete", "-net", cidr).Run() })
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("tun:\n routes:\n - route: "+cidr+"\n mtu: 1300\n"))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
require.NoError(t, c.ReloadConfigString("tun: {}\n"))
msgs, err := fetchRoutes()
require.NoError(t, err)
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
}
+1 -2
View File
@@ -589,8 +589,7 @@ func (t *tun) addRoutes(logErrors bool) error {
func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
if !r.Install {
continue
}
+1 -2
View File
@@ -420,8 +420,7 @@ func (t *tun) addRoutes(logErrors bool) error {
func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
if !r.Install {
continue
}
+1 -2
View File
@@ -399,8 +399,7 @@ func (t *tun) addRoutes(logErrors bool) error {
func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes {
if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
if !r.Install {
continue
}
+13 -33
View File
@@ -60,20 +60,6 @@ type sshDeviceInfoFlags struct {
func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
c.RegisterReloadCallback(func(c *config.C) {
if c.GetBool("sshd.enabled", false) {
// Restarting the server drops every open session, the one that asked for this reload included. Only restart
// when it isn't running or its config changed. The host key file can change under the same config, new
// handshakes use the new key without a restart
if ssh.Running() && !c.HasChanged("sshd") {
key, err := loadSSHHostKey(c)
if err == nil {
err = ssh.SetHostKey(key)
}
if err != nil {
l.Error("Failed to reconfigure the sshd", "error", err)
ssh.Stop()
}
return
}
sshRun, err := configSSH(l, ssh, c)
if err != nil {
l.Error("Failed to reconfigure the sshd", "error", err)
@@ -88,22 +74,6 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
})
}
// loadSSHHostKey returns sshd.host_key, read from its file when it names one.
func loadSSHHostKey(c *config.C) ([]byte, error) {
hostKeyPathOrKey := c.GetString("sshd.host_key", "")
if hostKeyPathOrKey == "" {
return nil, fmt.Errorf("sshd.host_key must be provided")
}
if strings.Contains(hostKeyPathOrKey, "-----BEGIN") {
return []byte(hostKeyPathOrKey), nil
}
b, err := os.ReadFile(hostKeyPathOrKey)
if err != nil {
return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err)
}
return b, nil
}
// configSSH reads the ssh info out of the passed-in Config and
// updates the passed-in SSHServer. On success, it returns a function
// that callers may invoke to run the configured ssh server. On
@@ -122,9 +92,19 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
return nil, fmt.Errorf("sshd.listen can not use port 22")
}
hostKeyBytes, err := loadSSHHostKey(c)
if err != nil {
return nil, err
hostKeyPathOrKey := c.GetString("sshd.host_key", "")
if hostKeyPathOrKey == "" {
return nil, fmt.Errorf("sshd.host_key must be provided")
}
var hostKeyBytes []byte
if strings.Contains(hostKeyPathOrKey, "-----BEGIN") {
hostKeyBytes = []byte(hostKeyPathOrKey)
} else {
hostKeyBytes, err = os.ReadFile(hostKeyPathOrKey)
if err != nil {
return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err)
}
}
err = ssh.SetHostKey(hostKeyBytes)
+13 -48
View File
@@ -8,18 +8,14 @@ import (
"log/slog"
"net"
"sync"
"sync/atomic"
"github.com/armon/go-radix"
"golang.org/x/crypto/ssh"
)
type SSHServer struct {
// config is replaced whole when the host key changes, ssh.ServerConfig.AddHostKey only replaces a key of the same
// algorithm and appends any other
config atomic.Pointer[ssh.ServerConfig]
authenticate func(ssh.ConnMetadata, ssh.PublicKey) (*ssh.Permissions, error)
l *slog.Logger
config *ssh.ServerConfig
l *slog.Logger
certChecker *ssh.CertChecker
@@ -32,12 +28,7 @@ type SSHServer struct {
// List of available commands
helpCommand *Command
commands *radix.Tree
listener atomic.Pointer[net.Listener]
// runs counts the Runs that are listening. A reload's Run can start before the one it replaced has returned
runs atomic.Int32
// addr is where the newest Run listens, nil when none is
addr atomic.Pointer[net.Addr]
listener net.Listener
// ctx parents per-Run contexts. Cancelling it (e.g. via Control.Stop) tears the server down even
// across reloads, since each Run derives a fresh child rather than reusing this one directly.
@@ -94,8 +85,10 @@ func NewSSHServer(ctx context.Context, l *slog.Logger) (*SSHServer, error) {
},
}
s.authenticate = cc.Authenticate
s.config.Store(s.newConfig())
s.config = &ssh.ServerConfig{
PublicKeyCallback: cc.Authenticate,
ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"),
}
s.RegisterCommand(&Command{
Name: "help",
@@ -114,32 +107,10 @@ func (s *SSHServer) SetHostKey(hostPrivateKey []byte) error {
return fmt.Errorf("failed to parse private key: %s", err)
}
config := s.newConfig()
config.AddHostKey(private)
s.config.Store(config)
s.config.AddHostKey(private)
return nil
}
func (s *SSHServer) newConfig() *ssh.ServerConfig {
return &ssh.ServerConfig{
PublicKeyCallback: s.authenticate,
ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"),
}
}
// Addr is the address the newest Run listens on, nil when none is.
func (s *SSHServer) Addr() net.Addr {
if a := s.addr.Load(); a != nil {
return *a
}
return nil
}
// Running is true while a Run is listening.
func (s *SSHServer) Running() bool {
return s.runs.Load() > 0
}
func (s *SSHServer) ClearTrustedCAs() {
s.authLock.Lock()
s.trustedCAs = []ssh.PublicKey{}
@@ -210,16 +181,10 @@ func (s *SSHServer) Run(addr string) error {
// this run owns. They start equal but a fast reload may overwrite s.listener with the next run's
// listener before this run's watcher fires, so each run must close its own listener via the local
// reference.
s.listener.Store(&listener)
s.runs.Add(1)
bound := listener.Addr()
s.addr.Store(&bound)
s.listener = listener
runCtx, cancel := context.WithCancel(s.ctx)
defer cancel()
// Deferred after cancel so they run first, a run on its way out no longer counts as listening
defer s.runs.Add(-1)
defer s.addr.CompareAndSwap(&bound, nil)
// Close the listener when this run's context is cancelled. That can come from the parent
// (Control.Stop), from Run returning normally (defer cancel above), or transitively when a sibling
@@ -232,7 +197,7 @@ func (s *SSHServer) Run(addr string) error {
}
}()
s.l.Info("SSH server is listening", "sshListener", bound)
s.l.Info("SSH server is listening", "sshListener", addr)
// Run loops until there is an error
s.run(runCtx, listener)
@@ -261,7 +226,7 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
<-sessionContext.Done()
c.Close()
}()
conn, chans, reqs, err := ssh.NewServerConn(c, s.config.Load())
conn, chans, reqs, err := ssh.NewServerConn(c, s.config)
fp := ""
if conn != nil {
fp = conn.Permissions.Extensions["fp"]
@@ -299,8 +264,8 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
}
func (s *SSHServer) Stop() {
if l := s.listener.Load(); l != nil {
if err := (*l).Close(); err != nil && !errors.Is(err, net.ErrClosed) {
if s.listener != nil {
if err := s.listener.Close(); err != nil {
s.l.Warn("Failed to close the sshd listener", "error", err)
}
}
-196
View File
@@ -1,196 +0,0 @@
package nebula
import (
"crypto"
"crypto/ed25519"
"crypto/rand"
"crypto/rsa"
"encoding/pem"
"fmt"
"net"
"os"
"path/filepath"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/sshd"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/crypto/ssh"
)
type sshdReloadHarness struct {
c *config.C
server *sshd.SSHServer
keyFile string
signer ssh.Signer
extra string
}
func newSSHDReloadHarness(t *testing.T) *sshdReloadHarness {
t.Helper()
l := test.NewLogger()
h := &sshdReloadHarness{keyFile: filepath.Join(t.TempDir(), "host_key")}
h.writeHostKey(t, newEd25519Key(t))
_, userKey, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
h.signer, err = ssh.NewSignerFromKey(userKey)
require.NoError(t, err)
h.c = config.NewC(l)
require.NoError(t, h.c.LoadString(h.config()))
h.server, err = sshd.NewSSHServer(t.Context(), l)
require.NoError(t, err)
wireSSHReload(l, h.server, h.c)
run, err := configSSH(l, h.server, h.c)
require.NoError(t, err)
go run()
t.Cleanup(h.server.Stop)
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
return h
}
func newEd25519Key(t *testing.T) crypto.Signer {
_, key, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
return key
}
// writeHostKey replaces the host key file and returns the public key a client should now see
func (h *sshdReloadHarness) writeHostKey(t *testing.T, key crypto.Signer) ssh.PublicKey {
t.Helper()
block, err := ssh.MarshalPrivateKey(key, "")
require.NoError(t, err)
require.NoError(t, os.WriteFile(h.keyFile, pem.EncodeToMemory(block), 0o600))
pub, err := ssh.NewPublicKey(key.Public())
require.NoError(t, err)
return pub
}
func (h *sshdReloadHarness) config() string {
return fmt.Sprintf("sshd:\n enabled: true\n listen: 127.0.0.1:0\n host_key: %s\n authorized_users:\n - user: test\n keys: [%q]\n%s",
h.keyFile, string(ssh.MarshalAuthorizedKey(h.signer.PublicKey())), h.extra)
}
// dial connects to wherever the server listens now, a restart on :0 moves it. hostKeyAlgorithms limits which host keys
// the client accepts, none means any
func (h *sshdReloadHarness) dial(hostKeyAlgorithms ...string) (*ssh.Client, ssh.PublicKey, error) {
addr := h.server.Addr()
if addr == nil {
return nil, nil, fmt.Errorf("not listening")
}
var hostKey ssh.PublicKey
c, err := ssh.Dial("tcp", addr.String(), &ssh.ClientConfig{
User: "test",
Auth: []ssh.AuthMethod{ssh.PublicKeys(h.signer)},
HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error {
hostKey = key
return nil
},
HostKeyAlgorithms: hostKeyAlgorithms,
Timeout: time.Second,
})
return c, hostKey, err
}
func (h *sshdReloadHarness) connect(t *testing.T) (*ssh.Client, ssh.PublicKey) {
t.Helper()
var client *ssh.Client
var hostKey ssh.PublicKey
require.Eventually(t, func() bool {
var err error
client, hostKey, err = h.dial()
return err == nil
}, 5*time.Second, 20*time.Millisecond)
t.Cleanup(func() { _ = client.Close() })
return client, hostKey
}
// dropped is true when the server closed the client's connection within wait
func dropped(client *ssh.Client, wait time.Duration) bool {
done := make(chan struct{})
go func() {
_ = client.Wait()
close(done)
}()
select {
case <-done:
return true
case <-time.After(wait):
return false
}
}
// Restarting sshd drops every open session, the one that asked for the reload included. A reload that touches nothing
// sshd reads leaves it alone
func TestSSHDReload_UnchangedKeepsSessions(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
require.NoError(t, h.c.ReloadConfigString(h.config()+"punchy:\n punch: true\n"))
assert.False(t, dropped(client, 500*time.Millisecond), "an unchanged sshd was restarted")
}
func TestSSHDReload_ChangedConfigRestarts(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
h.extra = " trusted_cas: []\n"
require.NoError(t, h.c.ReloadConfigString(h.config()))
assert.True(t, dropped(client, 5*time.Second), "a changed sshd config was not applied")
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t)
}
// The config can stay the same while the host key file it names changes. New connections get the new key, open
// sessions are left alone
func TestSSHDReload_ChangedHostKeyFile(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
want := h.writeHostKey(t, newEd25519Key(t))
require.NoError(t, h.c.ReloadConfigString(h.config()))
_, got := h.connect(t)
assert.Equal(t, want.Marshal(), got.Marshal(), "a new host key was not applied")
assert.False(t, dropped(client, 500*time.Millisecond), "a new host key restarted sshd")
}
// A host key of another algorithm replaces the old one rather than being offered next to it
func TestSSHDReload_HostKeyAlgorithmChange(t *testing.T) {
h := newSSHDReloadHarness(t)
h.connect(t)
rsaKey, err := rsa.GenerateKey(rand.Reader, 2048)
require.NoError(t, err)
want := h.writeHostKey(t, rsaKey)
require.NoError(t, h.c.ReloadConfigString(h.config()))
_, got := h.connect(t)
assert.Equal(t, want.Marshal(), got.Marshal())
_, _, err = h.dial(ssh.KeyAlgoED25519)
assert.Error(t, err, "the old ed25519 host key is still offered")
}
// A server that isn't running, say its port was busy, is tried again on every reload as before
func TestSSHDReload_NotRunningRestarts(t *testing.T) {
h := newSSHDReloadHarness(t)
h.server.Stop()
require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond)
require.NoError(t, h.c.ReloadConfigString(h.config()))
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t)
}
// A host key that no longer loads stops sshd, the same as a config that doesn't
func TestSSHDReload_BadHostKeyFileStops(t *testing.T) {
h := newSSHDReloadHarness(t)
h.connect(t)
require.NoError(t, os.WriteFile(h.keyFile, []byte("not a key"), 0o600))
require.NoError(t, h.c.ReloadConfigString(h.config()))
require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond)
}