mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-16 18:56:52 +01:00
Compare commits
6
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
5bcceac53c | ||
|
|
afbdc87ed9 | ||
|
|
94372f4f51 | ||
|
|
42280f04ce | ||
|
|
11c1917387 | ||
|
|
5ba7aff4b3 |
No files matched your search
@@ -14,7 +14,7 @@ jobs:
|
|||||||
|
|
||||||
- uses: actions/setup-go@v7
|
- uses: actions/setup-go@v7
|
||||||
with:
|
with:
|
||||||
go-version: '1.26'
|
go-version: '1.27'
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
|
||||||
- name: Build
|
- name: Build
|
||||||
@@ -40,7 +40,7 @@ jobs:
|
|||||||
|
|
||||||
- uses: actions/setup-go@v7
|
- uses: actions/setup-go@v7
|
||||||
with:
|
with:
|
||||||
go-version: '1.26'
|
go-version: '1.27'
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
|
||||||
- name: Build
|
- name: Build
|
||||||
@@ -83,7 +83,7 @@ jobs:
|
|||||||
|
|
||||||
- uses: actions/setup-go@v7
|
- uses: actions/setup-go@v7
|
||||||
with:
|
with:
|
||||||
go-version: '1.26'
|
go-version: '1.27'
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
|
||||||
# GitHub holds ARNs, not credentials, and ARNs outlive a rotation
|
# GitHub holds ARNs, not credentials, and ARNs outlive a rotation
|
||||||
|
|||||||
@@ -34,7 +34,7 @@ jobs:
|
|||||||
|
|
||||||
- uses: actions/setup-go@v7
|
- uses: actions/setup-go@v7
|
||||||
with:
|
with:
|
||||||
go-version: '1.26'
|
go-version: '1.27'
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
|
||||||
- name: add hashicorp source
|
- name: add hashicorp source
|
||||||
@@ -66,7 +66,7 @@ jobs:
|
|||||||
|
|
||||||
- uses: actions/setup-go@v7
|
- uses: actions/setup-go@v7
|
||||||
with:
|
with:
|
||||||
go-version: '1.26'
|
go-version: '1.27'
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
|
||||||
- name: add hashicorp source
|
- name: add hashicorp source
|
||||||
@@ -92,7 +92,7 @@ jobs:
|
|||||||
|
|
||||||
- uses: actions/setup-go@v7
|
- uses: actions/setup-go@v7
|
||||||
with:
|
with:
|
||||||
go-version: '1.26'
|
go-version: '1.27'
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
|
||||||
# WSL2 + Ubuntu so the smoke can run a real linux peer with its own
|
# WSL2 + Ubuntu so the smoke can run a real linux peer with its own
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ jobs:
|
|||||||
|
|
||||||
- uses: actions/setup-go@v7
|
- uses: actions/setup-go@v7
|
||||||
with:
|
with:
|
||||||
go-version: '1.26'
|
go-version: '1.27'
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
|
||||||
- name: Smoke Docker
|
- name: Smoke Docker
|
||||||
@@ -51,7 +51,7 @@ jobs:
|
|||||||
|
|
||||||
- uses: actions/setup-go@v7
|
- uses: actions/setup-go@v7
|
||||||
with:
|
with:
|
||||||
go-version: '1.26'
|
go-version: '1.27'
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
|
||||||
- name: build
|
- name: build
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ jobs:
|
|||||||
|
|
||||||
- uses: actions/setup-go@v7
|
- uses: actions/setup-go@v7
|
||||||
with:
|
with:
|
||||||
go-version: '1.26'
|
go-version: '1.27'
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
|
||||||
- name: Install goimports
|
- name: Install goimports
|
||||||
@@ -42,7 +42,7 @@ jobs:
|
|||||||
- name: golangci-lint
|
- name: golangci-lint
|
||||||
uses: golangci/golangci-lint-action@v9
|
uses: golangci/golangci-lint-action@v9
|
||||||
with:
|
with:
|
||||||
version: v2.12
|
version: v2.14
|
||||||
|
|
||||||
test:
|
test:
|
||||||
name: Test ${{ matrix.name }}
|
name: Test ${{ matrix.name }}
|
||||||
@@ -87,7 +87,7 @@ jobs:
|
|||||||
|
|
||||||
- uses: actions/setup-go@v7
|
- uses: actions/setup-go@v7
|
||||||
with:
|
with:
|
||||||
go-version: '1.26'
|
go-version: '1.27'
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
|
||||||
- name: Build
|
- name: Build
|
||||||
@@ -132,7 +132,7 @@ jobs:
|
|||||||
|
|
||||||
- uses: actions/setup-go@v7
|
- uses: actions/setup-go@v7
|
||||||
with:
|
with:
|
||||||
go-version: '1.26'
|
go-version: '1.27'
|
||||||
check-latest: true
|
check-latest: true
|
||||||
|
|
||||||
- name: Build ${{ matrix.name }}
|
- name: Build ${{ matrix.name }}
|
||||||
|
|||||||
@@ -1,68 +0,0 @@
|
|||||||
# vpnemu
|
|
||||||
|
|
||||||
Reproduces, on macOS and without Tailscale, the way a VPN with an exit node takes nebula's vpn network route
|
|
||||||
away from its utun (#1931).
|
|
||||||
|
|
||||||
A macOS network extension never touches the routing table. Its routes are published in configd's dynamic store
|
|
||||||
under `State:/Network/Service/<id>/IPv4` and `/IPv6`, and configd's IPMonitor installs them. vpnemu creates a
|
|
||||||
utun and publishes those keys itself, so configd does exactly what it would do for the real app. It publishes
|
|
||||||
the routes Tailscale would with an exit node: every interface prefix that isn't a single IP, the rule in
|
|
||||||
Tailscale's `ipnlocal.internalAndExternalInterfacesFrom`.
|
|
||||||
|
|
||||||
| mode | Tailscale setting | published as | where nebula's prefix goes |
|
|
||||||
| --- | --- | --- | --- |
|
|
||||||
| `claim` | exit node, no LAN access | `AdditionalRoutes` | into vpnemu's utun |
|
|
||||||
| `exclude` | exit node, LAN access | `ExcludedRoutes`, service overrides primary | out the next interface (en0) |
|
|
||||||
|
|
||||||
configd answers `EEXIST` with a delete and an add, so whatever held the prefix loses it. When the service is
|
|
||||||
withdrawn, configd deletes its routes by prefix alone, whoever holds them by then.
|
|
||||||
|
|
||||||
## Build
|
|
||||||
|
|
||||||
It is its own module, so `purego` stays out of nebula's `go.mod`. purego reaches CoreFoundation and
|
|
||||||
SystemConfiguration without cgo, so it cross compiles and runs on a mac with no developer tools.
|
|
||||||
|
|
||||||
```sh
|
|
||||||
cd e2e/darwin/vpnemu
|
|
||||||
CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -o vpnemu .
|
|
||||||
# a cross compiled binary copied to a mac may need: codesign -s - -f vpnemu
|
|
||||||
```
|
|
||||||
|
|
||||||
## Run
|
|
||||||
|
|
||||||
```sh
|
|
||||||
sudo ./vpnemu -mode claim -keep en0 # runs until ctrl-c, then withdraws and closes its utun
|
|
||||||
sudo ./vpnemu -clear # withdraw a service a crashed run left behind
|
|
||||||
```
|
|
||||||
|
|
||||||
`-keep` names interfaces whose prefixes are never published. Keep the one your ssh session arrives on, or
|
|
||||||
`claim` takes your LAN into a utun nobody reads. `exclude` makes vpnemu primary, so anything off a kept
|
|
||||||
interface's subnet, internet included, goes into its utun and is dropped while it runs.
|
|
||||||
|
|
||||||
`-follow` (default on) republishes when interface prefixes change, as Tailscale does on link changes. With
|
|
||||||
`-follow=false` a route stays published after its interface is gone, which stages the route configd leaves
|
|
||||||
behind for a nebula that restarts before the VPN has caught up.
|
|
||||||
|
|
||||||
## A/B a nebula build
|
|
||||||
|
|
||||||
`repro.sh` starts nebula, then runs vpnemu, and reports egress and ping per peer before, during and after:
|
|
||||||
|
|
||||||
```sh
|
|
||||||
sudo ./repro.sh -n ./nebula -c config.yml -p 192.168.212.1 -p fd19:31::1 -m claim -k en0
|
|
||||||
```
|
|
||||||
|
|
||||||
Before #1931 (utun holds `/24` and `/64`):
|
|
||||||
|
|
||||||
```
|
|
||||||
during 192.168.212.1 egress=utun2 ping=FAIL
|
|
||||||
after 192.168.212.1 egress=en0 ping=FAIL
|
|
||||||
```
|
|
||||||
|
|
||||||
With #1931 (utun holds `/32` and `/128`, network routes added explicitly) every phase stays on nebula's utun.
|
|
||||||
|
|
||||||
### Leftover route
|
|
||||||
|
|
||||||
Run an old nebula, start `vpnemu -follow=false`, then restart onto a new nebula while vpnemu is still up. The
|
|
||||||
new nebula finds the vpn network routed to vpnemu's utun, replaces it and starts. Stop vpnemu and configd deletes
|
|
||||||
the prefix, taking nebula's route with it until nebula restarts. With real Tailscale this needs nebula to restart
|
|
||||||
before Tailscale notices the old utun is gone, a one time window when upgrading from a nebula without #1931.
|
|
||||||
@@ -1,8 +0,0 @@
|
|||||||
module github.com/slackhq/nebula/e2e/darwin/vpnemu
|
|
||||||
|
|
||||||
go 1.26.0
|
|
||||||
|
|
||||||
require (
|
|
||||||
github.com/ebitengine/purego v0.9.1
|
|
||||||
golang.org/x/sys v0.48.0
|
|
||||||
)
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
github.com/ebitengine/purego v0.9.1 h1:a/k2f2HQU3Pi399RPW1MOaZyhKJL9w/xFpKAg4q1s0A=
|
|
||||||
github.com/ebitengine/purego v0.9.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
|
||||||
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
|
|
||||||
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
|
|
||||||
@@ -1,292 +0,0 @@
|
|||||||
//go:build darwin
|
|
||||||
|
|
||||||
// vpnemu stands in for a macOS VPN app with an exit node in use, so route hijacks like the one
|
|
||||||
// Tailscale causes can be reproduced without Tailscale.
|
|
||||||
//
|
|
||||||
// A network extension does not touch the routing table itself. It hands its routes to
|
|
||||||
// nesessionmanager, which publishes them in configd's dynamic store under
|
|
||||||
// State:/Network/Service/<id>/IPv4 and /IPv6, and configd's IPMonitor installs them. vpnemu
|
|
||||||
// creates a utun and publishes the same keys directly, so configd does exactly what it would do
|
|
||||||
// for the real thing.
|
|
||||||
//
|
|
||||||
// Which routes it publishes follows Tailscale's rule for exit nodes
|
|
||||||
// (ipnlocal.internalAndExternalInterfacesFrom): every interface address prefix that is not a
|
|
||||||
// single IP counts as an attached LAN. With LAN access off those prefixes are claimed into the
|
|
||||||
// tunnel (-mode claim, published as AdditionalRoutes). With LAN access on they are excluded and
|
|
||||||
// leave through the next interface (-mode exclude, published as ExcludedRoutes on a service that
|
|
||||||
// overrides the primary).
|
|
||||||
//
|
|
||||||
// It runs until interrupted, then withdraws the service and closes its utun. Needs root.
|
|
||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"flag"
|
|
||||||
"fmt"
|
|
||||||
"log"
|
|
||||||
"net"
|
|
||||||
"net/netip"
|
|
||||||
"os"
|
|
||||||
"os/exec"
|
|
||||||
"os/signal"
|
|
||||||
"slices"
|
|
||||||
"strings"
|
|
||||||
"syscall"
|
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/ebitengine/purego"
|
|
||||||
"golang.org/x/sys/unix"
|
|
||||||
)
|
|
||||||
|
|
||||||
const (
|
|
||||||
serviceID = "VPNEMU"
|
|
||||||
// Tailscale's own addresses, single IPs, so the rule skips them like Tailscale skips its own
|
|
||||||
addr4 = "100.100.0.1"
|
|
||||||
addr6 = "fd7a:115c:a1e0::1"
|
|
||||||
|
|
||||||
utunControlName = "com.apple.net.utun_control"
|
|
||||||
utunOptIfname = 2
|
|
||||||
cfStringUTF8 = 0x08000100
|
|
||||||
)
|
|
||||||
|
|
||||||
var (
|
|
||||||
cfStringCreateWithCString func(alloc uintptr, s *byte, enc uint32) uintptr
|
|
||||||
cfDataCreate func(alloc uintptr, b *byte, n int) uintptr
|
|
||||||
cfPropertyListCreateWithData func(alloc, data uintptr, opts uint, format *int, err *uintptr) uintptr
|
|
||||||
cfRelease func(ref uintptr)
|
|
||||||
scDynamicStoreCreate func(alloc, name, callout, ctx uintptr) uintptr
|
|
||||||
scDynamicStoreSetValue func(store, key, value uintptr) bool
|
|
||||||
scDynamicStoreRemoveValue func(store, key uintptr) bool
|
|
||||||
scError func() int32
|
|
||||||
)
|
|
||||||
|
|
||||||
// loadFrameworks binds the few CoreFoundation and SystemConfiguration calls we need. purego does
|
|
||||||
// it without cgo, so this cross compiles and runs on a mac with no developer tools. JXA can read
|
|
||||||
// the dynamic store but its writes fail, and scutil cannot build the arrays of dictionaries the
|
|
||||||
// route keys hold.
|
|
||||||
func loadFrameworks() {
|
|
||||||
cf, err := purego.Dlopen("/System/Library/Frameworks/CoreFoundation.framework/CoreFoundation", purego.RTLD_NOW|purego.RTLD_GLOBAL)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
sc, err := purego.Dlopen("/System/Library/Frameworks/SystemConfiguration.framework/SystemConfiguration", purego.RTLD_NOW|purego.RTLD_GLOBAL)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
purego.RegisterLibFunc(&cfStringCreateWithCString, cf, "CFStringCreateWithCString")
|
|
||||||
purego.RegisterLibFunc(&cfDataCreate, cf, "CFDataCreate")
|
|
||||||
purego.RegisterLibFunc(&cfPropertyListCreateWithData, cf, "CFPropertyListCreateWithData")
|
|
||||||
purego.RegisterLibFunc(&cfRelease, cf, "CFRelease")
|
|
||||||
purego.RegisterLibFunc(&scDynamicStoreCreate, sc, "SCDynamicStoreCreate")
|
|
||||||
purego.RegisterLibFunc(&scDynamicStoreSetValue, sc, "SCDynamicStoreSetValue")
|
|
||||||
purego.RegisterLibFunc(&scDynamicStoreRemoveValue, sc, "SCDynamicStoreRemoveValue")
|
|
||||||
purego.RegisterLibFunc(&scError, sc, "SCError")
|
|
||||||
}
|
|
||||||
|
|
||||||
func cfString(s string) uintptr {
|
|
||||||
b := append([]byte(s), 0)
|
|
||||||
return cfStringCreateWithCString(0, &b[0], cfStringUTF8)
|
|
||||||
}
|
|
||||||
|
|
||||||
// cfPlist builds a property list from its xml form, the least code way to make nested CF values
|
|
||||||
func cfPlist(xml string) uintptr {
|
|
||||||
b := []byte(xml)
|
|
||||||
d := cfDataCreate(0, &b[0], len(b))
|
|
||||||
defer cfRelease(d)
|
|
||||||
var errRef uintptr
|
|
||||||
p := cfPropertyListCreateWithData(0, d, 0, nil, &errRef)
|
|
||||||
if p == 0 {
|
|
||||||
log.Fatalf("invalid property list:\n%s", xml)
|
|
||||||
}
|
|
||||||
return p
|
|
||||||
}
|
|
||||||
|
|
||||||
type store struct {
|
|
||||||
ref uintptr
|
|
||||||
k4, k6 uintptr
|
|
||||||
}
|
|
||||||
|
|
||||||
func openStore() *store {
|
|
||||||
return &store{
|
|
||||||
ref: scDynamicStoreCreate(0, cfString("vpnemu"), 0, 0),
|
|
||||||
k4: cfString("State:/Network/Service/" + serviceID + "/IPv4"),
|
|
||||||
k6: cfString("State:/Network/Service/" + serviceID + "/IPv6"),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *store) set(key uintptr, xml string) {
|
|
||||||
v := cfPlist(xml)
|
|
||||||
defer cfRelease(v)
|
|
||||||
if !scDynamicStoreSetValue(s.ref, key, v) {
|
|
||||||
log.Fatalf("SCDynamicStoreSetValue failed: SCError %d", scError())
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *store) withdraw() {
|
|
||||||
scDynamicStoreRemoveValue(s.ref, s.k4)
|
|
||||||
scDynamicStoreRemoveValue(s.ref, s.k6)
|
|
||||||
}
|
|
||||||
|
|
||||||
// createUtun opens a utun the kernel numbers for us and gives it Tailscale-like single addresses.
|
|
||||||
// The interface lives as long as the returned fd.
|
|
||||||
func createUtun() (int, string) {
|
|
||||||
fd, err := unix.Socket(unix.AF_SYSTEM, unix.SOCK_DGRAM, unix.AF_SYS_CONTROL)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("system socket: %v", err)
|
|
||||||
}
|
|
||||||
info := &unix.CtlInfo{}
|
|
||||||
copy(info.Name[:], utunControlName)
|
|
||||||
if err = unix.IoctlCtlInfo(fd, info); err != nil {
|
|
||||||
log.Fatalf("CTLIOCGINFO: %v", err)
|
|
||||||
}
|
|
||||||
if err = unix.Connect(fd, &unix.SockaddrCtl{ID: info.Id, Unit: 0}); err != nil {
|
|
||||||
log.Fatalf("connect utun control: %v", err)
|
|
||||||
}
|
|
||||||
name, err := unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, utunOptIfname)
|
|
||||||
if err != nil {
|
|
||||||
log.Fatalf("utun name: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
for _, args := range [][]string{
|
|
||||||
{name, "inet", addr4, addr4, "netmask", "255.255.255.255", "up"},
|
|
||||||
{name, "inet6", addr6, "prefixlen", "128"},
|
|
||||||
} {
|
|
||||||
if out, err := exec.Command("ifconfig", args...).CombinedOutput(); err != nil {
|
|
||||||
log.Fatalf("ifconfig %s: %v: %s", strings.Join(args, " "), err, out)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return fd, name
|
|
||||||
}
|
|
||||||
|
|
||||||
// lanPrefixes applies Tailscale's attached-LAN rule to every interface but those in skip
|
|
||||||
func lanPrefixes(skip []string) (v4, v6 []netip.Prefix) {
|
|
||||||
ifs, err := net.Interfaces()
|
|
||||||
if err != nil {
|
|
||||||
log.Fatal(err)
|
|
||||||
}
|
|
||||||
for _, i := range ifs {
|
|
||||||
if slices.Contains(skip, i.Name) || i.Flags&net.FlagLoopback != 0 {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
addrs, _ := i.Addrs()
|
|
||||||
for _, a := range addrs {
|
|
||||||
n, ok := a.(*net.IPNet)
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
ip, _ := netip.AddrFromSlice(n.IP)
|
|
||||||
ip = ip.Unmap()
|
|
||||||
ones, _ := n.Mask.Size()
|
|
||||||
pfx := netip.PrefixFrom(ip, ones).Masked()
|
|
||||||
// Tailscale skips single IPs. Link-local is our addition, every interface has the
|
|
||||||
// same fe80::/64 and claiming it would only break the harness's own host.
|
|
||||||
if pfx.IsSingleIP() || ip.IsLinkLocalUnicast() {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if ip.Is4() {
|
|
||||||
v4 = append(v4, pfx)
|
|
||||||
} else {
|
|
||||||
v6 = append(v6, pfx)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
slices.SortFunc(v4, func(a, b netip.Prefix) int { return a.Addr().Compare(b.Addr()) })
|
|
||||||
slices.SortFunc(v6, func(a, b netip.Prefix) int { return a.Addr().Compare(b.Addr()) })
|
|
||||||
return v4, v6
|
|
||||||
}
|
|
||||||
|
|
||||||
func routes4(ps []netip.Prefix) string {
|
|
||||||
var b strings.Builder
|
|
||||||
for _, p := range ps {
|
|
||||||
mask := net.IP(net.CIDRMask(p.Bits(), 32)).String()
|
|
||||||
fmt.Fprintf(&b, "<dict><key>DestinationAddress</key><string>%s</string><key>SubnetMask</key><string>%s</string></dict>", p.Addr(), mask)
|
|
||||||
}
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func routes6(ps []netip.Prefix) string {
|
|
||||||
var b strings.Builder
|
|
||||||
for _, p := range ps {
|
|
||||||
fmt.Fprintf(&b, "<dict><key>DestinationAddress</key><string>%s</string><key>PrefixLength</key><integer>%d</integer></dict>", p.Addr(), p.Bits())
|
|
||||||
}
|
|
||||||
return b.String()
|
|
||||||
}
|
|
||||||
|
|
||||||
func (s *store) publish(ifName string, exclude bool, v4, v6 []netip.Prefix) {
|
|
||||||
routesKey, primary := "AdditionalRoutes", ""
|
|
||||||
if exclude {
|
|
||||||
// configd only steers excluded routes for a service with a default route that is primary.
|
|
||||||
// While this runs, everything not on a kept interface's subnet goes into the utun and dies.
|
|
||||||
routesKey = "ExcludedRoutes"
|
|
||||||
primary = "<key>Router</key><string>" + addr4 + "</string><key>OverridePrimary</key><integer>1</integer>"
|
|
||||||
}
|
|
||||||
|
|
||||||
const head = `<?xml version="1.0" encoding="UTF-8"?><plist version="1.0"><dict>`
|
|
||||||
s.set(s.k4, head+
|
|
||||||
"<key>InterfaceName</key><string>"+ifName+"</string>"+
|
|
||||||
"<key>Addresses</key><array><string>"+addr4+"</string></array>"+
|
|
||||||
"<key>SubnetMasks</key><array><string>255.255.255.255</string></array>"+primary+
|
|
||||||
"<key>"+routesKey+"</key><array>"+routes4(v4)+"</array></dict></plist>")
|
|
||||||
s.set(s.k6, head+
|
|
||||||
"<key>InterfaceName</key><string>"+ifName+"</string>"+
|
|
||||||
"<key>Addresses</key><array><string>"+addr6+"</string></array>"+
|
|
||||||
"<key>PrefixLength</key><array><integer>128</integer></array>"+
|
|
||||||
"<key>"+routesKey+"</key><array>"+routes6(v6)+"</array></dict></plist>")
|
|
||||||
|
|
||||||
log.Printf("published %s on %s, %s: %v %v", serviceID, ifName, routesKey, v4, v6)
|
|
||||||
}
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
mode := flag.String("mode", "claim", "claim: exit node without LAN access, the prefixes are routed into the vpn\n"+
|
|
||||||
"exclude: exit node with LAN access, the prefixes leave via the next interface and the vpn becomes primary")
|
|
||||||
keep := flag.String("keep", "en0", "comma separated interfaces whose prefixes are never published, keep the one your ssh session uses")
|
|
||||||
follow := flag.Bool("follow", true, "republish when interface prefixes change, as Tailscale does on link changes.\n"+
|
|
||||||
"Off, a route stays published after its interface is gone, which is how a leftover route is staged")
|
|
||||||
clear := flag.Bool("clear", false, "withdraw a service a crashed run left behind, then exit")
|
|
||||||
flag.Parse()
|
|
||||||
|
|
||||||
if *mode != "claim" && *mode != "exclude" {
|
|
||||||
log.Fatalf("unknown -mode %q", *mode)
|
|
||||||
}
|
|
||||||
if os.Geteuid() != 0 {
|
|
||||||
log.Fatal("needs root, to create the utun and write configd's dynamic store")
|
|
||||||
}
|
|
||||||
|
|
||||||
loadFrameworks()
|
|
||||||
s := openStore()
|
|
||||||
if *clear {
|
|
||||||
s.withdraw()
|
|
||||||
log.Print("withdrawn")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
fd, ifName := createUtun()
|
|
||||||
exclude := *mode == "exclude"
|
|
||||||
skip := append(strings.Split(*keep, ","), ifName)
|
|
||||||
|
|
||||||
v4, v6 := lanPrefixes(skip)
|
|
||||||
s.publish(ifName, exclude, v4, v6)
|
|
||||||
|
|
||||||
sig := make(chan os.Signal, 1)
|
|
||||||
signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
|
|
||||||
tick := time.NewTicker(time.Second)
|
|
||||||
defer tick.Stop()
|
|
||||||
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
case <-sig:
|
|
||||||
s.withdraw()
|
|
||||||
_ = unix.Close(fd)
|
|
||||||
log.Print("withdrawn")
|
|
||||||
return
|
|
||||||
case <-tick.C:
|
|
||||||
if !*follow {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
n4, n6 := lanPrefixes(skip)
|
|
||||||
if !slices.Equal(n4, v4) || !slices.Equal(n6, v6) {
|
|
||||||
v4, v6 = n4, n6
|
|
||||||
s.publish(ifName, exclude, v4, v6)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,76 +0,0 @@
|
|||||||
#!/bin/sh
|
|
||||||
# Reports where traffic to nebula peers goes before, during and after a vpnemu exit node.
|
|
||||||
#
|
|
||||||
# usage: sudo ./repro.sh -n <nebula binary> -c <nebula config> -p <peer vpn addr> [-p <peer>...]
|
|
||||||
# [-m claim|exclude] [-k <keep interfaces>]
|
|
||||||
#
|
|
||||||
# Each peer should answer pings over the overlay. A healthy run shows the nebula utun as egress
|
|
||||||
# and the ping working in all three phases.
|
|
||||||
set -u
|
|
||||||
|
|
||||||
MODE=claim
|
|
||||||
KEEP=en0
|
|
||||||
PEERS=""
|
|
||||||
NEBULA=""
|
|
||||||
CONFIG=""
|
|
||||||
while getopts n:c:p:m:k: o; do
|
|
||||||
case $o in
|
|
||||||
n) NEBULA=$OPTARG ;;
|
|
||||||
c) CONFIG=$OPTARG ;;
|
|
||||||
p) PEERS="$PEERS $OPTARG" ;;
|
|
||||||
m) MODE=$OPTARG ;;
|
|
||||||
k) KEEP=$OPTARG ;;
|
|
||||||
*) exit 2 ;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
if [ -z "$NEBULA" ] || [ -z "$CONFIG" ] || [ -z "$PEERS" ]; then
|
|
||||||
sed -n '4,5p' "$0"
|
|
||||||
exit 2
|
|
||||||
fi
|
|
||||||
|
|
||||||
HERE=$(cd "$(dirname "$0")" && pwd)
|
|
||||||
VPNEMU=${VPNEMU:-$HERE/vpnemu}
|
|
||||||
LOG=$(mktemp -t nebula-repro)
|
|
||||||
|
|
||||||
report() {
|
|
||||||
for p in $PEERS; do
|
|
||||||
case $p in
|
|
||||||
*:*) fam=-inet6 ping=ping6 ;;
|
|
||||||
*) fam=-inet ping="ping -t 3" ;;
|
|
||||||
esac
|
|
||||||
egress=$(route -n get $fam "$p" 2>/dev/null | awk '/interface:/{print $2}')
|
|
||||||
if $ping -c 2 "$p" >/dev/null 2>&1; then ok=ok; else ok=FAIL; fi
|
|
||||||
printf '%-8s %-24s egress=%-8s ping=%s\n' "$1" "$p" "${egress:-none}" "$ok"
|
|
||||||
done
|
|
||||||
}
|
|
||||||
|
|
||||||
cleanup() {
|
|
||||||
[ -n "${EMU:-}" ] && kill "$EMU" 2>/dev/null
|
|
||||||
[ -n "${NEB:-}" ] && kill "$NEB" 2>/dev/null
|
|
||||||
wait 2>/dev/null
|
|
||||||
}
|
|
||||||
trap cleanup EXIT INT TERM
|
|
||||||
|
|
||||||
"$NEBULA" -config "$CONFIG" >"$LOG" 2>&1 &
|
|
||||||
NEB=$!
|
|
||||||
sleep 3
|
|
||||||
if ! kill -0 $NEB 2>/dev/null; then
|
|
||||||
echo "nebula exited:"
|
|
||||||
tail -5 "$LOG"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
echo "nebula on $(grep -o 'interface=utun[0-9]*' "$LOG" | head -1 | cut -d= -f2), log in $LOG"
|
|
||||||
report before
|
|
||||||
|
|
||||||
"$VPNEMU" -mode "$MODE" -keep "$KEEP" &
|
|
||||||
EMU=$!
|
|
||||||
sleep 3
|
|
||||||
report during
|
|
||||||
|
|
||||||
kill $EMU
|
|
||||||
wait $EMU 2>/dev/null
|
|
||||||
EMU=
|
|
||||||
sleep 2
|
|
||||||
report after
|
|
||||||
|
|
||||||
grep -E 'level=(WARN|ERROR).*(route|Route)' "$LOG"
|
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
module github.com/slackhq/nebula
|
module github.com/slackhq/nebula
|
||||||
|
|
||||||
go 1.26.0
|
go 1.27.2
|
||||||
|
|
||||||
require (
|
require (
|
||||||
dario.cat/mergo v1.0.2
|
dario.cat/mergo v1.0.2
|
||||||
@@ -33,7 +33,7 @@ require (
|
|||||||
golang.zx2c4.com/wireguard/windows v1.0.1
|
golang.zx2c4.com/wireguard/windows v1.0.1
|
||||||
google.golang.org/protobuf v1.36.12
|
google.golang.org/protobuf v1.36.12
|
||||||
gopkg.in/yaml.v3 v3.0.1
|
gopkg.in/yaml.v3 v3.0.1
|
||||||
gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe
|
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
@@ -45,5 +45,6 @@ require (
|
|||||||
github.com/prometheus/common v0.70.1 // indirect
|
github.com/prometheus/common v0.70.1 // indirect
|
||||||
github.com/prometheus/procfs v0.21.1 // indirect
|
github.com/prometheus/procfs v0.21.1 // indirect
|
||||||
github.com/vishvananda/netns v0.0.5 // indirect
|
github.com/vishvananda/netns v0.0.5 // indirect
|
||||||
golang.org/x/time v0.5.0 // indirect
|
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc // indirect
|
||||||
|
golang.org/x/time v0.15.0 // indirect
|
||||||
)
|
)
|
||||||
@@ -157,6 +157,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
|
|||||||
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
|
||||||
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
|
||||||
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
|
||||||
|
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc h1:TS73t7x3KarrNd5qAipmspBDS1rkMcgVG/fS1aRb4Rc=
|
||||||
|
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc=
|
||||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||||
@@ -203,8 +205,8 @@ golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
|
|||||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||||
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
|
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
|
||||||
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
|
||||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||||
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||||
@@ -243,5 +245,5 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
|||||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||||
gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe h1:fre4i6mv4iBuz5lCMOzHD1rH1ljqHWSICFmZRbbgp3g=
|
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980 h1:ribyrTYQ/QYeUIs8x8bVLYwGyepa7RnHLqJzegT5YYk=
|
||||||
gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU=
|
gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980/go.mod h1:pK/dapF5Bf3KGW7/kd5CI5IC4lYh9FMFY1qETAIGLXk=
|
||||||
+98
-14
@@ -6,8 +6,10 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log/slog"
|
"log/slog"
|
||||||
|
"math/bits"
|
||||||
"net/netip"
|
"net/netip"
|
||||||
"os"
|
"os"
|
||||||
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"syscall"
|
"syscall"
|
||||||
"unsafe"
|
"unsafe"
|
||||||
@@ -28,7 +30,7 @@ type tun struct {
|
|||||||
DefaultMTU int
|
DefaultMTU int
|
||||||
Routes atomic.Pointer[[]Route]
|
Routes atomic.Pointer[[]Route]
|
||||||
routeTree atomic.Pointer[bart.Table[routing.Gateways]]
|
routeTree atomic.Pointer[bart.Table[routing.Gateways]]
|
||||||
linkAddr *netroute.LinkAddr
|
linkAddr atomic.Pointer[netroute.LinkAddr]
|
||||||
// hostOwned means the fd arrived from the OS, which has already configured addressing, mtu
|
// hostOwned means the fd arrived from the OS, which has already configured addressing, mtu
|
||||||
// and routes for it. NEPacketTunnelProvider on darwin does this.
|
// and routes for it. NEPacketTunnelProvider on darwin does this.
|
||||||
hostOwned bool
|
hostOwned bool
|
||||||
@@ -243,7 +245,7 @@ func (t *tun) Activate() error {
|
|||||||
if linkAddr == nil {
|
if linkAddr == nil {
|
||||||
return fmt.Errorf("unable to discover link_addr for tun interface")
|
return fmt.Errorf("unable to discover link_addr for tun interface")
|
||||||
}
|
}
|
||||||
t.linkAddr = linkAddr
|
t.linkAddr.Store(linkAddr)
|
||||||
|
|
||||||
for _, network := range t.vpnNetworks {
|
for _, network := range t.vpnNetworks {
|
||||||
if network.Addr().Is4() {
|
if network.Addr().Is4() {
|
||||||
@@ -269,6 +271,7 @@ func (t *tun) Activate() error {
|
|||||||
return t.addRoutes(false)
|
return t.addRoutes(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
|
||||||
func (t *tun) activate4(network netip.Prefix) error {
|
func (t *tun) activate4(network netip.Prefix) error {
|
||||||
s, err := unix.Socket(
|
s, err := unix.Socket(
|
||||||
unix.AF_INET,
|
unix.AF_INET,
|
||||||
@@ -295,22 +298,32 @@ func (t *tun) activate4(network netip.Prefix) error {
|
|||||||
MaskAddr: unix.RawSockaddrInet4{
|
MaskAddr: unix.RawSockaddrInet4{
|
||||||
Len: unix.SizeofSockaddrInet4,
|
Len: unix.SizeofSockaddrInet4,
|
||||||
Family: unix.AF_INET,
|
Family: unix.AF_INET,
|
||||||
Addr: prefixToMask(network).As4(),
|
Addr: [4]byte{255, 255, 255, 255}, //always make the address a /32
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// On Darwin, VPNs don't edit the routing table directly.
|
||||||
|
// A network extension hands its routes to the system, and configd installs them.
|
||||||
|
// configd replaces any route already holding the same prefix, including ours.
|
||||||
|
// As an example, Tailscale with an exit node treats any interface prefix wider than one address as a LAN.
|
||||||
|
// With LAN access off, it routes each LAN into Tailscale's utun.
|
||||||
|
// With LAN access on, it excludes each LAN from its tunnel.
|
||||||
|
// configd routes an excluded prefix like its own routes would, ignoring ours, so it goes out the default route.
|
||||||
|
// Either way our route is replaced, and traffic for the vpn network no longer reaches nebula.
|
||||||
|
// A single address isn't a LAN to Tailscale, so our route is left alone.
|
||||||
|
// We route the vpn network to the tun explicitly instead.
|
||||||
|
|
||||||
if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil {
|
if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil {
|
||||||
return fmt.Errorf("failed to set tun v4 address: %s", err)
|
return fmt.Errorf("failed to set tun v4 address: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
err = addRoute(network, t.linkAddr)
|
if err := addRoute(network, t.linkAddr.Load()); err != nil {
|
||||||
if err != nil {
|
return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err)
|
||||||
return err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// activate6 is activate4 for IPv6
|
||||||
func (t *tun) activate6(network netip.Prefix) error {
|
func (t *tun) activate6(network netip.Prefix) error {
|
||||||
s, err := unix.Socket(
|
s, err := unix.Socket(
|
||||||
unix.AF_INET6,
|
unix.AF_INET6,
|
||||||
@@ -332,20 +345,24 @@ func (t *tun) activate6(network netip.Prefix) error {
|
|||||||
PrefixMask: unix.RawSockaddrInet6{
|
PrefixMask: unix.RawSockaddrInet6{
|
||||||
Len: unix.SizeofSockaddrInet6,
|
Len: unix.SizeofSockaddrInet6,
|
||||||
Family: unix.AF_INET6,
|
Family: unix.AF_INET6,
|
||||||
Addr: prefixToMask(network).As16(),
|
// Always set the netmask to be a /128. See activate4 for why we do this.
|
||||||
|
Addr: [16]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
|
||||||
},
|
},
|
||||||
Lifetime: addrLifetime{
|
Lifetime: addrLifetime{
|
||||||
// never expires
|
// never expires
|
||||||
Vltime: 0xffffffff,
|
Vltime: 0xffffffff,
|
||||||
Pltime: 0xffffffff,
|
Pltime: 0xffffffff,
|
||||||
},
|
},
|
||||||
Flags: _IN6_IFF_NODAD,
|
Flags: _IN6_IFF_NODAD, // No DAD, a utun has no neighbors.
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil {
|
if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil {
|
||||||
return fmt.Errorf("failed to set tun address: %s", err)
|
return fmt.Errorf("failed to set tun address: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if err := addRoute(network, t.linkAddr.Load()); err != nil {
|
||||||
|
return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -394,6 +411,56 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
|
|||||||
return routing.Gateways{}
|
return routing.Gateways{}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func fetchRoutes() ([]netroute.Message, error) {
|
||||||
|
rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_DUMP, 0)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return netroute.ParseRIB(unix.NET_RT_DUMP, rib)
|
||||||
|
}
|
||||||
|
|
||||||
|
// routeInterfaceIn returns the index of the interface prefix is routed through, or 0 if msgs has no route for exactly
|
||||||
|
// that prefix. Routes scoped to an interface are skipped, they don't stop us adding ours.
|
||||||
|
func routeInterfaceIn(msgs []netroute.Message, prefix netip.Prefix) int {
|
||||||
|
for _, msg := range msgs {
|
||||||
|
m, ok := msg.(*netroute.RouteMessage)
|
||||||
|
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK || m.Flags&unix.RTF_IFSCOPE != 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if p, ok := routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK]); ok && p == prefix.Masked() {
|
||||||
|
return m.Index
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
// routePrefix returns the prefix a route covers. Host routes have no netmask.
|
||||||
|
func routePrefix(dst, mask netroute.Addr) (netip.Prefix, bool) {
|
||||||
|
switch d := dst.(type) {
|
||||||
|
case *netroute.Inet4Addr:
|
||||||
|
bits := 32
|
||||||
|
if m, ok := mask.(*netroute.Inet4Addr); ok {
|
||||||
|
bits = maskBits(m.IP[:])
|
||||||
|
}
|
||||||
|
return netip.PrefixFrom(netip.AddrFrom4(d.IP), bits), true
|
||||||
|
case *netroute.Inet6Addr:
|
||||||
|
bits := 128
|
||||||
|
if m, ok := mask.(*netroute.Inet6Addr); ok {
|
||||||
|
bits = maskBits(m.IP[:])
|
||||||
|
}
|
||||||
|
return netip.PrefixFrom(netip.AddrFrom16(d.IP), bits), true
|
||||||
|
}
|
||||||
|
return netip.Prefix{}, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func maskBits(mask []byte) int {
|
||||||
|
n := 0
|
||||||
|
for _, b := range mask {
|
||||||
|
n += bits.OnesCount8(b)
|
||||||
|
}
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
|
||||||
// Get the LinkAddr for the interface of the given name
|
// Get the LinkAddr for the interface of the given name
|
||||||
// Is there an easier way to fetch this when we create the interface?
|
// Is there an easier way to fetch this when we create the interface?
|
||||||
// Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers.
|
// Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers.
|
||||||
@@ -428,7 +495,14 @@ func (t *tun) addRoutes(logErrors bool) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Activate hasn't run yet, it will install the routes
|
||||||
|
linkAddr := t.linkAddr.Load()
|
||||||
|
if linkAddr == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
routes := *t.Routes.Load()
|
routes := *t.Routes.Load()
|
||||||
|
systemRoutes := sync.OnceValues(fetchRoutes)
|
||||||
|
|
||||||
for _, r := range routes {
|
for _, r := range routes {
|
||||||
if len(r.Via) == 0 || !r.Install {
|
if len(r.Via) == 0 || !r.Install {
|
||||||
@@ -436,10 +510,16 @@ func (t *tun) addRoutes(logErrors bool) error {
|
|||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
err := addRoute(r.Cidr, t.linkAddr)
|
err := addRoute(r.Cidr, linkAddr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, unix.EEXIST) {
|
if errors.Is(err, unix.EEXIST) {
|
||||||
t.l.Warn("unable to add unsafe_route, identical route already exists", "route", r.Cidr)
|
// Every route we added already exists on a reload, only warn about someone else's
|
||||||
|
msgs, err := systemRoutes()
|
||||||
|
if err != nil {
|
||||||
|
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
|
||||||
|
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
|
||||||
|
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
|
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
|
||||||
if logErrors {
|
if logErrors {
|
||||||
@@ -450,6 +530,8 @@ func (t *tun) addRoutes(logErrors bool) error {
|
|||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
t.l.Info("Added route", "route", r)
|
t.l.Info("Added route", "route", r)
|
||||||
|
// A cidr listed twice would otherwise find our own route missing from an older fetch
|
||||||
|
systemRoutes = sync.OnceValues(fetchRoutes)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -457,16 +539,18 @@ func (t *tun) addRoutes(logErrors bool) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) removeRoutes(routes []Route) error {
|
func (t *tun) removeRoutes(routes []Route) error {
|
||||||
if t.hostOwned {
|
linkAddr := t.linkAddr.Load()
|
||||||
|
if t.hostOwned || linkAddr == nil {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, r := range routes {
|
for _, r := range routes {
|
||||||
if !r.Install {
|
if len(r.Via) == 0 || !r.Install {
|
||||||
|
// addRoutes never installs these
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
err := delRoute(r.Cidr, t.linkAddr)
|
err := delRoute(r.Cidr, linkAddr)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.l.Error("Failed to remove route", "error", err, "route", r)
|
t.l.Error("Failed to remove route", "error", err, "route", r)
|
||||||
} else {
|
} else {
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
//go:build darwin && !ios && !e2e_testing
|
||||||
|
|
||||||
|
package overlay
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net"
|
||||||
|
"net/netip"
|
||||||
|
"os"
|
||||||
|
"os/exec"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/slackhq/nebula/config"
|
||||||
|
"github.com/slackhq/nebula/routing"
|
||||||
|
"github.com/slackhq/nebula/test"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
netroute "golang.org/x/net/route"
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Every mac routes loopback through lo0
|
||||||
|
func TestRouteInterfaceIn(t *testing.T) {
|
||||||
|
msgs, err := fetchRoutes()
|
||||||
|
require.NoError(t, err)
|
||||||
|
routeInterface := func(s string) int { return routeInterfaceIn(msgs, netip.MustParsePrefix(s)) }
|
||||||
|
|
||||||
|
lo0, err := net.InterfaceByName("lo0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
assert.Equal(t, lo0.Index, routeInterface("127.0.0.0/8"))
|
||||||
|
assert.Equal(t, lo0.Index, routeInterface("127.0.0.1/32"), "a host route has no netmask")
|
||||||
|
assert.Equal(t, lo0.Index, routeInterface("::1/128"))
|
||||||
|
assert.Equal(t, 0, routeInterface("127.0.0.0/16"), "only an exact prefix counts")
|
||||||
|
|
||||||
|
// Most macs also list the default route scoped to their other interfaces
|
||||||
|
out, err := exec.Command("route", "-n", "get", "default").Output()
|
||||||
|
require.NoError(t, err)
|
||||||
|
var name string
|
||||||
|
for _, line := range strings.Split(string(out), "\n") {
|
||||||
|
if v, ok := strings.CutPrefix(strings.TrimSpace(line), "interface: "); ok {
|
||||||
|
name = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if name == "" {
|
||||||
|
t.Skip("no default route")
|
||||||
|
}
|
||||||
|
def, err := net.InterfaceByName(name)
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, def.Index, routeInterface("0.0.0.0/0"))
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRouteInterfaceIn_SkipsScopedCopies(t *testing.T) {
|
||||||
|
route := func(index, flags int) netroute.Message {
|
||||||
|
addrs := make([]netroute.Addr, unix.RTAX_MAX)
|
||||||
|
addrs[unix.RTAX_DST] = &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}}
|
||||||
|
addrs[unix.RTAX_NETMASK] = &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}}
|
||||||
|
return &netroute.RouteMessage{Index: index, Flags: flags, Addrs: addrs}
|
||||||
|
}
|
||||||
|
p := netip.MustParsePrefix("192.168.1.0/24")
|
||||||
|
assert.Equal(t, 7, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE), route(7, unix.RTF_UP)}, p))
|
||||||
|
assert.Equal(t, 0, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE)}, p))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A reload can happen before Activate
|
||||||
|
func TestTunRoutesBeforeActivate(t *testing.T) {
|
||||||
|
tn := &tun{l: test.NewLogger()}
|
||||||
|
routes := []Route{{
|
||||||
|
Cidr: netip.MustParsePrefix("10.253.1.0/24"),
|
||||||
|
Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)},
|
||||||
|
Install: true,
|
||||||
|
}}
|
||||||
|
tn.Routes.Store(&routes)
|
||||||
|
|
||||||
|
assert.NoError(t, tn.addRoutes(true))
|
||||||
|
assert.NoError(t, tn.removeRoutes(routes))
|
||||||
|
}
|
||||||
|
|
||||||
|
// A tun.routes entry with no via is never installed, so removing it must not delete a route for the same cidr that
|
||||||
|
// belongs to someone else
|
||||||
|
func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
|
||||||
|
if os.Geteuid() != 0 {
|
||||||
|
t.Skip("creating a utun and adding routes needs root")
|
||||||
|
}
|
||||||
|
lo0, err := net.InterfaceByName("lo0")
|
||||||
|
require.NoError(t, err)
|
||||||
|
cidr := "10.250.0.128/25"
|
||||||
|
require.NoError(t, exec.Command("route", "-q", "-n", "add", "-net", cidr, "-interface", "lo0").Run())
|
||||||
|
t.Cleanup(func() { _ = exec.Command("route", "-q", "-n", "delete", "-net", cidr).Run() })
|
||||||
|
|
||||||
|
l := test.NewLogger()
|
||||||
|
c := config.NewC(l)
|
||||||
|
require.NoError(t, c.LoadString("tun:\n routes:\n - route: "+cidr+"\n mtu: 1300\n"))
|
||||||
|
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
t.Cleanup(func() { _ = tn.Close() })
|
||||||
|
require.NoError(t, tn.Activate())
|
||||||
|
|
||||||
|
require.NoError(t, c.ReloadConfigString("tun: {}\n"))
|
||||||
|
msgs, err := fetchRoutes()
|
||||||
|
require.NoError(t, err)
|
||||||
|
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
|
||||||
|
}
|
||||||
@@ -589,7 +589,8 @@ func (t *tun) addRoutes(logErrors bool) error {
|
|||||||
|
|
||||||
func (t *tun) removeRoutes(routes []Route) error {
|
func (t *tun) removeRoutes(routes []Route) error {
|
||||||
for _, r := range routes {
|
for _, r := range routes {
|
||||||
if !r.Install {
|
if len(r.Via) == 0 || !r.Install {
|
||||||
|
// addRoutes never installs these
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+96
-3
@@ -552,13 +552,17 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
|
|||||||
if cidr.Addr().Is6() {
|
if cidr.Addr().Is6() {
|
||||||
nr.Priority = 256
|
nr.Priority = 256
|
||||||
}
|
}
|
||||||
err := netlink.RouteReplace(&nr)
|
held, err := t.otherLinkRoutes()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
err = t.replaceRoute(&nr, held)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.l.Warn("Failed to set default route MTU, retrying", "error", err, "cidr", cidr)
|
t.l.Warn("Failed to set default route MTU, retrying", "error", err, "cidr", cidr)
|
||||||
//retry twice more -- on some systems there appears to be a race condition where if we set routes too soon, netlink says `invalid argument`
|
//retry twice more -- on some systems there appears to be a race condition where if we set routes too soon, netlink says `invalid argument`
|
||||||
for range 2 {
|
for range 2 {
|
||||||
time.Sleep(100 * time.Millisecond)
|
time.Sleep(100 * time.Millisecond)
|
||||||
err = netlink.RouteReplace(&nr)
|
err = t.replaceRoute(&nr, held)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
break
|
break
|
||||||
} else {
|
} else {
|
||||||
@@ -578,6 +582,15 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (t *tun) addRoutes(logErrors bool) error {
|
func (t *tun) addRoutes(logErrors bool) error {
|
||||||
|
held, err := t.otherLinkRoutes()
|
||||||
|
if err != nil {
|
||||||
|
if logErrors {
|
||||||
|
t.l.Error("Failed to add routes", "error", err)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
// Path routes
|
// Path routes
|
||||||
routes := *t.Routes.Load()
|
routes := *t.Routes.Load()
|
||||||
for _, r := range routes {
|
for _, r := range routes {
|
||||||
@@ -602,7 +615,7 @@ func (t *tun) addRoutes(logErrors bool) error {
|
|||||||
nr.Priority = r.Metric
|
nr.Priority = r.Metric
|
||||||
}
|
}
|
||||||
|
|
||||||
err := netlink.RouteReplace(&nr)
|
err := t.replaceRoute(&nr, held)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
|
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
|
||||||
if logErrors {
|
if logErrors {
|
||||||
@@ -618,6 +631,86 @@ func (t *tun) addRoutes(logErrors bool) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ip6RoutePrioUser is the metric the kernel gives an IPv6 route added without one
|
||||||
|
const ip6RoutePrioUser = 1024
|
||||||
|
|
||||||
|
// routeKey is where the main table files a route. Replacing a route replaces the first one with the same key, whatever
|
||||||
|
// device it is on
|
||||||
|
type routeKey struct {
|
||||||
|
dst netip.Prefix
|
||||||
|
priority int
|
||||||
|
}
|
||||||
|
|
||||||
|
func routeKeyOf(dst *net.IPNet, family, priority int) (routeKey, bool) {
|
||||||
|
var p netip.Prefix
|
||||||
|
if dst == nil {
|
||||||
|
// A default route
|
||||||
|
if family == unix.AF_INET6 {
|
||||||
|
p = netip.PrefixFrom(netip.IPv6Unspecified(), 0)
|
||||||
|
} else {
|
||||||
|
p = netip.PrefixFrom(netip.IPv4Unspecified(), 0)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
addr, ok := netip.AddrFromSlice(dst.IP)
|
||||||
|
if !ok {
|
||||||
|
return routeKey{}, false
|
||||||
|
}
|
||||||
|
ones, _ := dst.Mask.Size()
|
||||||
|
p = netip.PrefixFrom(addr.Unmap(), ones).Masked()
|
||||||
|
}
|
||||||
|
if priority == 0 && p.Addr().Is6() {
|
||||||
|
priority = ip6RoutePrioUser
|
||||||
|
}
|
||||||
|
return routeKey{dst: p, priority: priority}, true
|
||||||
|
}
|
||||||
|
|
||||||
|
// otherLinkRoutes returns the keys of the main table routes that are not on our device
|
||||||
|
func (t *tun) otherLinkRoutes() (map[routeKey]struct{}, error) {
|
||||||
|
routes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, &netlink.Route{Table: unix.RT_TABLE_MAIN}, netlink.RT_FILTER_TABLE)
|
||||||
|
// An interrupted dump may miss a route, which then gets replaced like it always used to be
|
||||||
|
if err != nil && !errors.Is(err, netlink.ErrDumpInterrupted) {
|
||||||
|
return nil, fmt.Errorf("failed to list the system routes: %w", err)
|
||||||
|
}
|
||||||
|
held := make(map[routeKey]struct{}, len(routes))
|
||||||
|
for _, r := range routes {
|
||||||
|
if r.LinkIndex == t.deviceIndex {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if k, ok := routeKeyOf(r.Dst, r.Family, r.Priority); ok {
|
||||||
|
held[k] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return held, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// replaceRoute puts nr in the main table in place of our own route with its key. Replacing would take the route of
|
||||||
|
// another device holding that key, so ours goes in behind it instead. Theirs keeps routing and the kernel falls back
|
||||||
|
// to ours once theirs is removed, like when leaving the lan that had it. A different metric decides which one wins.
|
||||||
|
func (t *tun) replaceRoute(nr *netlink.Route, held map[routeKey]struct{}) error {
|
||||||
|
family := unix.AF_INET6
|
||||||
|
if nr.Dst.IP.To4() != nil {
|
||||||
|
family = unix.AF_INET
|
||||||
|
}
|
||||||
|
k, ok := routeKeyOf(nr.Dst, family, nr.Priority)
|
||||||
|
if _, taken := held[k]; !ok || !taken {
|
||||||
|
return netlink.RouteReplace(nr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Appending doesn't update a route that is already ours, it adds a second one
|
||||||
|
ours := netlink.Route{
|
||||||
|
LinkIndex: nr.LinkIndex,
|
||||||
|
Dst: nr.Dst,
|
||||||
|
Scope: nr.Scope,
|
||||||
|
Table: nr.Table,
|
||||||
|
Priority: k.priority,
|
||||||
|
}
|
||||||
|
if err := netlink.RouteDel(&ours); err != nil && !errors.Is(err, unix.ESRCH) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
t.l.Warn("Another interface already routes this destination at the same metric, our route is used once theirs is removed", "destination", k.dst, "metric", k.priority)
|
||||||
|
return netlink.RouteAppend(nr)
|
||||||
|
}
|
||||||
|
|
||||||
func (t *tun) removeRoutes(routes []Route) {
|
func (t *tun) removeRoutes(routes []Route) {
|
||||||
for _, r := range routes {
|
for _, r := range routes {
|
||||||
if !r.Install {
|
if !r.Install {
|
||||||
|
|||||||
@@ -0,0 +1,133 @@
|
|||||||
|
//go:build linux && !android && !e2e_testing
|
||||||
|
|
||||||
|
package overlay
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net"
|
||||||
|
"net/netip"
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/DefinedNet/netlink"
|
||||||
|
"github.com/slackhq/nebula/config"
|
||||||
|
"github.com/slackhq/nebula/test"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"golang.org/x/sys/unix"
|
||||||
|
)
|
||||||
|
|
||||||
|
// An unsafe_route with the same destination and metric as a lan's route must not replace it. Ours waits behind theirs
|
||||||
|
// and takes over when the lan goes away. Run as root, or unprivileged with `unshare -rn`.
|
||||||
|
func TestTunRouteLeavesAnotherDevicesRoute(t *testing.T) {
|
||||||
|
if os.Geteuid() != 0 {
|
||||||
|
t.Skip("creating links and adding routes needs root")
|
||||||
|
}
|
||||||
|
|
||||||
|
lan := &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: "nebrt0"}, PeerName: "nebrt1"}
|
||||||
|
require.NoError(t, netlink.LinkAdd(lan))
|
||||||
|
t.Cleanup(func() { _ = netlink.LinkDel(lan) })
|
||||||
|
peer, err := netlink.LinkByName("nebrt1")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, netlink.LinkSetUp(lan))
|
||||||
|
require.NoError(t, netlink.LinkSetUp(peer))
|
||||||
|
|
||||||
|
lanAddrs := []*netlink.Addr{
|
||||||
|
{IPNet: &net.IPNet{IP: net.ParseIP("198.51.100.5"), Mask: net.CIDRMask(24, 32)}},
|
||||||
|
{IPNet: &net.IPNet{IP: net.ParseIP("2001:db8:1::5"), Mask: net.CIDRMask(64, 128)}, Flags: unix.IFA_F_NODAD},
|
||||||
|
}
|
||||||
|
for _, a := range lanAddrs {
|
||||||
|
require.NoError(t, netlink.AddrAdd(lan, a))
|
||||||
|
}
|
||||||
|
|
||||||
|
l := test.NewLogger()
|
||||||
|
c := config.NewC(l)
|
||||||
|
// The kernel puts a v4 lan route at metric 0, our default, and a v6 one at 256
|
||||||
|
require.NoError(t, c.LoadString(`
|
||||||
|
tun:
|
||||||
|
dev: nebrt2
|
||||||
|
unsafe_routes:
|
||||||
|
- route: 198.51.100.0/24
|
||||||
|
via: 10.250.0.2
|
||||||
|
- route: 2001:db8:1::/64
|
||||||
|
via: fd00:250::2
|
||||||
|
metric: 256
|
||||||
|
`))
|
||||||
|
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
t.Cleanup(func() { _ = tn.Close() })
|
||||||
|
require.NoError(t, tn.Activate())
|
||||||
|
|
||||||
|
via := func(dst string) int {
|
||||||
|
routes, err := netlink.RouteGet(net.ParseIP(dst))
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NotEmpty(t, routes)
|
||||||
|
return routes[0].LinkIndex
|
||||||
|
}
|
||||||
|
devices := func(dst string) []int {
|
||||||
|
_, n, err := net.ParseCIDR(dst)
|
||||||
|
require.NoError(t, err)
|
||||||
|
routes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, &netlink.Route{Dst: n}, netlink.RT_FILTER_DST)
|
||||||
|
require.NoError(t, err)
|
||||||
|
var out []int
|
||||||
|
for _, r := range routes {
|
||||||
|
out = append(out, r.LinkIndex)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, dst := range []string{"198.51.100.0/24", "2001:db8:1::/64"} {
|
||||||
|
assert.ElementsMatch(t, []int{lan.Index, tn.deviceIndex}, devices(dst), dst)
|
||||||
|
}
|
||||||
|
assert.Equal(t, lan.Index, via("198.51.100.9"))
|
||||||
|
assert.Equal(t, lan.Index, via("2001:db8:1::9"))
|
||||||
|
|
||||||
|
// Leaving the lan
|
||||||
|
for _, a := range lanAddrs {
|
||||||
|
require.NoError(t, netlink.AddrDel(lan, a))
|
||||||
|
}
|
||||||
|
assert.Equal(t, tn.deviceIndex, via("198.51.100.9"))
|
||||||
|
assert.Equal(t, tn.deviceIndex, via("2001:db8:1::9"))
|
||||||
|
|
||||||
|
// Dropping the unsafe_routes removes only ours
|
||||||
|
for _, a := range lanAddrs {
|
||||||
|
require.NoError(t, netlink.AddrAdd(lan, a))
|
||||||
|
}
|
||||||
|
require.NoError(t, c.ReloadConfigString("tun:\n dev: nebrt2\n"))
|
||||||
|
for _, dst := range []string{"198.51.100.0/24", "2001:db8:1::/64"} {
|
||||||
|
assert.Equal(t, []int{lan.Index}, devices(dst), dst)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The route for our own vpn network is replaced to set its mtu, which must not take a lan's route for the same network
|
||||||
|
func TestTunVpnNetworkLeavesAnotherDevicesRoute(t *testing.T) {
|
||||||
|
if os.Geteuid() != 0 {
|
||||||
|
t.Skip("creating links and adding routes needs root")
|
||||||
|
}
|
||||||
|
|
||||||
|
lan := &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: "nebrt0"}, PeerName: "nebrt1"}
|
||||||
|
require.NoError(t, netlink.LinkAdd(lan))
|
||||||
|
t.Cleanup(func() { _ = netlink.LinkDel(lan) })
|
||||||
|
require.NoError(t, netlink.LinkSetUp(lan))
|
||||||
|
require.NoError(t, netlink.AddrAdd(lan, &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP("10.250.0.5"), Mask: net.CIDRMask(24, 32)}}))
|
||||||
|
|
||||||
|
l := test.NewLogger()
|
||||||
|
c := config.NewC(l)
|
||||||
|
require.NoError(t, c.LoadString("tun:\n dev: nebrt2\n"))
|
||||||
|
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
|
||||||
|
require.NoError(t, err)
|
||||||
|
t.Cleanup(func() { _ = tn.Close() })
|
||||||
|
require.NoError(t, tn.Activate())
|
||||||
|
|
||||||
|
_, n, err := net.ParseCIDR("10.250.0.0/24")
|
||||||
|
require.NoError(t, err)
|
||||||
|
routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, &netlink.Route{Dst: n}, netlink.RT_FILTER_DST)
|
||||||
|
require.NoError(t, err)
|
||||||
|
var devices []int
|
||||||
|
for _, r := range routes {
|
||||||
|
devices = append(devices, r.LinkIndex)
|
||||||
|
if r.LinkIndex == tn.deviceIndex {
|
||||||
|
assert.Equal(t, tn.DefaultMTU, r.MTU, "ours still carries the mtu")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert.ElementsMatch(t, []int{lan.Index, tn.deviceIndex}, devices)
|
||||||
|
}
|
||||||
@@ -420,7 +420,8 @@ func (t *tun) addRoutes(logErrors bool) error {
|
|||||||
|
|
||||||
func (t *tun) removeRoutes(routes []Route) error {
|
func (t *tun) removeRoutes(routes []Route) error {
|
||||||
for _, r := range routes {
|
for _, r := range routes {
|
||||||
if !r.Install {
|
if len(r.Via) == 0 || !r.Install {
|
||||||
|
// addRoutes never installs these
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -399,7 +399,8 @@ func (t *tun) addRoutes(logErrors bool) error {
|
|||||||
|
|
||||||
func (t *tun) removeRoutes(routes []Route) error {
|
func (t *tun) removeRoutes(routes []Route) error {
|
||||||
for _, r := range routes {
|
for _, r := range routes {
|
||||||
if !r.Install {
|
if len(r.Via) == 0 || !r.Install {
|
||||||
|
// addRoutes never installs these
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -60,6 +60,20 @@ type sshDeviceInfoFlags struct {
|
|||||||
func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
|
func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
|
||||||
c.RegisterReloadCallback(func(c *config.C) {
|
c.RegisterReloadCallback(func(c *config.C) {
|
||||||
if c.GetBool("sshd.enabled", false) {
|
if c.GetBool("sshd.enabled", false) {
|
||||||
|
// Restarting the server drops every open session, the one that asked for this reload included. Only restart
|
||||||
|
// when it isn't running or its config changed. The host key file can change under the same config, new
|
||||||
|
// handshakes use the new key without a restart
|
||||||
|
if ssh.Running() && !c.HasChanged("sshd") {
|
||||||
|
key, err := loadSSHHostKey(c)
|
||||||
|
if err == nil {
|
||||||
|
err = ssh.SetHostKey(key)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
l.Error("Failed to reconfigure the sshd", "error", err)
|
||||||
|
ssh.Stop()
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
sshRun, err := configSSH(l, ssh, c)
|
sshRun, err := configSSH(l, ssh, c)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
l.Error("Failed to reconfigure the sshd", "error", err)
|
l.Error("Failed to reconfigure the sshd", "error", err)
|
||||||
@@ -74,6 +88,22 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// loadSSHHostKey returns sshd.host_key, read from its file when it names one.
|
||||||
|
func loadSSHHostKey(c *config.C) ([]byte, error) {
|
||||||
|
hostKeyPathOrKey := c.GetString("sshd.host_key", "")
|
||||||
|
if hostKeyPathOrKey == "" {
|
||||||
|
return nil, fmt.Errorf("sshd.host_key must be provided")
|
||||||
|
}
|
||||||
|
if strings.Contains(hostKeyPathOrKey, "-----BEGIN") {
|
||||||
|
return []byte(hostKeyPathOrKey), nil
|
||||||
|
}
|
||||||
|
b, err := os.ReadFile(hostKeyPathOrKey)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err)
|
||||||
|
}
|
||||||
|
return b, nil
|
||||||
|
}
|
||||||
|
|
||||||
// configSSH reads the ssh info out of the passed-in Config and
|
// configSSH reads the ssh info out of the passed-in Config and
|
||||||
// updates the passed-in SSHServer. On success, it returns a function
|
// updates the passed-in SSHServer. On success, it returns a function
|
||||||
// that callers may invoke to run the configured ssh server. On
|
// that callers may invoke to run the configured ssh server. On
|
||||||
@@ -92,19 +122,9 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
|
|||||||
return nil, fmt.Errorf("sshd.listen can not use port 22")
|
return nil, fmt.Errorf("sshd.listen can not use port 22")
|
||||||
}
|
}
|
||||||
|
|
||||||
hostKeyPathOrKey := c.GetString("sshd.host_key", "")
|
hostKeyBytes, err := loadSSHHostKey(c)
|
||||||
if hostKeyPathOrKey == "" {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("sshd.host_key must be provided")
|
return nil, err
|
||||||
}
|
|
||||||
|
|
||||||
var hostKeyBytes []byte
|
|
||||||
if strings.Contains(hostKeyPathOrKey, "-----BEGIN") {
|
|
||||||
hostKeyBytes = []byte(hostKeyPathOrKey)
|
|
||||||
} else {
|
|
||||||
hostKeyBytes, err = os.ReadFile(hostKeyPathOrKey)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
err = ssh.SetHostKey(hostKeyBytes)
|
err = ssh.SetHostKey(hostKeyBytes)
|
||||||
|
|||||||
+48
-13
@@ -8,14 +8,18 @@ import (
|
|||||||
"log/slog"
|
"log/slog"
|
||||||
"net"
|
"net"
|
||||||
"sync"
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
|
||||||
"github.com/armon/go-radix"
|
"github.com/armon/go-radix"
|
||||||
"golang.org/x/crypto/ssh"
|
"golang.org/x/crypto/ssh"
|
||||||
)
|
)
|
||||||
|
|
||||||
type SSHServer struct {
|
type SSHServer struct {
|
||||||
config *ssh.ServerConfig
|
// config is replaced whole when the host key changes, ssh.ServerConfig.AddHostKey only replaces a key of the same
|
||||||
l *slog.Logger
|
// algorithm and appends any other
|
||||||
|
config atomic.Pointer[ssh.ServerConfig]
|
||||||
|
authenticate func(ssh.ConnMetadata, ssh.PublicKey) (*ssh.Permissions, error)
|
||||||
|
l *slog.Logger
|
||||||
|
|
||||||
certChecker *ssh.CertChecker
|
certChecker *ssh.CertChecker
|
||||||
|
|
||||||
@@ -28,7 +32,12 @@ type SSHServer struct {
|
|||||||
// List of available commands
|
// List of available commands
|
||||||
helpCommand *Command
|
helpCommand *Command
|
||||||
commands *radix.Tree
|
commands *radix.Tree
|
||||||
listener net.Listener
|
listener atomic.Pointer[net.Listener]
|
||||||
|
|
||||||
|
// runs counts the Runs that are listening. A reload's Run can start before the one it replaced has returned
|
||||||
|
runs atomic.Int32
|
||||||
|
// addr is where the newest Run listens, nil when none is
|
||||||
|
addr atomic.Pointer[net.Addr]
|
||||||
|
|
||||||
// ctx parents per-Run contexts. Cancelling it (e.g. via Control.Stop) tears the server down even
|
// ctx parents per-Run contexts. Cancelling it (e.g. via Control.Stop) tears the server down even
|
||||||
// across reloads, since each Run derives a fresh child rather than reusing this one directly.
|
// across reloads, since each Run derives a fresh child rather than reusing this one directly.
|
||||||
@@ -85,10 +94,8 @@ func NewSSHServer(ctx context.Context, l *slog.Logger) (*SSHServer, error) {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
s.config = &ssh.ServerConfig{
|
s.authenticate = cc.Authenticate
|
||||||
PublicKeyCallback: cc.Authenticate,
|
s.config.Store(s.newConfig())
|
||||||
ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"),
|
|
||||||
}
|
|
||||||
|
|
||||||
s.RegisterCommand(&Command{
|
s.RegisterCommand(&Command{
|
||||||
Name: "help",
|
Name: "help",
|
||||||
@@ -107,10 +114,32 @@ func (s *SSHServer) SetHostKey(hostPrivateKey []byte) error {
|
|||||||
return fmt.Errorf("failed to parse private key: %s", err)
|
return fmt.Errorf("failed to parse private key: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
s.config.AddHostKey(private)
|
config := s.newConfig()
|
||||||
|
config.AddHostKey(private)
|
||||||
|
s.config.Store(config)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (s *SSHServer) newConfig() *ssh.ServerConfig {
|
||||||
|
return &ssh.ServerConfig{
|
||||||
|
PublicKeyCallback: s.authenticate,
|
||||||
|
ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Addr is the address the newest Run listens on, nil when none is.
|
||||||
|
func (s *SSHServer) Addr() net.Addr {
|
||||||
|
if a := s.addr.Load(); a != nil {
|
||||||
|
return *a
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Running is true while a Run is listening.
|
||||||
|
func (s *SSHServer) Running() bool {
|
||||||
|
return s.runs.Load() > 0
|
||||||
|
}
|
||||||
|
|
||||||
func (s *SSHServer) ClearTrustedCAs() {
|
func (s *SSHServer) ClearTrustedCAs() {
|
||||||
s.authLock.Lock()
|
s.authLock.Lock()
|
||||||
s.trustedCAs = []ssh.PublicKey{}
|
s.trustedCAs = []ssh.PublicKey{}
|
||||||
@@ -181,10 +210,16 @@ func (s *SSHServer) Run(addr string) error {
|
|||||||
// this run owns. They start equal but a fast reload may overwrite s.listener with the next run's
|
// this run owns. They start equal but a fast reload may overwrite s.listener with the next run's
|
||||||
// listener before this run's watcher fires, so each run must close its own listener via the local
|
// listener before this run's watcher fires, so each run must close its own listener via the local
|
||||||
// reference.
|
// reference.
|
||||||
s.listener = listener
|
s.listener.Store(&listener)
|
||||||
|
s.runs.Add(1)
|
||||||
|
bound := listener.Addr()
|
||||||
|
s.addr.Store(&bound)
|
||||||
|
|
||||||
runCtx, cancel := context.WithCancel(s.ctx)
|
runCtx, cancel := context.WithCancel(s.ctx)
|
||||||
defer cancel()
|
defer cancel()
|
||||||
|
// Deferred after cancel so they run first, a run on its way out no longer counts as listening
|
||||||
|
defer s.runs.Add(-1)
|
||||||
|
defer s.addr.CompareAndSwap(&bound, nil)
|
||||||
|
|
||||||
// Close the listener when this run's context is cancelled. That can come from the parent
|
// Close the listener when this run's context is cancelled. That can come from the parent
|
||||||
// (Control.Stop), from Run returning normally (defer cancel above), or transitively when a sibling
|
// (Control.Stop), from Run returning normally (defer cancel above), or transitively when a sibling
|
||||||
@@ -197,7 +232,7 @@ func (s *SSHServer) Run(addr string) error {
|
|||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
s.l.Info("SSH server is listening", "sshListener", addr)
|
s.l.Info("SSH server is listening", "sshListener", bound)
|
||||||
|
|
||||||
// Run loops until there is an error
|
// Run loops until there is an error
|
||||||
s.run(runCtx, listener)
|
s.run(runCtx, listener)
|
||||||
@@ -226,7 +261,7 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
|
|||||||
<-sessionContext.Done()
|
<-sessionContext.Done()
|
||||||
c.Close()
|
c.Close()
|
||||||
}()
|
}()
|
||||||
conn, chans, reqs, err := ssh.NewServerConn(c, s.config)
|
conn, chans, reqs, err := ssh.NewServerConn(c, s.config.Load())
|
||||||
fp := ""
|
fp := ""
|
||||||
if conn != nil {
|
if conn != nil {
|
||||||
fp = conn.Permissions.Extensions["fp"]
|
fp = conn.Permissions.Extensions["fp"]
|
||||||
@@ -264,8 +299,8 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *SSHServer) Stop() {
|
func (s *SSHServer) Stop() {
|
||||||
if s.listener != nil {
|
if l := s.listener.Load(); l != nil {
|
||||||
if err := s.listener.Close(); err != nil {
|
if err := (*l).Close(); err != nil && !errors.Is(err, net.ErrClosed) {
|
||||||
s.l.Warn("Failed to close the sshd listener", "error", err)
|
s.l.Warn("Failed to close the sshd listener", "error", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,196 @@
|
|||||||
|
package nebula
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto"
|
||||||
|
"crypto/ed25519"
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/rsa"
|
||||||
|
"encoding/pem"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/slackhq/nebula/config"
|
||||||
|
"github.com/slackhq/nebula/sshd"
|
||||||
|
"github.com/slackhq/nebula/test"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
"golang.org/x/crypto/ssh"
|
||||||
|
)
|
||||||
|
|
||||||
|
type sshdReloadHarness struct {
|
||||||
|
c *config.C
|
||||||
|
server *sshd.SSHServer
|
||||||
|
keyFile string
|
||||||
|
signer ssh.Signer
|
||||||
|
extra string
|
||||||
|
}
|
||||||
|
|
||||||
|
func newSSHDReloadHarness(t *testing.T) *sshdReloadHarness {
|
||||||
|
t.Helper()
|
||||||
|
l := test.NewLogger()
|
||||||
|
h := &sshdReloadHarness{keyFile: filepath.Join(t.TempDir(), "host_key")}
|
||||||
|
h.writeHostKey(t, newEd25519Key(t))
|
||||||
|
|
||||||
|
_, userKey, err := ed25519.GenerateKey(rand.Reader)
|
||||||
|
require.NoError(t, err)
|
||||||
|
h.signer, err = ssh.NewSignerFromKey(userKey)
|
||||||
|
require.NoError(t, err)
|
||||||
|
|
||||||
|
h.c = config.NewC(l)
|
||||||
|
require.NoError(t, h.c.LoadString(h.config()))
|
||||||
|
h.server, err = sshd.NewSSHServer(t.Context(), l)
|
||||||
|
require.NoError(t, err)
|
||||||
|
wireSSHReload(l, h.server, h.c)
|
||||||
|
run, err := configSSH(l, h.server, h.c)
|
||||||
|
require.NoError(t, err)
|
||||||
|
go run()
|
||||||
|
t.Cleanup(h.server.Stop)
|
||||||
|
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
|
||||||
|
return h
|
||||||
|
}
|
||||||
|
|
||||||
|
func newEd25519Key(t *testing.T) crypto.Signer {
|
||||||
|
_, key, err := ed25519.GenerateKey(rand.Reader)
|
||||||
|
require.NoError(t, err)
|
||||||
|
return key
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeHostKey replaces the host key file and returns the public key a client should now see
|
||||||
|
func (h *sshdReloadHarness) writeHostKey(t *testing.T, key crypto.Signer) ssh.PublicKey {
|
||||||
|
t.Helper()
|
||||||
|
block, err := ssh.MarshalPrivateKey(key, "")
|
||||||
|
require.NoError(t, err)
|
||||||
|
require.NoError(t, os.WriteFile(h.keyFile, pem.EncodeToMemory(block), 0o600))
|
||||||
|
pub, err := ssh.NewPublicKey(key.Public())
|
||||||
|
require.NoError(t, err)
|
||||||
|
return pub
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *sshdReloadHarness) config() string {
|
||||||
|
return fmt.Sprintf("sshd:\n enabled: true\n listen: 127.0.0.1:0\n host_key: %s\n authorized_users:\n - user: test\n keys: [%q]\n%s",
|
||||||
|
h.keyFile, string(ssh.MarshalAuthorizedKey(h.signer.PublicKey())), h.extra)
|
||||||
|
}
|
||||||
|
|
||||||
|
// dial connects to wherever the server listens now, a restart on :0 moves it. hostKeyAlgorithms limits which host keys
|
||||||
|
// the client accepts, none means any
|
||||||
|
func (h *sshdReloadHarness) dial(hostKeyAlgorithms ...string) (*ssh.Client, ssh.PublicKey, error) {
|
||||||
|
addr := h.server.Addr()
|
||||||
|
if addr == nil {
|
||||||
|
return nil, nil, fmt.Errorf("not listening")
|
||||||
|
}
|
||||||
|
var hostKey ssh.PublicKey
|
||||||
|
c, err := ssh.Dial("tcp", addr.String(), &ssh.ClientConfig{
|
||||||
|
User: "test",
|
||||||
|
Auth: []ssh.AuthMethod{ssh.PublicKeys(h.signer)},
|
||||||
|
HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error {
|
||||||
|
hostKey = key
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
HostKeyAlgorithms: hostKeyAlgorithms,
|
||||||
|
Timeout: time.Second,
|
||||||
|
})
|
||||||
|
return c, hostKey, err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (h *sshdReloadHarness) connect(t *testing.T) (*ssh.Client, ssh.PublicKey) {
|
||||||
|
t.Helper()
|
||||||
|
var client *ssh.Client
|
||||||
|
var hostKey ssh.PublicKey
|
||||||
|
require.Eventually(t, func() bool {
|
||||||
|
var err error
|
||||||
|
client, hostKey, err = h.dial()
|
||||||
|
return err == nil
|
||||||
|
}, 5*time.Second, 20*time.Millisecond)
|
||||||
|
t.Cleanup(func() { _ = client.Close() })
|
||||||
|
return client, hostKey
|
||||||
|
}
|
||||||
|
|
||||||
|
// dropped is true when the server closed the client's connection within wait
|
||||||
|
func dropped(client *ssh.Client, wait time.Duration) bool {
|
||||||
|
done := make(chan struct{})
|
||||||
|
go func() {
|
||||||
|
_ = client.Wait()
|
||||||
|
close(done)
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-done:
|
||||||
|
return true
|
||||||
|
case <-time.After(wait):
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Restarting sshd drops every open session, the one that asked for the reload included. A reload that touches nothing
|
||||||
|
// sshd reads leaves it alone
|
||||||
|
func TestSSHDReload_UnchangedKeepsSessions(t *testing.T) {
|
||||||
|
h := newSSHDReloadHarness(t)
|
||||||
|
client, _ := h.connect(t)
|
||||||
|
|
||||||
|
require.NoError(t, h.c.ReloadConfigString(h.config()+"punchy:\n punch: true\n"))
|
||||||
|
assert.False(t, dropped(client, 500*time.Millisecond), "an unchanged sshd was restarted")
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSSHDReload_ChangedConfigRestarts(t *testing.T) {
|
||||||
|
h := newSSHDReloadHarness(t)
|
||||||
|
client, _ := h.connect(t)
|
||||||
|
|
||||||
|
h.extra = " trusted_cas: []\n"
|
||||||
|
require.NoError(t, h.c.ReloadConfigString(h.config()))
|
||||||
|
assert.True(t, dropped(client, 5*time.Second), "a changed sshd config was not applied")
|
||||||
|
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
|
||||||
|
h.connect(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The config can stay the same while the host key file it names changes. New connections get the new key, open
|
||||||
|
// sessions are left alone
|
||||||
|
func TestSSHDReload_ChangedHostKeyFile(t *testing.T) {
|
||||||
|
h := newSSHDReloadHarness(t)
|
||||||
|
client, _ := h.connect(t)
|
||||||
|
|
||||||
|
want := h.writeHostKey(t, newEd25519Key(t))
|
||||||
|
require.NoError(t, h.c.ReloadConfigString(h.config()))
|
||||||
|
_, got := h.connect(t)
|
||||||
|
assert.Equal(t, want.Marshal(), got.Marshal(), "a new host key was not applied")
|
||||||
|
assert.False(t, dropped(client, 500*time.Millisecond), "a new host key restarted sshd")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A host key of another algorithm replaces the old one rather than being offered next to it
|
||||||
|
func TestSSHDReload_HostKeyAlgorithmChange(t *testing.T) {
|
||||||
|
h := newSSHDReloadHarness(t)
|
||||||
|
h.connect(t)
|
||||||
|
|
||||||
|
rsaKey, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||||
|
require.NoError(t, err)
|
||||||
|
want := h.writeHostKey(t, rsaKey)
|
||||||
|
require.NoError(t, h.c.ReloadConfigString(h.config()))
|
||||||
|
|
||||||
|
_, got := h.connect(t)
|
||||||
|
assert.Equal(t, want.Marshal(), got.Marshal())
|
||||||
|
_, _, err = h.dial(ssh.KeyAlgoED25519)
|
||||||
|
assert.Error(t, err, "the old ed25519 host key is still offered")
|
||||||
|
}
|
||||||
|
|
||||||
|
// A server that isn't running, say its port was busy, is tried again on every reload as before
|
||||||
|
func TestSSHDReload_NotRunningRestarts(t *testing.T) {
|
||||||
|
h := newSSHDReloadHarness(t)
|
||||||
|
h.server.Stop()
|
||||||
|
require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond)
|
||||||
|
|
||||||
|
require.NoError(t, h.c.ReloadConfigString(h.config()))
|
||||||
|
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
|
||||||
|
h.connect(t)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A host key that no longer loads stops sshd, the same as a config that doesn't
|
||||||
|
func TestSSHDReload_BadHostKeyFileStops(t *testing.T) {
|
||||||
|
h := newSSHDReloadHarness(t)
|
||||||
|
h.connect(t)
|
||||||
|
|
||||||
|
require.NoError(t, os.WriteFile(h.keyFile, []byte("not a key"), 0o600))
|
||||||
|
require.NoError(t, h.c.ReloadConfigString(h.config()))
|
||||||
|
require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond)
|
||||||
|
}
|
||||||
Reference in new issue
Block a user