Compare commits

..
Author SHA1 Message Date
JackDoan 5bcceac53c linux: Don't replace another interface's route with the same destination and metric 2026-10-15 10:01:05 -05:00
John Maguire afbdc87ed9 Update gvisor and bump to go1.27 (#1940)
gvisor's arm64 race-build compare-and-swap (pkg/sync/race_arm64.s)
loaded the new value into the register meant for the old one (R1
instead of R2), so it never succeeded. Sleepers could not commit to
parking and spun: under -race on arm64 every idle service netstack
busied about 14 cores, and TestService's static host lookup of
localhost missed its 250ms timeout and stalled for the 30s static_map
cadence. Fixed in gvisor 1b2946c4fa.

Go 1.27.2 is now the minimum because gvisor's go branch requires it.
golangci-lint is bumped to v2.14, the first release that can read
go1.27.2 export data.
2026-10-14 21:30:59 -04:00
Nate Brown 94372f4f51 Don't delete routes we never installed on darwin and the BSDs (#1933) 2026-10-13 16:34:38 -05:00
John Maguire 42280f04ce darwin: Assign host address to the utun and route vpn explicitly (#1931)
Tailscale with an exit node treats every interface address prefix as an
attached LAN and either claims it or excludes it, and configd then rewrites
the matching route away from nebula's utun. Interfaces holding a single
address are skipped, so assign /32 and /128 and add the network route
ourselves for both families.
2026-10-13 13:50:37 -04:00
Nate Brown 11c1917387 Warn when an unsafe_route is already routed through another interface on darwin (#1925) 2026-10-12 16:57:14 -05:00
Nate Brown 5ba7aff4b3 ssh: Only restart sshd on reload when its config or host key changed (#1923) 2026-10-12 16:56:51 -05:00
Ryan Huber 8a2de9b1b5 examples: set socket buffers and raise tun tx_queue (#1928)
* examples: set socket buffers and raise tun tx_queue

* examples: use the default tcp max window for socket buffers

* examples: note buffers are linux only, clarify tx_queue default

* tun: default tx_queue to 5000
2026-10-11 00:30:07 -04:00
20 changed files with 746 additions and 220 deletions

No files matched your search

+3 -3
View File
@@ -14,7 +14,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -40,7 +40,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -83,7 +83,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
# GitHub holds ARNs, not credentials, and ARNs outlive a rotation # GitHub holds ARNs, not credentials, and ARNs outlive a rotation
+3 -3
View File
@@ -34,7 +34,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: add hashicorp source - name: add hashicorp source
@@ -66,7 +66,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: add hashicorp source - name: add hashicorp source
@@ -92,7 +92,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
# WSL2 + Ubuntu so the smoke can run a real linux peer with its own # WSL2 + Ubuntu so the smoke can run a real linux peer with its own
+2 -2
View File
@@ -22,7 +22,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: Smoke Docker - name: Smoke Docker
@@ -51,7 +51,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: build - name: build
+4 -4
View File
@@ -22,7 +22,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: Install goimports - name: Install goimports
@@ -42,7 +42,7 @@ jobs:
- name: golangci-lint - name: golangci-lint
uses: golangci/golangci-lint-action@v9 uses: golangci/golangci-lint-action@v9
with: with:
version: v2.12 version: v2.14
test: test:
name: Test ${{ matrix.name }} name: Test ${{ matrix.name }}
@@ -87,7 +87,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: Build - name: Build
@@ -132,7 +132,7 @@ jobs:
- uses: actions/setup-go@v7 - uses: actions/setup-go@v7
with: with:
go-version: '1.26' go-version: '1.27'
check-latest: true check-latest: true
- name: Build ${{ matrix.name }} - name: Build ${{ matrix.name }}
+5 -5
View File
@@ -135,12 +135,12 @@ listen:
# superpacket, so the receive scratch is batch * 64KiB per listening socket (~4MiB per routine at the # superpacket, so the receive scratch is batch * 64KiB per listening socket (~4MiB per routine at the
# default of 64). Lower this to trade peak per-syscall throughput for memory on constrained hosts. # default of 64). Lower this to trade peak per-syscall throughput for memory on constrained hosts.
#batch: 64 #batch: 64
# Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel # Linux only. Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel
# Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default) # Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default)
# Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide # Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide
# max, net.core.rmem_max and net.core.wmem_max # max, net.core.rmem_max and net.core.wmem_max
#read_buffer: 10485760 read_buffer: 3412512
#write_buffer: 10485760 write_buffer: 3412512
# On Windows only # On Windows only
# When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port. # When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port.
@@ -262,8 +262,8 @@ tun:
drop_local_broadcast: false drop_local_broadcast: false
# Toggles forwarding of multicast packets # Toggles forwarding of multicast packets
drop_multicast: false drop_multicast: false
# Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 500 # Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 5000
tx_queue: 500 tx_queue: 5000
# Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic # Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic
# Anything above 8937, here or on a route below, is capped to 8937 with a warning # Anything above 8937, here or on a route below, is capped to 8937 with a warning
mtu: 1300 mtu: 1300
+4 -3
View File
@@ -1,6 +1,6 @@
module github.com/slackhq/nebula module github.com/slackhq/nebula
go 1.26.0 go 1.27.2
require ( require (
dario.cat/mergo v1.0.2 dario.cat/mergo v1.0.2
@@ -33,7 +33,7 @@ require (
golang.zx2c4.com/wireguard/windows v1.0.1 golang.zx2c4.com/wireguard/windows v1.0.1
google.golang.org/protobuf v1.36.12 google.golang.org/protobuf v1.36.12
gopkg.in/yaml.v3 v3.0.1 gopkg.in/yaml.v3 v3.0.1
gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980
) )
require ( require (
@@ -45,5 +45,6 @@ require (
github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/common v0.70.1 // indirect
github.com/prometheus/procfs v0.21.1 // indirect github.com/prometheus/procfs v0.21.1 // indirect
github.com/vishvananda/netns v0.0.5 // indirect github.com/vishvananda/netns v0.0.5 // indirect
golang.org/x/time v0.5.0 // indirect golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc // indirect
golang.org/x/time v0.15.0 // indirect
) )
+6 -4
View File
@@ -157,6 +157,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc h1:TS73t7x3KarrNd5qAipmspBDS1rkMcgVG/fS1aRb4Rc=
golang.org/x/exp v0.0.0-20250711185948-6ae5c78190dc/go.mod h1:A+z0yzpGtvnG90cToK5n2tu8UJVP2XUATh+r+sfOOOc=
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -203,8 +205,8 @@ golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -243,5 +245,5 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe h1:fre4i6mv4iBuz5lCMOzHD1rH1ljqHWSICFmZRbbgp3g= gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980 h1:ribyrTYQ/QYeUIs8x8bVLYwGyepa7RnHLqJzegT5YYk=
gvisor.dev/gvisor v0.0.0-20240423190808-9d7a357edefe/go.mod h1:sxc3Uvk/vHcd3tj7/DHVBoR5wvWT/MmRq2pj7HRJnwU= gvisor.dev/gvisor v0.0.0-20261014193016-04d31b385980/go.mod h1:pK/dapF5Bf3KGW7/kd5CI5IC4lYh9FMFY1qETAIGLXk=
+1 -1
View File
@@ -43,7 +43,7 @@ type DeviceFactory func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix,
func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) { func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
switch { switch {
case c.GetBool("tun.disabled", false): case c.GetBool("tun.disabled", false):
tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l) tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 5000), c.GetBool("stats.message_metrics", false), l)
return tun, nil return tun, nil
default: default:
+98 -14
View File
@@ -6,8 +6,10 @@ import (
"errors" "errors"
"fmt" "fmt"
"log/slog" "log/slog"
"math/bits"
"net/netip" "net/netip"
"os" "os"
"sync"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -28,7 +30,7 @@ type tun struct {
DefaultMTU int DefaultMTU int
Routes atomic.Pointer[[]Route] Routes atomic.Pointer[[]Route]
routeTree atomic.Pointer[bart.Table[routing.Gateways]] routeTree atomic.Pointer[bart.Table[routing.Gateways]]
linkAddr *netroute.LinkAddr linkAddr atomic.Pointer[netroute.LinkAddr]
// hostOwned means the fd arrived from the OS, which has already configured addressing, mtu // hostOwned means the fd arrived from the OS, which has already configured addressing, mtu
// and routes for it. NEPacketTunnelProvider on darwin does this. // and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool hostOwned bool
@@ -243,7 +245,7 @@ func (t *tun) Activate() error {
if linkAddr == nil { if linkAddr == nil {
return fmt.Errorf("unable to discover link_addr for tun interface") return fmt.Errorf("unable to discover link_addr for tun interface")
} }
t.linkAddr = linkAddr t.linkAddr.Store(linkAddr)
for _, network := range t.vpnNetworks { for _, network := range t.vpnNetworks {
if network.Addr().Is4() { if network.Addr().Is4() {
@@ -269,6 +271,7 @@ func (t *tun) Activate() error {
return t.addRoutes(false) return t.addRoutes(false)
} }
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
func (t *tun) activate4(network netip.Prefix) error { func (t *tun) activate4(network netip.Prefix) error {
s, err := unix.Socket( s, err := unix.Socket(
unix.AF_INET, unix.AF_INET,
@@ -295,22 +298,32 @@ func (t *tun) activate4(network netip.Prefix) error {
MaskAddr: unix.RawSockaddrInet4{ MaskAddr: unix.RawSockaddrInet4{
Len: unix.SizeofSockaddrInet4, Len: unix.SizeofSockaddrInet4,
Family: unix.AF_INET, Family: unix.AF_INET,
Addr: prefixToMask(network).As4(), Addr: [4]byte{255, 255, 255, 255}, //always make the address a /32
}, },
} }
// On Darwin, VPNs don't edit the routing table directly.
// A network extension hands its routes to the system, and configd installs them.
// configd replaces any route already holding the same prefix, including ours.
// As an example, Tailscale with an exit node treats any interface prefix wider than one address as a LAN.
// With LAN access off, it routes each LAN into Tailscale's utun.
// With LAN access on, it excludes each LAN from its tunnel.
// configd routes an excluded prefix like its own routes would, ignoring ours, so it goes out the default route.
// Either way our route is replaced, and traffic for the vpn network no longer reaches nebula.
// A single address isn't a LAN to Tailscale, so our route is left alone.
// We route the vpn network to the tun explicitly instead.
if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil { if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun v4 address: %s", err) return fmt.Errorf("failed to set tun v4 address: %s", err)
} }
err = addRoute(network, t.linkAddr) if err := addRoute(network, t.linkAddr.Load()); err != nil {
if err != nil { return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err)
return err
} }
return nil return nil
} }
// activate6 is activate4 for IPv6
func (t *tun) activate6(network netip.Prefix) error { func (t *tun) activate6(network netip.Prefix) error {
s, err := unix.Socket( s, err := unix.Socket(
unix.AF_INET6, unix.AF_INET6,
@@ -332,20 +345,24 @@ func (t *tun) activate6(network netip.Prefix) error {
PrefixMask: unix.RawSockaddrInet6{ PrefixMask: unix.RawSockaddrInet6{
Len: unix.SizeofSockaddrInet6, Len: unix.SizeofSockaddrInet6,
Family: unix.AF_INET6, Family: unix.AF_INET6,
Addr: prefixToMask(network).As16(), // Always set the netmask to be a /128. See activate4 for why we do this.
Addr: [16]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
}, },
Lifetime: addrLifetime{ Lifetime: addrLifetime{
// never expires // never expires
Vltime: 0xffffffff, Vltime: 0xffffffff,
Pltime: 0xffffffff, Pltime: 0xffffffff,
}, },
Flags: _IN6_IFF_NODAD, Flags: _IN6_IFF_NODAD, // No DAD, a utun has no neighbors.
} }
if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil { if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun address: %s", err) return fmt.Errorf("failed to set tun address: %s", err)
} }
if err := addRoute(network, t.linkAddr.Load()); err != nil {
return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err)
}
return nil return nil
} }
@@ -394,6 +411,56 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
return routing.Gateways{} return routing.Gateways{}
} }
func fetchRoutes() ([]netroute.Message, error) {
rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_DUMP, 0)
if err != nil {
return nil, err
}
return netroute.ParseRIB(unix.NET_RT_DUMP, rib)
}
// routeInterfaceIn returns the index of the interface prefix is routed through, or 0 if msgs has no route for exactly
// that prefix. Routes scoped to an interface are skipped, they don't stop us adding ours.
func routeInterfaceIn(msgs []netroute.Message, prefix netip.Prefix) int {
for _, msg := range msgs {
m, ok := msg.(*netroute.RouteMessage)
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK || m.Flags&unix.RTF_IFSCOPE != 0 {
continue
}
if p, ok := routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK]); ok && p == prefix.Masked() {
return m.Index
}
}
return 0
}
// routePrefix returns the prefix a route covers. Host routes have no netmask.
func routePrefix(dst, mask netroute.Addr) (netip.Prefix, bool) {
switch d := dst.(type) {
case *netroute.Inet4Addr:
bits := 32
if m, ok := mask.(*netroute.Inet4Addr); ok {
bits = maskBits(m.IP[:])
}
return netip.PrefixFrom(netip.AddrFrom4(d.IP), bits), true
case *netroute.Inet6Addr:
bits := 128
if m, ok := mask.(*netroute.Inet6Addr); ok {
bits = maskBits(m.IP[:])
}
return netip.PrefixFrom(netip.AddrFrom16(d.IP), bits), true
}
return netip.Prefix{}, false
}
func maskBits(mask []byte) int {
n := 0
for _, b := range mask {
n += bits.OnesCount8(b)
}
return n
}
// Get the LinkAddr for the interface of the given name // Get the LinkAddr for the interface of the given name
// Is there an easier way to fetch this when we create the interface? // Is there an easier way to fetch this when we create the interface?
// Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers. // Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers.
@@ -428,7 +495,14 @@ func (t *tun) addRoutes(logErrors bool) error {
return nil return nil
} }
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil {
return nil
}
routes := *t.Routes.Load() routes := *t.Routes.Load()
systemRoutes := sync.OnceValues(fetchRoutes)
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if len(r.Via) == 0 || !r.Install {
@@ -436,10 +510,16 @@ func (t *tun) addRoutes(logErrors bool) error {
continue continue
} }
err := addRoute(r.Cidr, t.linkAddr) err := addRoute(r.Cidr, linkAddr)
if err != nil { if err != nil {
if errors.Is(err, unix.EEXIST) { if errors.Is(err, unix.EEXIST) {
t.l.Warn("unable to add unsafe_route, identical route already exists", "route", r.Cidr) // Every route we added already exists on a reload, only warn about someone else's
msgs, err := systemRoutes()
if err != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
}
} else { } else {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
@@ -450,6 +530,8 @@ func (t *tun) addRoutes(logErrors bool) error {
} }
} else { } else {
t.l.Info("Added route", "route", r) t.l.Info("Added route", "route", r)
// A cidr listed twice would otherwise find our own route missing from an older fetch
systemRoutes = sync.OnceValues(fetchRoutes)
} }
} }
@@ -457,16 +539,18 @@ func (t *tun) addRoutes(logErrors bool) error {
} }
func (t *tun) removeRoutes(routes []Route) error { func (t *tun) removeRoutes(routes []Route) error {
if t.hostOwned { linkAddr := t.linkAddr.Load()
if t.hostOwned || linkAddr == nil {
return nil return nil
} }
for _, r := range routes { for _, r := range routes {
if !r.Install { if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue continue
} }
err := delRoute(r.Cidr, t.linkAddr) err := delRoute(r.Cidr, linkAddr)
if err != nil { if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
} else { } else {
+103
View File
@@ -0,0 +1,103 @@
//go:build darwin && !ios && !e2e_testing
package overlay
import (
"net"
"net/netip"
"os"
"os/exec"
"strings"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
netroute "golang.org/x/net/route"
"golang.org/x/sys/unix"
)
// Every mac routes loopback through lo0
func TestRouteInterfaceIn(t *testing.T) {
msgs, err := fetchRoutes()
require.NoError(t, err)
routeInterface := func(s string) int { return routeInterfaceIn(msgs, netip.MustParsePrefix(s)) }
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
assert.Equal(t, lo0.Index, routeInterface("127.0.0.0/8"))
assert.Equal(t, lo0.Index, routeInterface("127.0.0.1/32"), "a host route has no netmask")
assert.Equal(t, lo0.Index, routeInterface("::1/128"))
assert.Equal(t, 0, routeInterface("127.0.0.0/16"), "only an exact prefix counts")
// Most macs also list the default route scoped to their other interfaces
out, err := exec.Command("route", "-n", "get", "default").Output()
require.NoError(t, err)
var name string
for _, line := range strings.Split(string(out), "\n") {
if v, ok := strings.CutPrefix(strings.TrimSpace(line), "interface: "); ok {
name = v
}
}
if name == "" {
t.Skip("no default route")
}
def, err := net.InterfaceByName(name)
require.NoError(t, err)
assert.Equal(t, def.Index, routeInterface("0.0.0.0/0"))
}
func TestRouteInterfaceIn_SkipsScopedCopies(t *testing.T) {
route := func(index, flags int) netroute.Message {
addrs := make([]netroute.Addr, unix.RTAX_MAX)
addrs[unix.RTAX_DST] = &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}}
addrs[unix.RTAX_NETMASK] = &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}}
return &netroute.RouteMessage{Index: index, Flags: flags, Addrs: addrs}
}
p := netip.MustParsePrefix("192.168.1.0/24")
assert.Equal(t, 7, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE), route(7, unix.RTF_UP)}, p))
assert.Equal(t, 0, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE)}, p))
}
// A reload can happen before Activate
func TestTunRoutesBeforeActivate(t *testing.T) {
tn := &tun{l: test.NewLogger()}
routes := []Route{{
Cidr: netip.MustParsePrefix("10.253.1.0/24"),
Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)},
Install: true,
}}
tn.Routes.Store(&routes)
assert.NoError(t, tn.addRoutes(true))
assert.NoError(t, tn.removeRoutes(routes))
}
// A tun.routes entry with no via is never installed, so removing it must not delete a route for the same cidr that
// belongs to someone else
func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating a utun and adding routes needs root")
}
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
cidr := "10.250.0.128/25"
require.NoError(t, exec.Command("route", "-q", "-n", "add", "-net", cidr, "-interface", "lo0").Run())
t.Cleanup(func() { _ = exec.Command("route", "-q", "-n", "delete", "-net", cidr).Run() })
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("tun:\n routes:\n - route: "+cidr+"\n mtu: 1300\n"))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
require.NoError(t, c.ReloadConfigString("tun: {}\n"))
msgs, err := fetchRoutes()
require.NoError(t, err)
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
}
+2 -1
View File
@@ -589,7 +589,8 @@ func (t *tun) addRoutes(logErrors bool) error {
func (t *tun) removeRoutes(routes []Route) error { func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes { for _, r := range routes {
if !r.Install { if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue continue
} }
+97 -4
View File
@@ -221,7 +221,7 @@ func newTunGeneric(c *config.C, l *slog.Logger, fd int, vnetHdr bool, offloadFla
vnetHdr: vnetHdr, vnetHdr: vnetHdr,
offloadFlags: offloadFlags, offloadFlags: offloadFlags,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
TXQueueLen: c.GetInt("tun.tx_queue", 500), TXQueueLen: c.GetInt("tun.tx_queue", 5000),
useSystemRoutes: c.GetBool("tun.use_system_route_table", false), useSystemRoutes: c.GetBool("tun.use_system_route_table", false),
useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0), useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0),
routesFromSystem: map[netip.Prefix]routing.Gateways{}, routesFromSystem: map[netip.Prefix]routing.Gateways{},
@@ -552,13 +552,17 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
if cidr.Addr().Is6() { if cidr.Addr().Is6() {
nr.Priority = 256 nr.Priority = 256
} }
err := netlink.RouteReplace(&nr) held, err := t.otherLinkRoutes()
if err != nil {
return err
}
err = t.replaceRoute(&nr, held)
if err != nil { if err != nil {
t.l.Warn("Failed to set default route MTU, retrying", "error", err, "cidr", cidr) t.l.Warn("Failed to set default route MTU, retrying", "error", err, "cidr", cidr)
//retry twice more -- on some systems there appears to be a race condition where if we set routes too soon, netlink says `invalid argument` //retry twice more -- on some systems there appears to be a race condition where if we set routes too soon, netlink says `invalid argument`
for range 2 { for range 2 {
time.Sleep(100 * time.Millisecond) time.Sleep(100 * time.Millisecond)
err = netlink.RouteReplace(&nr) err = t.replaceRoute(&nr, held)
if err == nil { if err == nil {
break break
} else { } else {
@@ -578,6 +582,15 @@ func (t *tun) setDefaultRoute(cidr netip.Prefix) error {
} }
func (t *tun) addRoutes(logErrors bool) error { func (t *tun) addRoutes(logErrors bool) error {
held, err := t.otherLinkRoutes()
if err != nil {
if logErrors {
t.l.Error("Failed to add routes", "error", err)
return nil
}
return err
}
// Path routes // Path routes
routes := *t.Routes.Load() routes := *t.Routes.Load()
for _, r := range routes { for _, r := range routes {
@@ -602,7 +615,7 @@ func (t *tun) addRoutes(logErrors bool) error {
nr.Priority = r.Metric nr.Priority = r.Metric
} }
err := netlink.RouteReplace(&nr) err := t.replaceRoute(&nr, held)
if err != nil { if err != nil {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
@@ -618,6 +631,86 @@ func (t *tun) addRoutes(logErrors bool) error {
return nil return nil
} }
// ip6RoutePrioUser is the metric the kernel gives an IPv6 route added without one
const ip6RoutePrioUser = 1024
// routeKey is where the main table files a route. Replacing a route replaces the first one with the same key, whatever
// device it is on
type routeKey struct {
dst netip.Prefix
priority int
}
func routeKeyOf(dst *net.IPNet, family, priority int) (routeKey, bool) {
var p netip.Prefix
if dst == nil {
// A default route
if family == unix.AF_INET6 {
p = netip.PrefixFrom(netip.IPv6Unspecified(), 0)
} else {
p = netip.PrefixFrom(netip.IPv4Unspecified(), 0)
}
} else {
addr, ok := netip.AddrFromSlice(dst.IP)
if !ok {
return routeKey{}, false
}
ones, _ := dst.Mask.Size()
p = netip.PrefixFrom(addr.Unmap(), ones).Masked()
}
if priority == 0 && p.Addr().Is6() {
priority = ip6RoutePrioUser
}
return routeKey{dst: p, priority: priority}, true
}
// otherLinkRoutes returns the keys of the main table routes that are not on our device
func (t *tun) otherLinkRoutes() (map[routeKey]struct{}, error) {
routes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, &netlink.Route{Table: unix.RT_TABLE_MAIN}, netlink.RT_FILTER_TABLE)
// An interrupted dump may miss a route, which then gets replaced like it always used to be
if err != nil && !errors.Is(err, netlink.ErrDumpInterrupted) {
return nil, fmt.Errorf("failed to list the system routes: %w", err)
}
held := make(map[routeKey]struct{}, len(routes))
for _, r := range routes {
if r.LinkIndex == t.deviceIndex {
continue
}
if k, ok := routeKeyOf(r.Dst, r.Family, r.Priority); ok {
held[k] = struct{}{}
}
}
return held, nil
}
// replaceRoute puts nr in the main table in place of our own route with its key. Replacing would take the route of
// another device holding that key, so ours goes in behind it instead. Theirs keeps routing and the kernel falls back
// to ours once theirs is removed, like when leaving the lan that had it. A different metric decides which one wins.
func (t *tun) replaceRoute(nr *netlink.Route, held map[routeKey]struct{}) error {
family := unix.AF_INET6
if nr.Dst.IP.To4() != nil {
family = unix.AF_INET
}
k, ok := routeKeyOf(nr.Dst, family, nr.Priority)
if _, taken := held[k]; !ok || !taken {
return netlink.RouteReplace(nr)
}
// Appending doesn't update a route that is already ours, it adds a second one
ours := netlink.Route{
LinkIndex: nr.LinkIndex,
Dst: nr.Dst,
Scope: nr.Scope,
Table: nr.Table,
Priority: k.priority,
}
if err := netlink.RouteDel(&ours); err != nil && !errors.Is(err, unix.ESRCH) {
return err
}
t.l.Warn("Another interface already routes this destination at the same metric, our route is used once theirs is removed", "destination", k.dst, "metric", k.priority)
return netlink.RouteAppend(nr)
}
func (t *tun) removeRoutes(routes []Route) { func (t *tun) removeRoutes(routes []Route) {
for _, r := range routes { for _, r := range routes {
if !r.Install { if !r.Install {
+133
View File
@@ -0,0 +1,133 @@
//go:build linux && !android && !e2e_testing
package overlay
import (
"net"
"net/netip"
"os"
"testing"
"github.com/DefinedNet/netlink"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sys/unix"
)
// An unsafe_route with the same destination and metric as a lan's route must not replace it. Ours waits behind theirs
// and takes over when the lan goes away. Run as root, or unprivileged with `unshare -rn`.
func TestTunRouteLeavesAnotherDevicesRoute(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating links and adding routes needs root")
}
lan := &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: "nebrt0"}, PeerName: "nebrt1"}
require.NoError(t, netlink.LinkAdd(lan))
t.Cleanup(func() { _ = netlink.LinkDel(lan) })
peer, err := netlink.LinkByName("nebrt1")
require.NoError(t, err)
require.NoError(t, netlink.LinkSetUp(lan))
require.NoError(t, netlink.LinkSetUp(peer))
lanAddrs := []*netlink.Addr{
{IPNet: &net.IPNet{IP: net.ParseIP("198.51.100.5"), Mask: net.CIDRMask(24, 32)}},
{IPNet: &net.IPNet{IP: net.ParseIP("2001:db8:1::5"), Mask: net.CIDRMask(64, 128)}, Flags: unix.IFA_F_NODAD},
}
for _, a := range lanAddrs {
require.NoError(t, netlink.AddrAdd(lan, a))
}
l := test.NewLogger()
c := config.NewC(l)
// The kernel puts a v4 lan route at metric 0, our default, and a v6 one at 256
require.NoError(t, c.LoadString(`
tun:
dev: nebrt2
unsafe_routes:
- route: 198.51.100.0/24
via: 10.250.0.2
- route: 2001:db8:1::/64
via: fd00:250::2
metric: 256
`))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
via := func(dst string) int {
routes, err := netlink.RouteGet(net.ParseIP(dst))
require.NoError(t, err)
require.NotEmpty(t, routes)
return routes[0].LinkIndex
}
devices := func(dst string) []int {
_, n, err := net.ParseCIDR(dst)
require.NoError(t, err)
routes, err := netlink.RouteListFiltered(netlink.FAMILY_ALL, &netlink.Route{Dst: n}, netlink.RT_FILTER_DST)
require.NoError(t, err)
var out []int
for _, r := range routes {
out = append(out, r.LinkIndex)
}
return out
}
for _, dst := range []string{"198.51.100.0/24", "2001:db8:1::/64"} {
assert.ElementsMatch(t, []int{lan.Index, tn.deviceIndex}, devices(dst), dst)
}
assert.Equal(t, lan.Index, via("198.51.100.9"))
assert.Equal(t, lan.Index, via("2001:db8:1::9"))
// Leaving the lan
for _, a := range lanAddrs {
require.NoError(t, netlink.AddrDel(lan, a))
}
assert.Equal(t, tn.deviceIndex, via("198.51.100.9"))
assert.Equal(t, tn.deviceIndex, via("2001:db8:1::9"))
// Dropping the unsafe_routes removes only ours
for _, a := range lanAddrs {
require.NoError(t, netlink.AddrAdd(lan, a))
}
require.NoError(t, c.ReloadConfigString("tun:\n dev: nebrt2\n"))
for _, dst := range []string{"198.51.100.0/24", "2001:db8:1::/64"} {
assert.Equal(t, []int{lan.Index}, devices(dst), dst)
}
}
// The route for our own vpn network is replaced to set its mtu, which must not take a lan's route for the same network
func TestTunVpnNetworkLeavesAnotherDevicesRoute(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating links and adding routes needs root")
}
lan := &netlink.Veth{LinkAttrs: netlink.LinkAttrs{Name: "nebrt0"}, PeerName: "nebrt1"}
require.NoError(t, netlink.LinkAdd(lan))
t.Cleanup(func() { _ = netlink.LinkDel(lan) })
require.NoError(t, netlink.LinkSetUp(lan))
require.NoError(t, netlink.AddrAdd(lan, &netlink.Addr{IPNet: &net.IPNet{IP: net.ParseIP("10.250.0.5"), Mask: net.CIDRMask(24, 32)}}))
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("tun:\n dev: nebrt2\n"))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
_, n, err := net.ParseCIDR("10.250.0.0/24")
require.NoError(t, err)
routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, &netlink.Route{Dst: n}, netlink.RT_FILTER_DST)
require.NoError(t, err)
var devices []int
for _, r := range routes {
devices = append(devices, r.LinkIndex)
if r.LinkIndex == tn.deviceIndex {
assert.Equal(t, tn.DefaultMTU, r.MTU, "ours still carries the mtu")
}
}
assert.ElementsMatch(t, []int{lan.Index, tn.deviceIndex}, devices)
}
+2 -1
View File
@@ -420,7 +420,8 @@ func (t *tun) addRoutes(logErrors bool) error {
func (t *tun) removeRoutes(routes []Route) error { func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes { for _, r := range routes {
if !r.Install { if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue continue
} }
+2 -1
View File
@@ -399,7 +399,8 @@ func (t *tun) addRoutes(logErrors bool) error {
func (t *tun) removeRoutes(routes []Route) error { func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes { for _, r := range routes {
if !r.Install { if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue continue
} }
+33 -13
View File
@@ -60,6 +60,20 @@ type sshDeviceInfoFlags struct {
func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) { func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C) {
if c.GetBool("sshd.enabled", false) { if c.GetBool("sshd.enabled", false) {
// Restarting the server drops every open session, the one that asked for this reload included. Only restart
// when it isn't running or its config changed. The host key file can change under the same config, new
// handshakes use the new key without a restart
if ssh.Running() && !c.HasChanged("sshd") {
key, err := loadSSHHostKey(c)
if err == nil {
err = ssh.SetHostKey(key)
}
if err != nil {
l.Error("Failed to reconfigure the sshd", "error", err)
ssh.Stop()
}
return
}
sshRun, err := configSSH(l, ssh, c) sshRun, err := configSSH(l, ssh, c)
if err != nil { if err != nil {
l.Error("Failed to reconfigure the sshd", "error", err) l.Error("Failed to reconfigure the sshd", "error", err)
@@ -74,6 +88,22 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
}) })
} }
// loadSSHHostKey returns sshd.host_key, read from its file when it names one.
func loadSSHHostKey(c *config.C) ([]byte, error) {
hostKeyPathOrKey := c.GetString("sshd.host_key", "")
if hostKeyPathOrKey == "" {
return nil, fmt.Errorf("sshd.host_key must be provided")
}
if strings.Contains(hostKeyPathOrKey, "-----BEGIN") {
return []byte(hostKeyPathOrKey), nil
}
b, err := os.ReadFile(hostKeyPathOrKey)
if err != nil {
return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err)
}
return b, nil
}
// configSSH reads the ssh info out of the passed-in Config and // configSSH reads the ssh info out of the passed-in Config and
// updates the passed-in SSHServer. On success, it returns a function // updates the passed-in SSHServer. On success, it returns a function
// that callers may invoke to run the configured ssh server. On // that callers may invoke to run the configured ssh server. On
@@ -92,19 +122,9 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
return nil, fmt.Errorf("sshd.listen can not use port 22") return nil, fmt.Errorf("sshd.listen can not use port 22")
} }
hostKeyPathOrKey := c.GetString("sshd.host_key", "") hostKeyBytes, err := loadSSHHostKey(c)
if hostKeyPathOrKey == "" { if err != nil {
return nil, fmt.Errorf("sshd.host_key must be provided") return nil, err
}
var hostKeyBytes []byte
if strings.Contains(hostKeyPathOrKey, "-----BEGIN") {
hostKeyBytes = []byte(hostKeyPathOrKey)
} else {
hostKeyBytes, err = os.ReadFile(hostKeyPathOrKey)
if err != nil {
return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err)
}
} }
err = ssh.SetHostKey(hostKeyBytes) err = ssh.SetHostKey(hostKeyBytes)
+48 -13
View File
@@ -8,14 +8,18 @@ import (
"log/slog" "log/slog"
"net" "net"
"sync" "sync"
"sync/atomic"
"github.com/armon/go-radix" "github.com/armon/go-radix"
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
) )
type SSHServer struct { type SSHServer struct {
config *ssh.ServerConfig // config is replaced whole when the host key changes, ssh.ServerConfig.AddHostKey only replaces a key of the same
l *slog.Logger // algorithm and appends any other
config atomic.Pointer[ssh.ServerConfig]
authenticate func(ssh.ConnMetadata, ssh.PublicKey) (*ssh.Permissions, error)
l *slog.Logger
certChecker *ssh.CertChecker certChecker *ssh.CertChecker
@@ -28,7 +32,12 @@ type SSHServer struct {
// List of available commands // List of available commands
helpCommand *Command helpCommand *Command
commands *radix.Tree commands *radix.Tree
listener net.Listener listener atomic.Pointer[net.Listener]
// runs counts the Runs that are listening. A reload's Run can start before the one it replaced has returned
runs atomic.Int32
// addr is where the newest Run listens, nil when none is
addr atomic.Pointer[net.Addr]
// ctx parents per-Run contexts. Cancelling it (e.g. via Control.Stop) tears the server down even // ctx parents per-Run contexts. Cancelling it (e.g. via Control.Stop) tears the server down even
// across reloads, since each Run derives a fresh child rather than reusing this one directly. // across reloads, since each Run derives a fresh child rather than reusing this one directly.
@@ -85,10 +94,8 @@ func NewSSHServer(ctx context.Context, l *slog.Logger) (*SSHServer, error) {
}, },
} }
s.config = &ssh.ServerConfig{ s.authenticate = cc.Authenticate
PublicKeyCallback: cc.Authenticate, s.config.Store(s.newConfig())
ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"),
}
s.RegisterCommand(&Command{ s.RegisterCommand(&Command{
Name: "help", Name: "help",
@@ -107,10 +114,32 @@ func (s *SSHServer) SetHostKey(hostPrivateKey []byte) error {
return fmt.Errorf("failed to parse private key: %s", err) return fmt.Errorf("failed to parse private key: %s", err)
} }
s.config.AddHostKey(private) config := s.newConfig()
config.AddHostKey(private)
s.config.Store(config)
return nil return nil
} }
func (s *SSHServer) newConfig() *ssh.ServerConfig {
return &ssh.ServerConfig{
PublicKeyCallback: s.authenticate,
ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"),
}
}
// Addr is the address the newest Run listens on, nil when none is.
func (s *SSHServer) Addr() net.Addr {
if a := s.addr.Load(); a != nil {
return *a
}
return nil
}
// Running is true while a Run is listening.
func (s *SSHServer) Running() bool {
return s.runs.Load() > 0
}
func (s *SSHServer) ClearTrustedCAs() { func (s *SSHServer) ClearTrustedCAs() {
s.authLock.Lock() s.authLock.Lock()
s.trustedCAs = []ssh.PublicKey{} s.trustedCAs = []ssh.PublicKey{}
@@ -181,10 +210,16 @@ func (s *SSHServer) Run(addr string) error {
// this run owns. They start equal but a fast reload may overwrite s.listener with the next run's // this run owns. They start equal but a fast reload may overwrite s.listener with the next run's
// listener before this run's watcher fires, so each run must close its own listener via the local // listener before this run's watcher fires, so each run must close its own listener via the local
// reference. // reference.
s.listener = listener s.listener.Store(&listener)
s.runs.Add(1)
bound := listener.Addr()
s.addr.Store(&bound)
runCtx, cancel := context.WithCancel(s.ctx) runCtx, cancel := context.WithCancel(s.ctx)
defer cancel() defer cancel()
// Deferred after cancel so they run first, a run on its way out no longer counts as listening
defer s.runs.Add(-1)
defer s.addr.CompareAndSwap(&bound, nil)
// Close the listener when this run's context is cancelled. That can come from the parent // Close the listener when this run's context is cancelled. That can come from the parent
// (Control.Stop), from Run returning normally (defer cancel above), or transitively when a sibling // (Control.Stop), from Run returning normally (defer cancel above), or transitively when a sibling
@@ -197,7 +232,7 @@ func (s *SSHServer) Run(addr string) error {
} }
}() }()
s.l.Info("SSH server is listening", "sshListener", addr) s.l.Info("SSH server is listening", "sshListener", bound)
// Run loops until there is an error // Run loops until there is an error
s.run(runCtx, listener) s.run(runCtx, listener)
@@ -226,7 +261,7 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
<-sessionContext.Done() <-sessionContext.Done()
c.Close() c.Close()
}() }()
conn, chans, reqs, err := ssh.NewServerConn(c, s.config) conn, chans, reqs, err := ssh.NewServerConn(c, s.config.Load())
fp := "" fp := ""
if conn != nil { if conn != nil {
fp = conn.Permissions.Extensions["fp"] fp = conn.Permissions.Extensions["fp"]
@@ -264,8 +299,8 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
} }
func (s *SSHServer) Stop() { func (s *SSHServer) Stop() {
if s.listener != nil { if l := s.listener.Load(); l != nil {
if err := s.listener.Close(); err != nil { if err := (*l).Close(); err != nil && !errors.Is(err, net.ErrClosed) {
s.l.Warn("Failed to close the sshd listener", "error", err) s.l.Warn("Failed to close the sshd listener", "error", err)
} }
} }
+196
View File
@@ -0,0 +1,196 @@
package nebula
import (
"crypto"
"crypto/ed25519"
"crypto/rand"
"crypto/rsa"
"encoding/pem"
"fmt"
"net"
"os"
"path/filepath"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/sshd"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/crypto/ssh"
)
type sshdReloadHarness struct {
c *config.C
server *sshd.SSHServer
keyFile string
signer ssh.Signer
extra string
}
func newSSHDReloadHarness(t *testing.T) *sshdReloadHarness {
t.Helper()
l := test.NewLogger()
h := &sshdReloadHarness{keyFile: filepath.Join(t.TempDir(), "host_key")}
h.writeHostKey(t, newEd25519Key(t))
_, userKey, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
h.signer, err = ssh.NewSignerFromKey(userKey)
require.NoError(t, err)
h.c = config.NewC(l)
require.NoError(t, h.c.LoadString(h.config()))
h.server, err = sshd.NewSSHServer(t.Context(), l)
require.NoError(t, err)
wireSSHReload(l, h.server, h.c)
run, err := configSSH(l, h.server, h.c)
require.NoError(t, err)
go run()
t.Cleanup(h.server.Stop)
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
return h
}
func newEd25519Key(t *testing.T) crypto.Signer {
_, key, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
return key
}
// writeHostKey replaces the host key file and returns the public key a client should now see
func (h *sshdReloadHarness) writeHostKey(t *testing.T, key crypto.Signer) ssh.PublicKey {
t.Helper()
block, err := ssh.MarshalPrivateKey(key, "")
require.NoError(t, err)
require.NoError(t, os.WriteFile(h.keyFile, pem.EncodeToMemory(block), 0o600))
pub, err := ssh.NewPublicKey(key.Public())
require.NoError(t, err)
return pub
}
func (h *sshdReloadHarness) config() string {
return fmt.Sprintf("sshd:\n enabled: true\n listen: 127.0.0.1:0\n host_key: %s\n authorized_users:\n - user: test\n keys: [%q]\n%s",
h.keyFile, string(ssh.MarshalAuthorizedKey(h.signer.PublicKey())), h.extra)
}
// dial connects to wherever the server listens now, a restart on :0 moves it. hostKeyAlgorithms limits which host keys
// the client accepts, none means any
func (h *sshdReloadHarness) dial(hostKeyAlgorithms ...string) (*ssh.Client, ssh.PublicKey, error) {
addr := h.server.Addr()
if addr == nil {
return nil, nil, fmt.Errorf("not listening")
}
var hostKey ssh.PublicKey
c, err := ssh.Dial("tcp", addr.String(), &ssh.ClientConfig{
User: "test",
Auth: []ssh.AuthMethod{ssh.PublicKeys(h.signer)},
HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error {
hostKey = key
return nil
},
HostKeyAlgorithms: hostKeyAlgorithms,
Timeout: time.Second,
})
return c, hostKey, err
}
func (h *sshdReloadHarness) connect(t *testing.T) (*ssh.Client, ssh.PublicKey) {
t.Helper()
var client *ssh.Client
var hostKey ssh.PublicKey
require.Eventually(t, func() bool {
var err error
client, hostKey, err = h.dial()
return err == nil
}, 5*time.Second, 20*time.Millisecond)
t.Cleanup(func() { _ = client.Close() })
return client, hostKey
}
// dropped is true when the server closed the client's connection within wait
func dropped(client *ssh.Client, wait time.Duration) bool {
done := make(chan struct{})
go func() {
_ = client.Wait()
close(done)
}()
select {
case <-done:
return true
case <-time.After(wait):
return false
}
}
// Restarting sshd drops every open session, the one that asked for the reload included. A reload that touches nothing
// sshd reads leaves it alone
func TestSSHDReload_UnchangedKeepsSessions(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
require.NoError(t, h.c.ReloadConfigString(h.config()+"punchy:\n punch: true\n"))
assert.False(t, dropped(client, 500*time.Millisecond), "an unchanged sshd was restarted")
}
func TestSSHDReload_ChangedConfigRestarts(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
h.extra = " trusted_cas: []\n"
require.NoError(t, h.c.ReloadConfigString(h.config()))
assert.True(t, dropped(client, 5*time.Second), "a changed sshd config was not applied")
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t)
}
// The config can stay the same while the host key file it names changes. New connections get the new key, open
// sessions are left alone
func TestSSHDReload_ChangedHostKeyFile(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
want := h.writeHostKey(t, newEd25519Key(t))
require.NoError(t, h.c.ReloadConfigString(h.config()))
_, got := h.connect(t)
assert.Equal(t, want.Marshal(), got.Marshal(), "a new host key was not applied")
assert.False(t, dropped(client, 500*time.Millisecond), "a new host key restarted sshd")
}
// A host key of another algorithm replaces the old one rather than being offered next to it
func TestSSHDReload_HostKeyAlgorithmChange(t *testing.T) {
h := newSSHDReloadHarness(t)
h.connect(t)
rsaKey, err := rsa.GenerateKey(rand.Reader, 2048)
require.NoError(t, err)
want := h.writeHostKey(t, rsaKey)
require.NoError(t, h.c.ReloadConfigString(h.config()))
_, got := h.connect(t)
assert.Equal(t, want.Marshal(), got.Marshal())
_, _, err = h.dial(ssh.KeyAlgoED25519)
assert.Error(t, err, "the old ed25519 host key is still offered")
}
// A server that isn't running, say its port was busy, is tried again on every reload as before
func TestSSHDReload_NotRunningRestarts(t *testing.T) {
h := newSSHDReloadHarness(t)
h.server.Stop()
require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond)
require.NoError(t, h.c.ReloadConfigString(h.config()))
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t)
}
// A host key that no longer loads stops sshd, the same as a config that doesn't
func TestSSHDReload_BadHostKeyFileStops(t *testing.T) {
h := newSSHDReloadHarness(t)
h.connect(t)
require.NoError(t, os.WriteFile(h.keyFile, []byte("not a key"), 0o600))
require.NoError(t, h.c.ReloadConfigString(h.config()))
require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond)
}
+4 -24
View File
@@ -135,23 +135,12 @@ func (u *StdConn) Rebind() error {
return nil return nil
} }
// SetRecvBuffer tries SO_RCVBUFFORCE, which needs CAP_NET_ADMIN. Without it we
// fall back to SO_RCVBUF, which the kernel clamps to net.core.rmem_max.
func (u *StdConn) SetRecvBuffer(n int) error { func (u *StdConn) SetRecvBuffer(n int) error {
err := unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUFFORCE, n) return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUFFORCE, n)
if errors.Is(err, unix.EPERM) {
err = unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_RCVBUF, n)
}
return err
} }
// SetSendBuffer is SetRecvBuffer for SO_SNDBUFFORCE/SO_SNDBUF and net.core.wmem_max.
func (u *StdConn) SetSendBuffer(n int) error { func (u *StdConn) SetSendBuffer(n int) error {
err := unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUFFORCE, n) return unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUFFORCE, n)
if errors.Is(err, unix.EPERM) {
err = unix.SetsockoptInt(u.sysFd, unix.SOL_SOCKET, unix.SO_SNDBUF, n)
}
return err
} }
func (u *StdConn) SetSoMark(mark int) error { func (u *StdConn) SetSoMark(mark int) error {
@@ -407,13 +396,8 @@ func (u *StdConn) ReloadConfig(c *config.C) {
b := c.GetInt("listen.read_buffer", 0) b := c.GetInt("listen.read_buffer", 0)
if b > 0 { if b > 0 {
if err := u.SetRecvBuffer(b); err == nil { if err := u.SetRecvBuffer(b); err == nil {
// The kernel reports double the requested size if s, err := u.GetRecvBuffer(); err == nil {
if s, err := u.GetRecvBuffer(); err == nil && s >= 2*b {
u.l.Info("listen.read_buffer was set", "size", s) u.l.Info("listen.read_buffer was set", "size", s)
} else if err == nil {
u.l.Warn("listen.read_buffer was limited by the system",
"requested", b, "size", s,
"hint", "raise net.core.rmem_max or grant CAP_NET_ADMIN")
} else { } else {
u.l.Warn("Failed to get listen.read_buffer", "error", err) u.l.Warn("Failed to get listen.read_buffer", "error", err)
} }
@@ -425,12 +409,8 @@ func (u *StdConn) ReloadConfig(c *config.C) {
b = c.GetInt("listen.write_buffer", 0) b = c.GetInt("listen.write_buffer", 0)
if b > 0 { if b > 0 {
if err := u.SetSendBuffer(b); err == nil { if err := u.SetSendBuffer(b); err == nil {
if s, err := u.GetSendBuffer(); err == nil && s >= 2*b { if s, err := u.GetSendBuffer(); err == nil {
u.l.Info("listen.write_buffer was set", "size", s) u.l.Info("listen.write_buffer was set", "size", s)
} else if err == nil {
u.l.Warn("listen.write_buffer was limited by the system",
"requested", b, "size", s,
"hint", "raise net.core.wmem_max or grant CAP_NET_ADMIN")
} else { } else {
u.l.Warn("Failed to get listen.write_buffer", "error", err) u.l.Warn("Failed to get listen.write_buffer", "error", err)
} }
-124
View File
@@ -1,124 +0,0 @@
//go:build linux && !android && !e2e_testing
package udp
import (
"bytes"
"fmt"
"log/slog"
"net/netip"
"os"
"strconv"
"strings"
"testing"
"github.com/slackhq/nebula/config"
)
func readSysctlInt(t *testing.T, path string) int {
t.Helper()
b, err := os.ReadFile(path)
if err != nil {
t.Skipf("can't read %s: %v", path, err)
}
n, err := strconv.Atoi(strings.TrimSpace(string(b)))
if err != nil {
t.Skipf("can't parse %s: %v", path, err)
}
return n
}
// TestSetBufferAboveSysctlMax asks for more than net.core.{r,w}mem_max. With
// CAP_NET_ADMIN the FORCE variant takes it as is; without it we must fall back
// to the plain option and get the clamped size instead of EPERM.
func TestSetBufferAboveSysctlMax(t *testing.T) {
cases := []struct {
name string
sysctl string
set func(*StdConn, int) error
get func(*StdConn) (int, error)
}{
{"read", "/proc/sys/net/core/rmem_max", (*StdConn).SetRecvBuffer, (*StdConn).GetRecvBuffer},
{"write", "/proc/sys/net/core/wmem_max", (*StdConn).SetSendBuffer, (*StdConn).GetSendBuffer},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
sysMax := readSysctlInt(t, tc.sysctl)
c, err := NewListener(testLogger(), Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Batch: 1})
if err != nil {
t.Skipf("NewListener: %v", err)
}
sc := c.(*StdConn)
defer sc.Close()
want := sysMax + 1<<20
if err := tc.set(sc, want); err != nil {
t.Fatalf("set %d: %v", want, err)
}
got, err := tc.get(sc)
if err != nil {
t.Fatalf("get: %v", err)
}
// The kernel doubles the value; either path gives at least the clamped size.
if got < 2*sysMax {
t.Fatalf("effective size %d, want >= %d", got, 2*sysMax)
}
t.Logf("requested %d, %s=%d, effective %d (forced=%v)", want, tc.sysctl, sysMax, got, got >= 2*want)
})
}
}
// TestReloadConfigBufferLogLevel checks that a clamped buffer is a Warn, and a
// buffer that fits is still the Info it always was.
func TestReloadConfigBufferLogLevel(t *testing.T) {
sysMax := readSysctlInt(t, "/proc/sys/net/core/rmem_max")
if wmax := readSysctlInt(t, "/proc/sys/net/core/wmem_max"); wmax < sysMax {
sysMax = wmax
}
cases := []struct {
name string
size int
want []string
}{
{"fits", 64 * 1024, []string{"level=INFO msg=\"listen.read_buffer was set\"", "level=INFO msg=\"listen.write_buffer was set\""}},
{"too big", sysMax + 1<<20, []string{"level=WARN msg=\"listen.read_buffer was limited by the system\"", "level=WARN msg=\"listen.write_buffer was limited by the system\""}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
var buf bytes.Buffer
l := slog.New(slog.NewTextHandler(&buf, nil))
c, err := NewListener(l, Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0"), Batch: 1})
if err != nil {
t.Skipf("NewListener: %v", err)
}
sc := c.(*StdConn)
defer sc.Close()
if tc.size > sysMax {
// With CAP_NET_ADMIN the FORCE variant takes any size, nothing gets limited.
if err := sc.SetRecvBuffer(tc.size); err == nil {
if got, _ := sc.GetRecvBuffer(); got >= 2*tc.size {
t.Skip("FORCE path took the full size, can't observe the limit")
}
}
}
cfg := config.NewC(l)
if err := cfg.LoadString(fmt.Sprintf("listen:\n read_buffer: %d\n write_buffer: %d\n", tc.size, tc.size)); err != nil {
t.Fatal(err)
}
buf.Reset()
sc.ReloadConfig(cfg)
out := buf.String()
for _, w := range tc.want {
if !strings.Contains(out, w) {
t.Errorf("missing %s in:\n%s", w, out)
}
}
if strings.Contains(out, "level=ERROR") {
t.Errorf("unexpected error log:\n%s", out)
}
t.Log(out)
})
}
}