Compare commits

...
Author SHA1 Message Date
Ryan Huber 73d26dd87d tun: Allow excluding CPUs from process threads at startup
Set tun.avoid_cpus to a list of logical CPUs to remove from the affinity of
every thread of the process on Linux, before services start and reader CPUs
are chosen. Later threads inherit the restricted mask; enumeration repeats so
threads created meanwhile are covered. Inherited taskset and cgroup
restrictions are kept. Keeping Nebula off the underlay NIC's receive IRQ CPUs
and their SMT siblings is what this is for; the example configuration shows
how to find them.

Invalid IDs, a list that would leave a thread no CPU, and syscall failures
fail startup. Configuration tests only validate. Other platforms log and
ignore the setting. The exclusion is process-wide, including embedded use, and
not reloadable. It is independent of thread pinning.

New API: util.AvoidCPUs, next to the existing PinThreadToCPU and AllowedCPUs.

Tests run in a child process and check existing and new threads, preserved
inherited masks, validation without changes, and the unix.CPUSet bounds on
32- and 64-bit platforms. No isolated performance claim is made.
2026-10-15 14:40:45 +00:00
Nate Brown 94372f4f51 Don't delete routes we never installed on darwin and the BSDs (#1933) 2026-10-13 16:34:38 -05:00
John Maguire 42280f04ce darwin: Assign host address to the utun and route vpn explicitly (#1931)
Tailscale with an exit node treats every interface address prefix as an
attached LAN and either claims it or excludes it, and configd then rewrites
the matching route away from nebula's utun. Interfaces holding a single
address are skipped, so assign /32 and /128 and add the network route
ourselves for both families.
2026-10-13 13:50:37 -04:00
Nate Brown 11c1917387 Warn when an unsafe_route is already routed through another interface on darwin (#1925) 2026-10-12 16:57:14 -05:00
Nate Brown 5ba7aff4b3 ssh: Only restart sshd on reload when its config or host key changed (#1923) 2026-10-12 16:56:51 -05:00
Ryan Huber 8a2de9b1b5 examples: set socket buffers and raise tun tx_queue (#1928)
* examples: set socket buffers and raise tun tx_queue

* examples: use the default tcp max window for socket buffers

* examples: note buffers are linux only, clarify tx_queue default

* tun: default tx_queue to 5000
2026-10-11 00:30:07 -04:00
16 changed files with 737 additions and 48 deletions

No files matched your search

+13 -5
View File
@@ -135,12 +135,12 @@ listen:
# superpacket, so the receive scratch is batch * 64KiB per listening socket (~4MiB per routine at the # superpacket, so the receive scratch is batch * 64KiB per listening socket (~4MiB per routine at the
# default of 64). Lower this to trade peak per-syscall throughput for memory on constrained hosts. # default of 64). Lower this to trade peak per-syscall throughput for memory on constrained hosts.
#batch: 64 #batch: 64
# Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel # Linux only. Configure socket buffers for the udp side (outside), leave unset to use the system defaults. Values will be doubled by the kernel
# Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default) # Default is net.core.rmem_default and net.core.wmem_default (/proc/sys/net/core/rmem_default and /proc/sys/net/core/rmem_default)
# Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide # Maximum is limited by memory in the system, SO_RCVBUFFORCE and SO_SNDBUFFORCE is used to avoid having to raise the system wide
# max, net.core.rmem_max and net.core.wmem_max # max, net.core.rmem_max and net.core.wmem_max
#read_buffer: 10485760 read_buffer: 3412512
#write_buffer: 10485760 write_buffer: 3412512
# On Windows only # On Windows only
# When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port. # When true, Nebula installs a WFP (Windows Filtering Platform) PERMIT filter scoped to UDP at the listener port.
@@ -262,8 +262,8 @@ tun:
drop_local_broadcast: false drop_local_broadcast: false
# Toggles forwarding of multicast packets # Toggles forwarding of multicast packets
drop_multicast: false drop_multicast: false
# Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 500 # Sets the transmit queue length, if you notice lots of transmit drops on the tun it may help to raise this number. Default is 5000
tx_queue: 500 tx_queue: 5000
# Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic # Default MTU for every packet, safe setting is (and the default) 1300 for internet based traffic
# Anything above 8937, here or on a route below, is capped to 8937 with a warning # Anything above 8937, here or on a route below, is capped to 8937 with a warning
mtu: 1300 mtu: 1300
@@ -295,6 +295,14 @@ tun:
# - 2 # - 2
# - 4 # - 4
# Linux only. avoid_cpus removes these CPU IDs from the affinity of every thread of the process at startup,
# embedding applications included; later threads inherit the remaining set. Applied before cpu_affinity and the
# default pin selection, independent of pin_threads. Invalid IDs, or IDs that would leave no CPU, fail
# startup. Not reloadable; ignored on other platforms. To keep Nebula off the underlay NIC's receive IRQ CPUs, find
# them in /proc/interrupts and /proc/irq/<IRQ>/effective_affinity_list, add their SMT siblings from
# /sys/devices/system/cpu/cpu<N>/topology/thread_siblings_list, and revisit the list when IRQ placement changes.
#avoid_cpus: []
# Route based MTU overrides, you have known vpn ip paths that can support larger MTUs you can increase/decrease them here # Route based MTU overrides, you have known vpn ip paths that can support larger MTUs you can increase/decrease them here
routes: routes:
#- mtu: 8800 #- mtu: 8800
+40
View File
@@ -2,6 +2,7 @@ package nebula
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"log/slog" "log/slog"
"net" "net"
@@ -31,6 +32,10 @@ const maxRoutines = 1 << 16
const _ = noiseutil.RejectHeadroom - 4*maxRoutines const _ = noiseutil.RejectHeadroom - 4*maxRoutines
func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) { func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) {
// Apply before starting services or selecting per-reader pin CPUs.
if err := configureAvoidCPUs(c, l, configTest); err != nil {
return nil, err
}
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
// The goroutines started below stop only when this context does, and only a caller holding the // The goroutines started below stop only when this context does, and only a caller holding the
// Control can arrange that. Cancel whenever we are not handing one back, which covers an error // Control can arrange that. Cancel whenever we are not handing one back, which covers an error
@@ -416,6 +421,41 @@ func parseCpuAffinity(c *config.C, l *slog.Logger, routines int) []int {
return cpus return cpus
} }
// configureAvoidCPUs applies tun.avoid_cpus, a list of CPU IDs to keep every thread of the process off. Unlike
// tun.cpu_affinity, a bad entry fails startup rather than being ignored. Configuration tests only validate it.
func configureAvoidCPUs(c *config.C, l *slog.Logger, configTest bool) error {
raw := c.Get("tun.avoid_cpus")
if raw == nil {
return nil
}
values, ok := raw.([]any)
if !ok {
return fmt.Errorf("tun.avoid_cpus must be a list of non-negative integers")
}
cpus := make([]int, 0, len(values))
for _, value := range values {
cpu, ok := value.(int)
if !ok || cpu < 0 {
return fmt.Errorf("tun.avoid_cpus must contain non-negative integers, got %v", value)
}
cpus = append(cpus, cpu)
}
if len(cpus) == 0 {
return nil
}
if err := util.AvoidCPUs(cpus, !configTest); err != nil {
if errors.Is(err, errors.ErrUnsupported) {
l.Warn("tun.avoid_cpus is only supported on Linux; ignoring")
return nil
}
return fmt.Errorf("tun.avoid_cpus: %w", err)
}
if !configTest {
l.Info("Excluded CPUs from process threads", "cpus", cpus)
}
return nil
}
func moduleVersion() string { func moduleVersion() string {
info, ok := debug.ReadBuildInfo() info, ok := debug.ReadBuildInfo()
if !ok { if !ok {
+21
View File
@@ -59,6 +59,27 @@ func TestParseCpuAffinity(t *testing.T) {
} }
} }
func TestConfigureAvoidCPUs(t *testing.T) {
l := test.NewLogger()
for _, tc := range []struct {
value any
valid bool
}{
{nil, true}, {[]any{}, true}, {"auto", false}, {"1,2", false}, {[]any{-1}, false}, {[]any{1.5}, false},
{[]any{"2"}, false},
} {
c := config.NewC(l)
if tc.value != nil {
c.Settings["tun"] = map[string]any{"avoid_cpus": tc.value}
}
if err := configureAvoidCPUs(c, l, true); tc.valid {
require.NoError(t, err)
} else {
require.Error(t, err, "%v", tc.value)
}
}
}
// TestMain_ConfigTestReleasesItsGoroutines pins the rule that Main only leaves goroutines running // TestMain_ConfigTestReleasesItsGoroutines pins the rule that Main only leaves goroutines running
// when it hands back a Control to stop them with. // when it hands back a Control to stop them with.
// //
+1 -1
View File
@@ -43,7 +43,7 @@ type DeviceFactory func(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix,
func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) { func NewDeviceFromConfig(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, routines int) (Device, error) {
switch { switch {
case c.GetBool("tun.disabled", false): case c.GetBool("tun.disabled", false):
tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 500), c.GetBool("stats.message_metrics", false), l) tun := newDisabledTun(vpnNetworks, c.GetInt("tun.tx_queue", 5000), c.GetBool("stats.message_metrics", false), l)
return tun, nil return tun, nil
default: default:
+98 -14
View File
@@ -6,8 +6,10 @@ import (
"errors" "errors"
"fmt" "fmt"
"log/slog" "log/slog"
"math/bits"
"net/netip" "net/netip"
"os" "os"
"sync"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -28,7 +30,7 @@ type tun struct {
DefaultMTU int DefaultMTU int
Routes atomic.Pointer[[]Route] Routes atomic.Pointer[[]Route]
routeTree atomic.Pointer[bart.Table[routing.Gateways]] routeTree atomic.Pointer[bart.Table[routing.Gateways]]
linkAddr *netroute.LinkAddr linkAddr atomic.Pointer[netroute.LinkAddr]
// hostOwned means the fd arrived from the OS, which has already configured addressing, mtu // hostOwned means the fd arrived from the OS, which has already configured addressing, mtu
// and routes for it. NEPacketTunnelProvider on darwin does this. // and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool hostOwned bool
@@ -243,7 +245,7 @@ func (t *tun) Activate() error {
if linkAddr == nil { if linkAddr == nil {
return fmt.Errorf("unable to discover link_addr for tun interface") return fmt.Errorf("unable to discover link_addr for tun interface")
} }
t.linkAddr = linkAddr t.linkAddr.Store(linkAddr)
for _, network := range t.vpnNetworks { for _, network := range t.vpnNetworks {
if network.Addr().Is4() { if network.Addr().Is4() {
@@ -269,6 +271,7 @@ func (t *tun) Activate() error {
return t.addRoutes(false) return t.addRoutes(false)
} }
// activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
func (t *tun) activate4(network netip.Prefix) error { func (t *tun) activate4(network netip.Prefix) error {
s, err := unix.Socket( s, err := unix.Socket(
unix.AF_INET, unix.AF_INET,
@@ -295,22 +298,32 @@ func (t *tun) activate4(network netip.Prefix) error {
MaskAddr: unix.RawSockaddrInet4{ MaskAddr: unix.RawSockaddrInet4{
Len: unix.SizeofSockaddrInet4, Len: unix.SizeofSockaddrInet4,
Family: unix.AF_INET, Family: unix.AF_INET,
Addr: prefixToMask(network).As4(), Addr: [4]byte{255, 255, 255, 255}, //always make the address a /32
}, },
} }
// On Darwin, VPNs don't edit the routing table directly.
// A network extension hands its routes to the system, and configd installs them.
// configd replaces any route already holding the same prefix, including ours.
// As an example, Tailscale with an exit node treats any interface prefix wider than one address as a LAN.
// With LAN access off, it routes each LAN into Tailscale's utun.
// With LAN access on, it excludes each LAN from its tunnel.
// configd routes an excluded prefix like its own routes would, ignoring ours, so it goes out the default route.
// Either way our route is replaced, and traffic for the vpn network no longer reaches nebula.
// A single address isn't a LAN to Tailscale, so our route is left alone.
// We route the vpn network to the tun explicitly instead.
if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil { if err := ioctl(uintptr(s), unix.SIOCAIFADDR, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun v4 address: %s", err) return fmt.Errorf("failed to set tun v4 address: %s", err)
} }
err = addRoute(network, t.linkAddr) if err := addRoute(network, t.linkAddr.Load()); err != nil {
if err != nil { return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err)
return err
} }
return nil return nil
} }
// activate6 is activate4 for IPv6
func (t *tun) activate6(network netip.Prefix) error { func (t *tun) activate6(network netip.Prefix) error {
s, err := unix.Socket( s, err := unix.Socket(
unix.AF_INET6, unix.AF_INET6,
@@ -332,20 +345,24 @@ func (t *tun) activate6(network netip.Prefix) error {
PrefixMask: unix.RawSockaddrInet6{ PrefixMask: unix.RawSockaddrInet6{
Len: unix.SizeofSockaddrInet6, Len: unix.SizeofSockaddrInet6,
Family: unix.AF_INET6, Family: unix.AF_INET6,
Addr: prefixToMask(network).As16(), // Always set the netmask to be a /128. See activate4 for why we do this.
Addr: [16]byte{255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255},
}, },
Lifetime: addrLifetime{ Lifetime: addrLifetime{
// never expires // never expires
Vltime: 0xffffffff, Vltime: 0xffffffff,
Pltime: 0xffffffff, Pltime: 0xffffffff,
}, },
Flags: _IN6_IFF_NODAD, Flags: _IN6_IFF_NODAD, // No DAD, a utun has no neighbors.
} }
if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil { if err := ioctl(uintptr(s), _SIOCAIFADDR_IN6, uintptr(unsafe.Pointer(&ifr))); err != nil {
return fmt.Errorf("failed to set tun address: %s", err) return fmt.Errorf("failed to set tun address: %s", err)
} }
if err := addRoute(network, t.linkAddr.Load()); err != nil {
return fmt.Errorf("failed to add route for %v: %w", network.Masked(), err)
}
return nil return nil
} }
@@ -394,6 +411,56 @@ func (t *tun) RoutesFor(ip netip.Addr) routing.Gateways {
return routing.Gateways{} return routing.Gateways{}
} }
func fetchRoutes() ([]netroute.Message, error) {
rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_DUMP, 0)
if err != nil {
return nil, err
}
return netroute.ParseRIB(unix.NET_RT_DUMP, rib)
}
// routeInterfaceIn returns the index of the interface prefix is routed through, or 0 if msgs has no route for exactly
// that prefix. Routes scoped to an interface are skipped, they don't stop us adding ours.
func routeInterfaceIn(msgs []netroute.Message, prefix netip.Prefix) int {
for _, msg := range msgs {
m, ok := msg.(*netroute.RouteMessage)
if !ok || len(m.Addrs) <= unix.RTAX_NETMASK || m.Flags&unix.RTF_IFSCOPE != 0 {
continue
}
if p, ok := routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK]); ok && p == prefix.Masked() {
return m.Index
}
}
return 0
}
// routePrefix returns the prefix a route covers. Host routes have no netmask.
func routePrefix(dst, mask netroute.Addr) (netip.Prefix, bool) {
switch d := dst.(type) {
case *netroute.Inet4Addr:
bits := 32
if m, ok := mask.(*netroute.Inet4Addr); ok {
bits = maskBits(m.IP[:])
}
return netip.PrefixFrom(netip.AddrFrom4(d.IP), bits), true
case *netroute.Inet6Addr:
bits := 128
if m, ok := mask.(*netroute.Inet6Addr); ok {
bits = maskBits(m.IP[:])
}
return netip.PrefixFrom(netip.AddrFrom16(d.IP), bits), true
}
return netip.Prefix{}, false
}
func maskBits(mask []byte) int {
n := 0
for _, b := range mask {
n += bits.OnesCount8(b)
}
return n
}
// Get the LinkAddr for the interface of the given name // Get the LinkAddr for the interface of the given name
// Is there an easier way to fetch this when we create the interface? // Is there an easier way to fetch this when we create the interface?
// Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers. // Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers.
@@ -428,7 +495,14 @@ func (t *tun) addRoutes(logErrors bool) error {
return nil return nil
} }
// Activate hasn't run yet, it will install the routes
linkAddr := t.linkAddr.Load()
if linkAddr == nil {
return nil
}
routes := *t.Routes.Load() routes := *t.Routes.Load()
systemRoutes := sync.OnceValues(fetchRoutes)
for _, r := range routes { for _, r := range routes {
if len(r.Via) == 0 || !r.Install { if len(r.Via) == 0 || !r.Install {
@@ -436,10 +510,16 @@ func (t *tun) addRoutes(logErrors bool) error {
continue continue
} }
err := addRoute(r.Cidr, t.linkAddr) err := addRoute(r.Cidr, linkAddr)
if err != nil { if err != nil {
if errors.Is(err, unix.EEXIST) { if errors.Is(err, unix.EEXIST) {
t.l.Warn("unable to add unsafe_route, identical route already exists", "route", r.Cidr) // Every route we added already exists on a reload, only warn about someone else's
msgs, err := systemRoutes()
if err != nil {
t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
}
} else { } else {
retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
if logErrors { if logErrors {
@@ -450,6 +530,8 @@ func (t *tun) addRoutes(logErrors bool) error {
} }
} else { } else {
t.l.Info("Added route", "route", r) t.l.Info("Added route", "route", r)
// A cidr listed twice would otherwise find our own route missing from an older fetch
systemRoutes = sync.OnceValues(fetchRoutes)
} }
} }
@@ -457,16 +539,18 @@ func (t *tun) addRoutes(logErrors bool) error {
} }
func (t *tun) removeRoutes(routes []Route) error { func (t *tun) removeRoutes(routes []Route) error {
if t.hostOwned { linkAddr := t.linkAddr.Load()
if t.hostOwned || linkAddr == nil {
return nil return nil
} }
for _, r := range routes { for _, r := range routes {
if !r.Install { if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue continue
} }
err := delRoute(r.Cidr, t.linkAddr) err := delRoute(r.Cidr, linkAddr)
if err != nil { if err != nil {
t.l.Error("Failed to remove route", "error", err, "route", r) t.l.Error("Failed to remove route", "error", err, "route", r)
} else { } else {
+103
View File
@@ -0,0 +1,103 @@
//go:build darwin && !ios && !e2e_testing
package overlay
import (
"net"
"net/netip"
"os"
"os/exec"
"strings"
"testing"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/routing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
netroute "golang.org/x/net/route"
"golang.org/x/sys/unix"
)
// Every mac routes loopback through lo0
func TestRouteInterfaceIn(t *testing.T) {
msgs, err := fetchRoutes()
require.NoError(t, err)
routeInterface := func(s string) int { return routeInterfaceIn(msgs, netip.MustParsePrefix(s)) }
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
assert.Equal(t, lo0.Index, routeInterface("127.0.0.0/8"))
assert.Equal(t, lo0.Index, routeInterface("127.0.0.1/32"), "a host route has no netmask")
assert.Equal(t, lo0.Index, routeInterface("::1/128"))
assert.Equal(t, 0, routeInterface("127.0.0.0/16"), "only an exact prefix counts")
// Most macs also list the default route scoped to their other interfaces
out, err := exec.Command("route", "-n", "get", "default").Output()
require.NoError(t, err)
var name string
for _, line := range strings.Split(string(out), "\n") {
if v, ok := strings.CutPrefix(strings.TrimSpace(line), "interface: "); ok {
name = v
}
}
if name == "" {
t.Skip("no default route")
}
def, err := net.InterfaceByName(name)
require.NoError(t, err)
assert.Equal(t, def.Index, routeInterface("0.0.0.0/0"))
}
func TestRouteInterfaceIn_SkipsScopedCopies(t *testing.T) {
route := func(index, flags int) netroute.Message {
addrs := make([]netroute.Addr, unix.RTAX_MAX)
addrs[unix.RTAX_DST] = &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}}
addrs[unix.RTAX_NETMASK] = &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}}
return &netroute.RouteMessage{Index: index, Flags: flags, Addrs: addrs}
}
p := netip.MustParsePrefix("192.168.1.0/24")
assert.Equal(t, 7, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE), route(7, unix.RTF_UP)}, p))
assert.Equal(t, 0, routeInterfaceIn([]netroute.Message{route(4, unix.RTF_UP|unix.RTF_IFSCOPE)}, p))
}
// A reload can happen before Activate
func TestTunRoutesBeforeActivate(t *testing.T) {
tn := &tun{l: test.NewLogger()}
routes := []Route{{
Cidr: netip.MustParsePrefix("10.253.1.0/24"),
Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)},
Install: true,
}}
tn.Routes.Store(&routes)
assert.NoError(t, tn.addRoutes(true))
assert.NoError(t, tn.removeRoutes(routes))
}
// A tun.routes entry with no via is never installed, so removing it must not delete a route for the same cidr that
// belongs to someone else
func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
if os.Geteuid() != 0 {
t.Skip("creating a utun and adding routes needs root")
}
lo0, err := net.InterfaceByName("lo0")
require.NoError(t, err)
cidr := "10.250.0.128/25"
require.NoError(t, exec.Command("route", "-q", "-n", "add", "-net", cidr, "-interface", "lo0").Run())
t.Cleanup(func() { _ = exec.Command("route", "-q", "-n", "delete", "-net", cidr).Run() })
l := test.NewLogger()
c := config.NewC(l)
require.NoError(t, c.LoadString("tun:\n routes:\n - route: "+cidr+"\n mtu: 1300\n"))
tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24")}, false)
require.NoError(t, err)
t.Cleanup(func() { _ = tn.Close() })
require.NoError(t, tn.Activate())
require.NoError(t, c.ReloadConfigString("tun: {}\n"))
msgs, err := fetchRoutes()
require.NoError(t, err)
assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
}
+2 -1
View File
@@ -589,7 +589,8 @@ func (t *tun) addRoutes(logErrors bool) error {
func (t *tun) removeRoutes(routes []Route) error { func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes { for _, r := range routes {
if !r.Install { if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue continue
} }
+1 -1
View File
@@ -221,7 +221,7 @@ func newTunGeneric(c *config.C, l *slog.Logger, fd int, vnetHdr bool, offloadFla
vnetHdr: vnetHdr, vnetHdr: vnetHdr,
offloadFlags: offloadFlags, offloadFlags: offloadFlags,
vpnNetworks: vpnNetworks, vpnNetworks: vpnNetworks,
TXQueueLen: c.GetInt("tun.tx_queue", 500), TXQueueLen: c.GetInt("tun.tx_queue", 5000),
useSystemRoutes: c.GetBool("tun.use_system_route_table", false), useSystemRoutes: c.GetBool("tun.use_system_route_table", false),
useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0), useSystemRoutesBufferSize: c.GetInt("tun.use_system_route_table_buffer_size", 0),
routesFromSystem: map[netip.Prefix]routing.Gateways{}, routesFromSystem: map[netip.Prefix]routing.Gateways{},
+2 -1
View File
@@ -420,7 +420,8 @@ func (t *tun) addRoutes(logErrors bool) error {
func (t *tun) removeRoutes(routes []Route) error { func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes { for _, r := range routes {
if !r.Install { if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue continue
} }
+2 -1
View File
@@ -399,7 +399,8 @@ func (t *tun) addRoutes(logErrors bool) error {
func (t *tun) removeRoutes(routes []Route) error { func (t *tun) removeRoutes(routes []Route) error {
for _, r := range routes { for _, r := range routes {
if !r.Install { if len(r.Via) == 0 || !r.Install {
// addRoutes never installs these
continue continue
} }
+32 -12
View File
@@ -60,6 +60,20 @@ type sshDeviceInfoFlags struct {
func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) { func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
c.RegisterReloadCallback(func(c *config.C) { c.RegisterReloadCallback(func(c *config.C) {
if c.GetBool("sshd.enabled", false) { if c.GetBool("sshd.enabled", false) {
// Restarting the server drops every open session, the one that asked for this reload included. Only restart
// when it isn't running or its config changed. The host key file can change under the same config, new
// handshakes use the new key without a restart
if ssh.Running() && !c.HasChanged("sshd") {
key, err := loadSSHHostKey(c)
if err == nil {
err = ssh.SetHostKey(key)
}
if err != nil {
l.Error("Failed to reconfigure the sshd", "error", err)
ssh.Stop()
}
return
}
sshRun, err := configSSH(l, ssh, c) sshRun, err := configSSH(l, ssh, c)
if err != nil { if err != nil {
l.Error("Failed to reconfigure the sshd", "error", err) l.Error("Failed to reconfigure the sshd", "error", err)
@@ -74,6 +88,22 @@ func wireSSHReload(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) {
}) })
} }
// loadSSHHostKey returns sshd.host_key, read from its file when it names one.
func loadSSHHostKey(c *config.C) ([]byte, error) {
hostKeyPathOrKey := c.GetString("sshd.host_key", "")
if hostKeyPathOrKey == "" {
return nil, fmt.Errorf("sshd.host_key must be provided")
}
if strings.Contains(hostKeyPathOrKey, "-----BEGIN") {
return []byte(hostKeyPathOrKey), nil
}
b, err := os.ReadFile(hostKeyPathOrKey)
if err != nil {
return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err)
}
return b, nil
}
// configSSH reads the ssh info out of the passed-in Config and // configSSH reads the ssh info out of the passed-in Config and
// updates the passed-in SSHServer. On success, it returns a function // updates the passed-in SSHServer. On success, it returns a function
// that callers may invoke to run the configured ssh server. On // that callers may invoke to run the configured ssh server. On
@@ -92,19 +122,9 @@ func configSSH(l *slog.Logger, ssh *sshd.SSHServer, c *config.C) (func(), error)
return nil, fmt.Errorf("sshd.listen can not use port 22") return nil, fmt.Errorf("sshd.listen can not use port 22")
} }
hostKeyPathOrKey := c.GetString("sshd.host_key", "") hostKeyBytes, err := loadSSHHostKey(c)
if hostKeyPathOrKey == "" {
return nil, fmt.Errorf("sshd.host_key must be provided")
}
var hostKeyBytes []byte
if strings.Contains(hostKeyPathOrKey, "-----BEGIN") {
hostKeyBytes = []byte(hostKeyPathOrKey)
} else {
hostKeyBytes, err = os.ReadFile(hostKeyPathOrKey)
if err != nil { if err != nil {
return nil, fmt.Errorf("error while loading sshd.host_key file: %s", err) return nil, err
}
} }
err = ssh.SetHostKey(hostKeyBytes) err = ssh.SetHostKey(hostKeyBytes)
+47 -12
View File
@@ -8,13 +8,17 @@ import (
"log/slog" "log/slog"
"net" "net"
"sync" "sync"
"sync/atomic"
"github.com/armon/go-radix" "github.com/armon/go-radix"
"golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh"
) )
type SSHServer struct { type SSHServer struct {
config *ssh.ServerConfig // config is replaced whole when the host key changes, ssh.ServerConfig.AddHostKey only replaces a key of the same
// algorithm and appends any other
config atomic.Pointer[ssh.ServerConfig]
authenticate func(ssh.ConnMetadata, ssh.PublicKey) (*ssh.Permissions, error)
l *slog.Logger l *slog.Logger
certChecker *ssh.CertChecker certChecker *ssh.CertChecker
@@ -28,7 +32,12 @@ type SSHServer struct {
// List of available commands // List of available commands
helpCommand *Command helpCommand *Command
commands *radix.Tree commands *radix.Tree
listener net.Listener listener atomic.Pointer[net.Listener]
// runs counts the Runs that are listening. A reload's Run can start before the one it replaced has returned
runs atomic.Int32
// addr is where the newest Run listens, nil when none is
addr atomic.Pointer[net.Addr]
// ctx parents per-Run contexts. Cancelling it (e.g. via Control.Stop) tears the server down even // ctx parents per-Run contexts. Cancelling it (e.g. via Control.Stop) tears the server down even
// across reloads, since each Run derives a fresh child rather than reusing this one directly. // across reloads, since each Run derives a fresh child rather than reusing this one directly.
@@ -85,10 +94,8 @@ func NewSSHServer(ctx context.Context, l *slog.Logger) (*SSHServer, error) {
}, },
} }
s.config = &ssh.ServerConfig{ s.authenticate = cc.Authenticate
PublicKeyCallback: cc.Authenticate, s.config.Store(s.newConfig())
ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"),
}
s.RegisterCommand(&Command{ s.RegisterCommand(&Command{
Name: "help", Name: "help",
@@ -107,10 +114,32 @@ func (s *SSHServer) SetHostKey(hostPrivateKey []byte) error {
return fmt.Errorf("failed to parse private key: %s", err) return fmt.Errorf("failed to parse private key: %s", err)
} }
s.config.AddHostKey(private) config := s.newConfig()
config.AddHostKey(private)
s.config.Store(config)
return nil return nil
} }
func (s *SSHServer) newConfig() *ssh.ServerConfig {
return &ssh.ServerConfig{
PublicKeyCallback: s.authenticate,
ServerVersion: fmt.Sprintf("SSH-2.0-Nebula???"),
}
}
// Addr is the address the newest Run listens on, nil when none is.
func (s *SSHServer) Addr() net.Addr {
if a := s.addr.Load(); a != nil {
return *a
}
return nil
}
// Running is true while a Run is listening.
func (s *SSHServer) Running() bool {
return s.runs.Load() > 0
}
func (s *SSHServer) ClearTrustedCAs() { func (s *SSHServer) ClearTrustedCAs() {
s.authLock.Lock() s.authLock.Lock()
s.trustedCAs = []ssh.PublicKey{} s.trustedCAs = []ssh.PublicKey{}
@@ -181,10 +210,16 @@ func (s *SSHServer) Run(addr string) error {
// this run owns. They start equal but a fast reload may overwrite s.listener with the next run's // this run owns. They start equal but a fast reload may overwrite s.listener with the next run's
// listener before this run's watcher fires, so each run must close its own listener via the local // listener before this run's watcher fires, so each run must close its own listener via the local
// reference. // reference.
s.listener = listener s.listener.Store(&listener)
s.runs.Add(1)
bound := listener.Addr()
s.addr.Store(&bound)
runCtx, cancel := context.WithCancel(s.ctx) runCtx, cancel := context.WithCancel(s.ctx)
defer cancel() defer cancel()
// Deferred after cancel so they run first, a run on its way out no longer counts as listening
defer s.runs.Add(-1)
defer s.addr.CompareAndSwap(&bound, nil)
// Close the listener when this run's context is cancelled. That can come from the parent // Close the listener when this run's context is cancelled. That can come from the parent
// (Control.Stop), from Run returning normally (defer cancel above), or transitively when a sibling // (Control.Stop), from Run returning normally (defer cancel above), or transitively when a sibling
@@ -197,7 +232,7 @@ func (s *SSHServer) Run(addr string) error {
} }
}() }()
s.l.Info("SSH server is listening", "sshListener", addr) s.l.Info("SSH server is listening", "sshListener", bound)
// Run loops until there is an error // Run loops until there is an error
s.run(runCtx, listener) s.run(runCtx, listener)
@@ -226,7 +261,7 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
<-sessionContext.Done() <-sessionContext.Done()
c.Close() c.Close()
}() }()
conn, chans, reqs, err := ssh.NewServerConn(c, s.config) conn, chans, reqs, err := ssh.NewServerConn(c, s.config.Load())
fp := "" fp := ""
if conn != nil { if conn != nil {
fp = conn.Permissions.Extensions["fp"] fp = conn.Permissions.Extensions["fp"]
@@ -264,8 +299,8 @@ func (s *SSHServer) run(ctx context.Context, listener net.Listener) {
} }
func (s *SSHServer) Stop() { func (s *SSHServer) Stop() {
if s.listener != nil { if l := s.listener.Load(); l != nil {
if err := s.listener.Close(); err != nil { if err := (*l).Close(); err != nil && !errors.Is(err, net.ErrClosed) {
s.l.Warn("Failed to close the sshd listener", "error", err) s.l.Warn("Failed to close the sshd listener", "error", err)
} }
} }
+196
View File
@@ -0,0 +1,196 @@
package nebula
import (
"crypto"
"crypto/ed25519"
"crypto/rand"
"crypto/rsa"
"encoding/pem"
"fmt"
"net"
"os"
"path/filepath"
"testing"
"time"
"github.com/slackhq/nebula/config"
"github.com/slackhq/nebula/sshd"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/crypto/ssh"
)
type sshdReloadHarness struct {
c *config.C
server *sshd.SSHServer
keyFile string
signer ssh.Signer
extra string
}
func newSSHDReloadHarness(t *testing.T) *sshdReloadHarness {
t.Helper()
l := test.NewLogger()
h := &sshdReloadHarness{keyFile: filepath.Join(t.TempDir(), "host_key")}
h.writeHostKey(t, newEd25519Key(t))
_, userKey, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
h.signer, err = ssh.NewSignerFromKey(userKey)
require.NoError(t, err)
h.c = config.NewC(l)
require.NoError(t, h.c.LoadString(h.config()))
h.server, err = sshd.NewSSHServer(t.Context(), l)
require.NoError(t, err)
wireSSHReload(l, h.server, h.c)
run, err := configSSH(l, h.server, h.c)
require.NoError(t, err)
go run()
t.Cleanup(h.server.Stop)
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
return h
}
func newEd25519Key(t *testing.T) crypto.Signer {
_, key, err := ed25519.GenerateKey(rand.Reader)
require.NoError(t, err)
return key
}
// writeHostKey replaces the host key file and returns the public key a client should now see
func (h *sshdReloadHarness) writeHostKey(t *testing.T, key crypto.Signer) ssh.PublicKey {
t.Helper()
block, err := ssh.MarshalPrivateKey(key, "")
require.NoError(t, err)
require.NoError(t, os.WriteFile(h.keyFile, pem.EncodeToMemory(block), 0o600))
pub, err := ssh.NewPublicKey(key.Public())
require.NoError(t, err)
return pub
}
func (h *sshdReloadHarness) config() string {
return fmt.Sprintf("sshd:\n enabled: true\n listen: 127.0.0.1:0\n host_key: %s\n authorized_users:\n - user: test\n keys: [%q]\n%s",
h.keyFile, string(ssh.MarshalAuthorizedKey(h.signer.PublicKey())), h.extra)
}
// dial connects to wherever the server listens now, a restart on :0 moves it. hostKeyAlgorithms limits which host keys
// the client accepts, none means any
func (h *sshdReloadHarness) dial(hostKeyAlgorithms ...string) (*ssh.Client, ssh.PublicKey, error) {
addr := h.server.Addr()
if addr == nil {
return nil, nil, fmt.Errorf("not listening")
}
var hostKey ssh.PublicKey
c, err := ssh.Dial("tcp", addr.String(), &ssh.ClientConfig{
User: "test",
Auth: []ssh.AuthMethod{ssh.PublicKeys(h.signer)},
HostKeyCallback: func(_ string, _ net.Addr, key ssh.PublicKey) error {
hostKey = key
return nil
},
HostKeyAlgorithms: hostKeyAlgorithms,
Timeout: time.Second,
})
return c, hostKey, err
}
func (h *sshdReloadHarness) connect(t *testing.T) (*ssh.Client, ssh.PublicKey) {
t.Helper()
var client *ssh.Client
var hostKey ssh.PublicKey
require.Eventually(t, func() bool {
var err error
client, hostKey, err = h.dial()
return err == nil
}, 5*time.Second, 20*time.Millisecond)
t.Cleanup(func() { _ = client.Close() })
return client, hostKey
}
// dropped is true when the server closed the client's connection within wait
func dropped(client *ssh.Client, wait time.Duration) bool {
done := make(chan struct{})
go func() {
_ = client.Wait()
close(done)
}()
select {
case <-done:
return true
case <-time.After(wait):
return false
}
}
// Restarting sshd drops every open session, the one that asked for the reload included. A reload that touches nothing
// sshd reads leaves it alone
func TestSSHDReload_UnchangedKeepsSessions(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
require.NoError(t, h.c.ReloadConfigString(h.config()+"punchy:\n punch: true\n"))
assert.False(t, dropped(client, 500*time.Millisecond), "an unchanged sshd was restarted")
}
func TestSSHDReload_ChangedConfigRestarts(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
h.extra = " trusted_cas: []\n"
require.NoError(t, h.c.ReloadConfigString(h.config()))
assert.True(t, dropped(client, 5*time.Second), "a changed sshd config was not applied")
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t)
}
// The config can stay the same while the host key file it names changes. New connections get the new key, open
// sessions are left alone
func TestSSHDReload_ChangedHostKeyFile(t *testing.T) {
h := newSSHDReloadHarness(t)
client, _ := h.connect(t)
want := h.writeHostKey(t, newEd25519Key(t))
require.NoError(t, h.c.ReloadConfigString(h.config()))
_, got := h.connect(t)
assert.Equal(t, want.Marshal(), got.Marshal(), "a new host key was not applied")
assert.False(t, dropped(client, 500*time.Millisecond), "a new host key restarted sshd")
}
// A host key of another algorithm replaces the old one rather than being offered next to it
func TestSSHDReload_HostKeyAlgorithmChange(t *testing.T) {
h := newSSHDReloadHarness(t)
h.connect(t)
rsaKey, err := rsa.GenerateKey(rand.Reader, 2048)
require.NoError(t, err)
want := h.writeHostKey(t, rsaKey)
require.NoError(t, h.c.ReloadConfigString(h.config()))
_, got := h.connect(t)
assert.Equal(t, want.Marshal(), got.Marshal())
_, _, err = h.dial(ssh.KeyAlgoED25519)
assert.Error(t, err, "the old ed25519 host key is still offered")
}
// A server that isn't running, say its port was busy, is tried again on every reload as before
func TestSSHDReload_NotRunningRestarts(t *testing.T) {
h := newSSHDReloadHarness(t)
h.server.Stop()
require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond)
require.NoError(t, h.c.ReloadConfigString(h.config()))
require.Eventually(t, h.server.Running, 5*time.Second, 10*time.Millisecond)
h.connect(t)
}
// A host key that no longer loads stops sshd, the same as a config that doesn't
func TestSSHDReload_BadHostKeyFileStops(t *testing.T) {
h := newSSHDReloadHarness(t)
h.connect(t)
require.NoError(t, os.WriteFile(h.keyFile, []byte("not a key"), 0o600))
require.NoError(t, h.c.ReloadConfigString(h.config()))
require.Eventually(t, func() bool { return !h.server.Running() }, 5*time.Second, 10*time.Millisecond)
}
+74
View File
@@ -0,0 +1,74 @@
//go:build linux && !android && !e2e_testing
package util
import (
"errors"
"fmt"
"os"
"strconv"
"unsafe"
"golang.org/x/sys/unix"
)
// AvoidCPUs removes cpus from the affinity of every thread of the process, for use at startup. New threads inherit
// their parent's mask, so it repeats until a pass finds nothing left to change; a thread that exits meanwhile is
// skipped. Every mask is checked before any is changed. With apply false it only validates, for configuration tests.
// This affects the entire process, including other users of an embedded Nebula.
func AvoidCPUs(cpus []int, apply bool) error {
const capacity = int(unsafe.Sizeof(unix.CPUSet{})) * 8
for _, cpu := range cpus {
if cpu < 0 || cpu >= capacity {
return fmt.Errorf("CPU %d is out of range", cpu)
}
}
if len(cpus) == 0 {
return nil
}
for {
masks, err := avoidingMasks(cpus)
if err != nil || !apply || len(masks) == 0 {
return err
}
for tid, mask := range masks {
if err := unix.SchedSetaffinity(tid, &mask); err != nil && !errors.Is(err, unix.ESRCH) {
return fmt.Errorf("set thread %d affinity: %w", tid, err)
}
}
}
}
// avoidingMasks returns the mask without cpus of every thread whose mask has any of them.
func avoidingMasks(cpus []int) (map[int]unix.CPUSet, error) {
entries, err := os.ReadDir("/proc/self/task")
if err != nil {
return nil, err
}
masks := make(map[int]unix.CPUSet, len(entries))
for _, entry := range entries {
tid, err := strconv.Atoi(entry.Name())
if err != nil {
return nil, err
}
var mask unix.CPUSet
err = unix.SchedGetaffinity(tid, &mask)
if errors.Is(err, unix.ESRCH) {
continue
}
if err != nil {
return nil, fmt.Errorf("read thread %d affinity: %w", tid, err)
}
next := mask
for _, cpu := range cpus {
next.Clear(cpu)
}
if next.Count() == 0 {
return nil, fmt.Errorf("avoid_cpus leaves thread %d with no allowed CPU", tid)
}
if next != mask {
masks[tid] = next
}
}
return masks, nil
}
+97
View File
@@ -0,0 +1,97 @@
//go:build linux && !android && !e2e_testing
package util
import (
"fmt"
"os"
"os/exec"
"runtime"
"strconv"
"sync"
"testing"
"unsafe"
"github.com/stretchr/testify/require"
"golang.org/x/sys/unix"
)
// TestAvoidCPUs changes the affinity of the whole process, so it runs in a child, leaving the test runner untouched.
func TestAvoidCPUs(t *testing.T) {
if os.Getenv("NEBULA_AVOID_CPU_TEST") != "1" {
exe, err := os.Executable()
require.NoError(t, err)
cmd := exec.Command(exe, "-test.run=^TestAvoidCPUs$", "-test.v")
cmd.Env = append(os.Environ(), "NEBULA_AVOID_CPU_TEST=1")
out, err := cmd.CombinedOutput()
require.NoError(t, err, "%s", out)
return
}
allowed, err := AllowedCPUs()
require.NoError(t, err)
if len(allowed) < 2 {
t.Skip("requires at least two allowed CPUs")
}
cpu := allowed[0]
release := make(chan struct{})
var wg sync.WaitGroup
// lockThreads occupies 8 OS threads until release, so the runtime has to start more
lockThreads := func() {
var ready sync.WaitGroup
for range 8 {
ready.Add(1)
wg.Go(func() {
runtime.LockOSThread()
ready.Done()
<-release
})
}
ready.Wait()
}
lockThreads()
require.NoError(t, AvoidCPUs([]int{cpu}, false))
after, err := AllowedCPUs()
require.NoError(t, err)
require.Equal(t, allowed, after, "configuration check must not mutate affinity")
require.Error(t, AvoidCPUs(allowed, true), "must reject an empty mask before applying")
require.NoError(t, AvoidCPUs([]int{cpu, cpu}, true))
requireAvoided(t, cpu, allowed)
// Existing threads stay locked while new ones inherit the restriction
lockThreads()
requireAvoided(t, cpu, allowed)
close(release)
wg.Wait()
}
// requireAvoided checks no thread may run on cpu, and every thread still may on some CPU of allowed only.
func requireAvoided(t *testing.T, cpu int, allowed []int) {
t.Helper()
entries, err := os.ReadDir("/proc/self/task")
require.NoError(t, err)
for _, entry := range entries {
tid, err := strconv.Atoi(entry.Name())
require.NoError(t, err)
var mask unix.CPUSet
err = unix.SchedGetaffinity(tid, &mask)
if err == unix.ESRCH {
continue
}
require.NoError(t, err)
require.False(t, mask.IsSet(cpu), "thread %d", tid)
require.Positive(t, mask.Count())
for id := range len(mask) * 64 {
if mask.IsSet(id) {
require.Contains(t, allowed, id, "must not widen the inherited mask")
}
}
}
}
// TestAvoidCPUsRange checks the bounds of unix.CPUSet, whose words are 32 bits wide on 32-bit platforms.
func TestAvoidCPUsRange(t *testing.T) {
capacity := int(unsafe.Sizeof(unix.CPUSet{})) * 8
require.NoError(t, AvoidCPUs([]int{capacity - 1}, false))
require.EqualError(t, AvoidCPUs([]int{capacity}, false), fmt.Sprintf("CPU %d is out of range", capacity))
require.EqualError(t, AvoidCPUs([]int{-1}, false), "CPU -1 is out of range")
}
+8
View File
@@ -0,0 +1,8 @@
//go:build !linux || android || e2e_testing
package util
import "errors"
// AvoidCPUs is unsupported outside Linux: there is no thread affinity API to restrict.
func AvoidCPUs(_ []int, _ bool) error { return errors.ErrUnsupported }