diff --git a/examples/config.yml b/examples/config.yml index b2460778..f81d72ee 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -331,6 +331,9 @@ tun: # `mtu`: will default to tun mtu if this option is not specified # `metric`: will default to 0 if this option is not specified # `install`: will default to true, controls whether this route is installed in the systems routing table. + # On macOS, a route whose destination another interface already routes, like the local network, is installed once + # that route is removed. One deleted by hand stays deleted until nebula reloads or restarts, or until another + # interface routes its destination and lets it go. # This setting is reloadable. unsafe_routes: #- route: 172.16.1.0/24 diff --git a/overlay/tun_darwin.go b/overlay/tun_darwin.go index 5a62f061..6aeb891b 100644 --- a/overlay/tun_darwin.go +++ b/overlay/tun_darwin.go @@ -3,7 +3,6 @@ package overlay import ( - "errors" "fmt" "log/slog" "math/bits" @@ -12,6 +11,7 @@ import ( "sync" "sync/atomic" "syscall" + "time" "unsafe" "github.com/gaissmai/bart" @@ -35,6 +35,26 @@ type tun struct { // and routes for it. NEPacketTunnelProvider on darwin does this. hostOwned bool l *slog.Logger + + // routesMu serializes changes to the system routing table between reload and the route watcher, and guards the + // fields below it + routesMu sync.Mutex + // routeStates is who held each installable unsafe_route prefix when we last looked + routeStates map[netip.Prefix]routeState + // routeWatch is the routing socket watchRoutes reads, Close closes it + routeWatch *os.File + // routeSweep reconciles the unsafe_routes every routeSweepEvery + routeSweep *time.Timer + routeSweepEvery time.Duration + closed bool + + // routeMovers is who last changed each unsafe_route prefix since the last reconcile, for its logs. It has its own + // lock so the route watcher never waits on a reconcile + routeMoversMu sync.Mutex + routeMovers map[netip.Prefix]routeMover + + // lastRouteEvent is when the routing socket last reported something that may have moved a prefix, in unix nanos + lastRouteEvent atomic.Int64 } type ifReq struct { @@ -139,11 +159,12 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t } t := &tun{ - f: os.NewFile(uintptr(fd), ""), - Device: name, - vpnNetworks: vpnNetworks, - DefaultMTU: getMTU(l, c), - l: l, + f: os.NewFile(uintptr(fd), ""), + Device: name, + vpnNetworks: vpnNetworks, + DefaultMTU: getMTU(l, c), + routeSweepEvery: routeSweepInterval, + l: l, } err = t.reload(c, true, nil) @@ -219,6 +240,16 @@ func utunNameFromFd(fd int) string { } func (t *tun) Close() error { + t.routesMu.Lock() + t.closed = true + if t.routeWatch != nil { + _ = t.routeWatch.Close() + } + if t.routeSweep != nil { + t.routeSweep.Stop() + } + t.routesMu.Unlock() + if t.f != nil { return t.f.Close() } @@ -287,8 +318,22 @@ func (t *tun) Activate() error { return fmt.Errorf("failed to run tun device: %s", err) } + // Watch before adding so a route removed in between is still noticed + watch, err := openRouteSocket() + if err != nil { + t.l.Error("Failed to watch the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err) + } + // Unsafe path routes - return t.addRoutes(false, nil) + if err = t.addRoutes(false, nil); err != nil { + if watch != nil { + _ = watch.Close() + } + return err + } + + t.startRouteWatch(watch) + return nil } // activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly. @@ -393,7 +438,8 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error { } if !initial && !change { - return nil + // Any reload puts back the unsafe_routes someone deleted + return t.addRoutes(true, r) } routeTree, err := makeRouteTree(t.l, routes, false) @@ -511,82 +557,6 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) { return nil, nil } -func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error { - // The route tree is still ours, the system routing table is not - if t.hostOwned { - return nil - } - - // Activate hasn't run yet, it will install the routes - linkAddr := t.linkAddr.Load() - if linkAddr == nil { - return nil - } - - routes := *t.Routes.Load() - systemRoutes := sync.OnceValues(fetchRoutes) - - for _, r := range routes { - if len(r.Via) == 0 || !r.Install { - // We don't allow route MTUs so only install routes with a via - continue - } - - err := addRoute(r.Cidr, linkAddr) - if err != nil { - if errors.Is(err, unix.EEXIST) { - // Every route we added already exists on a reload, only warn about someone else's - msgs, err := systemRoutes() - if err != nil { - t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err) - } else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index { - t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr) - } - } else { - retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) - if logErrors { - retErr.Log(t.l) - rl.Failed() - } else { - return retErr - } - } - } else { - t.l.Info("Added route", "route", r) - // A cidr listed twice would otherwise find our own route missing from an older fetch - systemRoutes = sync.OnceValues(fetchRoutes) - } - } - - return nil -} - -func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error { - linkAddr := t.linkAddr.Load() - if t.hostOwned || linkAddr == nil { - return nil - } - - for _, r := range routes { - if len(r.Via) == 0 || !r.Install { - // addRoutes never installs these - continue - } - - err := delRoute(r.Cidr, linkAddr) - if errors.Is(err, unix.ESRCH) { - // Already gone, which is what we wanted - t.l.Info("Route was already removed", "route", r) - } else if err != nil { - t.l.Error("Failed to remove route", "error", err, "route", r) - rl.Failed() - } else { - t.l.Info("Removed route", "route", r) - } - } - return nil -} - func addRoute(prefix netip.Prefix, gateway netroute.Addr) error { sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) if err != nil { diff --git a/overlay/tun_darwin_route_test.go b/overlay/tun_darwin_route_test.go index 5eda4813..a0bac608 100644 --- a/overlay/tun_darwin_route_test.go +++ b/overlay/tun_darwin_route_test.go @@ -3,12 +3,17 @@ package overlay import ( + "encoding/binary" + "encoding/hex" + "fmt" "net" "net/netip" "os" "os/exec" + "path/filepath" "strings" "testing" + "time" "github.com/slackhq/nebula/config" "github.com/slackhq/nebula/routing" @@ -104,6 +109,367 @@ func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) { assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))) } +// Captured on macOS 15.6 while adding and removing 198.51.100.77/24 on a feth +const ( + // The kernel deleting the address's subnet route. It packs its sockaddrs without padding, x/net/route can't parse it + kernelSubnetDelete = "870005022300000000010002070000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010020000c63364000000000000000000141223000600000000000000000000000000000007ffffffffffff" + addressRemoved = "5000050db400000001010000230000000000000007020000ffffff00141223000606060066657468373766657468004d10020000c633644d000000000000000010020000c63364ff0000000000000000" + // With no interface address, which x/net/route skips + interfaceChanged = "7000050e000000004288000023000000060000060e000000dc05000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000073e5ce6a0000000000000000000001000000000000000000" +) + +func TestRoutesMayHaveMoved(t *testing.T) { + cidr := netip.MustParsePrefix("192.168.1.0/24") + via := routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)} + routes := []Route{{Cidr: cidr, Via: via, Install: true}} + // A request like route(8) sends, the kernel echoes it to every routing socket + request := func(typ, flags int, dst, mask [4]byte) []byte { + m := &netroute.RouteMessage{ + Version: unix.RTM_VERSION, + Type: typ, + Flags: flags, + ID: 1234, + Addrs: []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet4Addr{IP: dst}, + unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4}, + unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: mask}, + }, + } + b, err := m.Marshal() + require.NoError(t, err) + return b + } + raw := func(s string) []byte { + b, err := hex.DecodeString(s) + require.NoError(t, err) + return b + } + lan, mask := [4]byte{192, 168, 1, 0}, [4]byte{255, 255, 255, 0} + routesMayHaveMoved := func(b []byte, routes []Route) bool { + moved, _, _ := routesMayHaveMoved(b, routes) + return moved + } + + assert.True(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), routes)) + assert.True(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP, lan, mask), routes)) + assert.True(t, routesMayHaveMoved(request(unix.RTM_CHANGE, unix.RTF_UP, lan, mask), routes)) + assert.False(t, routesMayHaveMoved(request(unix.RTM_GET, unix.RTF_UP, lan, mask), routes)) + assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "a scoped route never held the prefix") + assert.False(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "configd adds a scoped copy for each interface") + assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, [4]byte{255, 255, 0, 0}), routes), "only an exact prefix counts") + assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Install: true}}), "a route with no via is never installed") + assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Via: via}}), "install: false") + + failed := request(unix.RTM_DELETE, unix.RTF_UP, lan, mask) + binary.NativeEndian.PutUint32(failed[24:28], uint32(unix.ESRCH)) + assert.False(t, routesMayHaveMoved(failed, routes), "a failed request is echoed too") + + _, err := netroute.ParseRIB(unix.NET_RT_DUMP, raw(kernelSubnetDelete)) + require.Error(t, err, "if x/net/route learns to parse these, match their prefix instead") + assert.True(t, routesMayHaveMoved(raw(kernelSubnetDelete), routes), "the kernel's own deletes could be any prefix") + neighbor := raw(kernelSubnetDelete) + binary.NativeEndian.PutUint32(neighbor[8:12], unix.RTF_UP|unix.RTF_HOST|unix.RTF_LLINFO) + assert.False(t, routesMayHaveMoved(neighbor, routes), "a neighbor expiring") + + assert.True(t, routesMayHaveMoved(raw(addressRemoved), routes)) + assert.True(t, routesMayHaveMoved(raw(interfaceChanged), routes), "detaching an interface deletes its routes without a message for each") +} + +// A request names its prefix and who made it, the kernel's own and interface changes name neither +func TestRoutesMayHaveMovedSaysWho(t *testing.T) { + cidr := netip.MustParsePrefix("192.168.1.0/24") + routes := []Route{{Cidr: cidr, Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}, Install: true}} + m := &netroute.RouteMessage{ + Version: unix.RTM_VERSION, + Type: unix.RTM_CHANGE, + Flags: unix.RTF_UP, + ID: 1234, + Addrs: []netroute.Addr{ + unix.RTAX_DST: &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}}, + unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4}, + unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}}, + }, + } + b, err := m.Marshal() + require.NoError(t, err) + moved, p, pid := routesMayHaveMoved(b, routes) + assert.True(t, moved) + assert.Equal(t, cidr, p) + assert.Equal(t, int32(1234), pid) + + for name, s := range map[string]string{"kernel": kernelSubnetDelete, "ifinfo": interfaceChanged} { + b, err := hex.DecodeString(s) + require.NoError(t, err) + moved, p, pid := routesMayHaveMoved(b, routes) + assert.True(t, moved, name) + assert.False(t, p.IsValid(), name) + assert.Zero(t, pid, name) + } +} + +func TestProcessName(t *testing.T) { + assert.Equal(t, "launchd", processName(1)) + self := filepath.Base(os.Args[0]) + assert.Equal(t, self[:min(len(self), 16)], processName(int32(os.Getpid()))) + assert.Empty(t, processName(-1)) +} + +// newRouteTestTun is an activated tun for the root tests below with an unsafe_route for each of cidrs. sweep replaces +// the minute between sweeps when it isn't zero. +func newRouteTestTun(t *testing.T, sweep time.Duration, cidrs ...string) (*tun, *config.C, string) { + requireRoot(t) + + cfg := "tun:\n unsafe_routes:\n" + for _, cidr := range cidrs { + via := "10.250.0.2" + if netip.MustParsePrefix(cidr).Addr().Is6() { + via = "fd00:250::2" + } + cfg += " - route: " + cidr + "\n via: " + via + "\n" + } + + l := test.NewLogger() + c := config.NewC(l) + require.NoError(t, c.LoadString(cfg)) + tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false) + require.NoError(t, err) + if sweep != 0 { + tn.routeSweepEvery = sweep + } + t.Cleanup(func() { _ = tn.Close() }) + require.NoError(t, tn.Activate()) + return tn, c, cfg +} + +func requireRoot(t *testing.T) { + if os.Geteuid() != 0 { + t.Skip("creating interfaces and adding routes needs root") + } +} + +// routeVia is the index of the interface that routes cidr, 0 if none does +func routeVia(t *testing.T, cidr string) int { + msgs, err := fetchRoutes() + require.NoError(t, err) + return routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)) +} + +// routeStateOf is what tn made of cidr when it last looked +func routeStateOf(tn *tun, cidr string) routeState { + tn.routesMu.Lock() + defer tn.routesMu.Unlock() + return tn.routeStates[netip.MustParsePrefix(cidr)] +} + +func routeArgs(action, cidr string, extra ...string) []string { + args := []string{"-q", "-n", action} + if netip.MustParsePrefix(cidr).Addr().Is6() { + args = append(args, "-inet6") + } + return append(append(args, "-net", cidr), extra...) +} + +func routeCmd(t *testing.T, action, cidr string, extra ...string) { + requireRouteOK(t, exec.Command("route", routeArgs(action, cidr, extra...)...)) +} + +// requireRouteOK runs cmd, which runs route(8). route exits 0 when the kernel rejects its request, only what it prints +// says so +func requireRouteOK(t *testing.T, cmd *exec.Cmd) { + out, err := cmd.CombinedOutput() + require.NoError(t, err, string(out)) + require.NotContains(t, string(out), "writing to routing socket") +} + +// cleanupRoute deletes cidr's route when the test ends, whoever holds it +func cleanupRoute(t *testing.T, cidr string) { + t.Cleanup(func() { _ = exec.Command("route", routeArgs("delete", cidr)...).Run() }) +} + +// routeThroughLo0 routes cidr through lo0, like a lan nebula started on +func routeThroughLo0(t *testing.T, cidr string) int { + requireRoot(t) + routeCmd(t, "add", cidr, "-interface", "lo0") + cleanupRoute(t, cidr) + lo0, err := net.InterfaceByName("lo0") + require.NoError(t, err) + return lo0.Index +} + +// Leaving the lan that held an unsafe_route's prefix must not need a restart to get the unsafe_route +func TestTunAddsRouteOnceItsPrefixIsFree(t *testing.T) { + for _, cidr := range []string{"10.251.0.0/24", "2001:db8:251::/64"} { + t.Run(cidr, func(t *testing.T) { + lo0 := routeThroughLo0(t, cidr) + tn, _, _ := newRouteTestTun(t, 0, cidr) + require.Equal(t, lo0, routeVia(t, cidr), "an existing route keeps its prefix") + + routeCmd(t, "delete", cidr) + assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond) + }) + } +} + +// The kernel deletes a route by prefix alone, so dropping an unsafe_route someone else holds must not delete theirs +func TestTunReloadLeavesRouteItYielded(t *testing.T) { + cidr := "10.251.1.0/24" + lo0 := routeThroughLo0(t, cidr) + _, c, _ := newRouteTestTun(t, 0, cidr) + + res := c.ReloadConfigString("tun: {}\n") + require.NoError(t, res.Err) + assert.False(t, res.Failed) + assert.Equal(t, lo0, routeVia(t, cidr)) +} + +// configd takes a prefix by deleting the route holding it and adding its own. Ours goes back once theirs is removed +func TestTunYieldsToARouteThatReplacesIt(t *testing.T) { + cidr := "10.251.3.0/24" + tn, _, _ := newRouteTestTun(t, 0, cidr) + ours := tn.linkAddr.Load().Index + require.Equal(t, ours, routeVia(t, cidr)) + + cleanupRoute(t, cidr) + requireRouteOK(t, exec.Command("sh", "-c", "route -q -n delete -net "+cidr+" && route -q -n add -net "+cidr+" -interface lo0")) + lo0, err := net.InterfaceByName("lo0") + require.NoError(t, err) + time.Sleep(2 * routeRetryDelay) + require.Equal(t, lo0.Index, routeVia(t, cidr)) + + routeCmd(t, "delete", cidr) + assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond) +} + +// A route someone deleted, with nothing in its place, stays gone until a reload, so they can put their own there +func TestTunLeavesRouteDeletedByHand(t *testing.T) { + cidr := "10.251.4.0/24" + tn, c, cfg := newRouteTestTun(t, 200*time.Millisecond, cidr) + ours := tn.linkAddr.Load().Index + require.Equal(t, ours, routeVia(t, cidr)) + + routeCmd(t, "delete", cidr) + assert.Never(t, func() bool { return routeVia(t, cidr) != 0 }, 3*routeRetryDelay, 50*time.Millisecond) + + // A reload with nothing changed is how to ask for it back + res := c.ReloadConfigString(cfg) + require.NoError(t, res.Err) + assert.False(t, res.Failed) + assert.Equal(t, ours, routeVia(t, cidr)) +} + +// configd deleting ours and adding the lan's more than routeRetryDelay apart, or a sweep landing in between, looks like +// a hand delete. Seeing the lan hold the prefix puts ours back in line, it goes back once theirs is removed +func TestTunYieldsAfterLeavingARouteDeletedByHand(t *testing.T) { + for name, tc := range map[string]struct { + cidr string + sweep time.Duration + misfile func(t *testing.T, tn *tun, cidr string) + }{ + // The table went quiet with nothing in its place. The minute sweep leaves noticing the lan's route to the watcher + "settled": {"10.251.6.0/24", 0, func(t *testing.T, tn *tun, cidr string) { + require.Eventually(t, func() bool { return routeStateOf(tn, cidr).kind == routeRemoved }, 5*time.Second, 50*time.Millisecond) + }}, + // The sweep that caps a table that never settles + "swept": {"10.251.7.0/24", 200 * time.Millisecond, func(t *testing.T, tn *tun, cidr string) { + tn.reconcileRoutes() + require.Equal(t, routeRemoved, routeStateOf(tn, cidr).kind) + }}, + } { + t.Run(name, func(t *testing.T) { + tn, _, _ := newRouteTestTun(t, tc.sweep, tc.cidr) + ours := tn.linkAddr.Load().Index + require.Equal(t, ours, routeVia(t, tc.cidr)) + + routeCmd(t, "delete", tc.cidr) + tc.misfile(t, tn, tc.cidr) + + lo0 := routeThroughLo0(t, tc.cidr) + require.Eventually(t, func() bool { + s := routeStateOf(tn, tc.cidr) + return s.kind == routeYielded && s.holder == lo0 + }, 5*time.Second, 50*time.Millisecond) + + routeCmd(t, "delete", tc.cidr) + assert.Eventually(t, func() bool { return routeVia(t, tc.cidr) == ours }, 5*time.Second, 50*time.Millisecond) + }) + } +} + +// Changing our route to another interface hands it the prefix, ours goes back once theirs is removed rather than being +// taken for a hand delete +func TestTunYieldsToARouteChangedAway(t *testing.T) { + for _, cidr := range []string{"10.251.8.0/24", "2001:db8:258::/64"} { + t.Run(cidr, func(t *testing.T) { + tn, _, _ := newRouteTestTun(t, 0, cidr) + ours := tn.linkAddr.Load().Index + require.Equal(t, ours, routeVia(t, cidr)) + + cleanupRoute(t, cidr) + routeCmd(t, "change", cidr, "-interface", "lo0") + lo0, err := net.InterfaceByName("lo0") + require.NoError(t, err) + require.Equal(t, lo0.Index, routeVia(t, cidr)) + require.Eventually(t, func() bool { + s := routeStateOf(tn, cidr) + return s.kind == routeYielded && s.holder == lo0.Index + }, 5*time.Second, 50*time.Millisecond) + + routeCmd(t, "delete", cidr) + assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond) + }) + } +} + +// Detaching an interface deletes its routes without a routing socket message for each, like unplugging the dock whose +// ethernet configd routed the lan through +func TestTunAddsRouteWhenItsInterfaceDetaches(t *testing.T) { + requireRoot(t) + var feth string + for i := 100; i < 200 && feth == ""; i++ { + if _, err := net.InterfaceByName(fmt.Sprintf("feth%d", i)); err != nil { + feth = fmt.Sprintf("feth%d", i) + } + } + require.NotEmpty(t, feth) + require.NoError(t, exec.Command("ifconfig", feth, "create").Run()) + t.Cleanup(func() { _ = exec.Command("ifconfig", feth, "destroy").Run() }) + require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5/24", "up").Run()) + + // Static and cloning with no gateway, like the lan routes configd installs. Named by the feth's address, macOS 27 + // refuses one named by the feth itself + cidr := "10.251.2.0/24" + routeCmd(t, "add", cidr, "-interface", "198.51.100.5") + cleanupRoute(t, cidr) + ifc, err := net.InterfaceByName(feth) + require.NoError(t, err) + + tn, _, _ := newRouteTestTun(t, 0, cidr) + require.Equal(t, ifc.Index, routeVia(t, cidr)) + + // Losing the address first leaves the route, the reconcile finds it still held + require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5", "-alias").Run()) + time.Sleep(2 * routeRetryDelay) + require.Equal(t, ifc.Index, routeVia(t, cidr)) + + // Only RTM_IFINFO says the interface went, its route goes with no message of its own + require.NoError(t, exec.Command("ifconfig", feth, "destroy").Run()) + assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond) +} + +// The routing socket drops what doesn't fit its buffer without a word, the sweep finds what it missed +func TestTunSweepAddsRouteTheWatcherMissed(t *testing.T) { + cidr := "10.251.5.0/24" + routeThroughLo0(t, cidr) + tn, _, _ := newRouteTestTun(t, 200*time.Millisecond, cidr) + + tn.routesMu.Lock() + require.NoError(t, tn.routeWatch.Close()) + tn.routesMu.Unlock() + + routeCmd(t, "delete", cidr) + assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond) +} + // Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free func TestTunDevNeedsRestart(t *testing.T) { for name, want := range map[string]bool{ diff --git a/overlay/tun_routes_darwin.go b/overlay/tun_routes_darwin.go new file mode 100644 index 00000000..1b41e4e7 --- /dev/null +++ b/overlay/tun_routes_darwin.go @@ -0,0 +1,451 @@ +//go:build !ios && !e2e_testing + +package overlay + +import ( + "errors" + "fmt" + "net" + "net/netip" + "os" + "strconv" + "sync/atomic" + "time" + "unsafe" + + "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/util" + netroute "golang.org/x/net/route" + "golang.org/x/sys/unix" +) + +// Darwin has no route metrics, the routing table holds one unscoped route per prefix. When another interface already +// routes an unsafe_route's prefix, like the lan we started on, theirs stays and ours is added once theirs is removed. +// Our own route removed by someone else, with nothing in its place, was removed on purpose and stays gone until a reload +// or until another interface takes the prefix, after which ours follows theirs like any other. + +const ( + // routeRetryDelay is how long the system routing table must be quiet before we judge who holds a prefix. configd + // replaces a route by deleting and adding, looking in between would find the prefix free + routeRetryDelay = time.Second + + // routeSweepInterval bounds how long a freed unsafe_route can go unnoticed. The routing socket drops messages that + // don't fit its receive buffer without telling the reader, and detaching an interface deletes its routes without + // a message for each + routeSweepInterval = time.Minute + + // routeSocketRcvBuf is well past the 8KiB default, which a burst of neighbor routes expiring can fill + routeSocketRcvBuf = 1 << 20 +) + +type routeStateKind int + +const ( + routeUnknown routeStateKind = iota + // routeInstalled is our route holding the prefix + routeInstalled + // routeYielded is another interface holding the prefix, ours is added once it lets go + routeYielded + // routeRemoved is our route deleted by someone else with nothing in its place, it is left alone until a reload or + // another interface takes the prefix + routeRemoved + // routeFailed is adding our route failing for some other reason, it is retried + routeFailed +) + +type routeState struct { + kind routeStateKind + // holder is the index of the interface holding the prefix when kind is routeYielded + holder int +} + +// routeMover is the process that last added, changed or deleted a route for an unsafe_route's prefix, for logs. Its name +// is looked up as the message arrives, route(8) has exited by the time the table settles +type routeMover struct { + pid int32 + name string +} + +// logArgs appends m to a log line's args, when we know who it was +func (m routeMover) logArgs(args ...any) []any { + if m.pid == 0 { + return args + } + args = append(args, "pid", m.pid) + if m.name != "" { + args = append(args, "process", m.name) + } + return args +} + +// processName is the name of process pid, truncated to 16 bytes, or empty when it has already exited +func processName(pid int32) string { + kp, err := unix.SysctlKinfoProc("kern.proc.pid", int(pid)) + if err != nil || kp.Proc.P_pid != pid { + return "" + } + return unix.ByteSliceToString(kp.Proc.P_comm[:]) +} + +// installRoutes returns the routes we put in the system routing table, one per prefix +func installRoutes(routes []Route) []Route { + seen := make(map[netip.Prefix]struct{}, len(routes)) + var out []Route + for _, r := range routes { + if len(r.Via) == 0 || !r.Install { + // We don't allow route MTUs so only install routes with a via + continue + } + p := r.Cidr.Masked() + if _, ok := seen[p]; ok { + continue + } + seen[p] = struct{}{} + out = append(out, r) + } + return out +} + +// addRoutes puts every unsafe_route in the system routing table, including ones someone deleted, unless another +// interface holds its prefix. It runs on start and on every reload. +func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error { + // The route tree is still ours, the system routing table is not + if t.hostOwned { + return nil + } + + t.routesMu.Lock() + defer t.routesMu.Unlock() + + // Activate hasn't run yet, it will install the routes + linkAddr := t.linkAddr.Load() + if linkAddr == nil || t.closed { + return nil + } + + var msgs []netroute.Message + var msgsErr error + fetched := false + states := make(map[netip.Prefix]routeState) + for _, r := range installRoutes(*t.Routes.Load()) { + p := r.Cidr.Masked() + prev := t.routeStates[p] + + err := addRoute(p, linkAddr) + if err == nil { + t.l.Info("Added route", "route", r) + states[p] = routeState{kind: routeInstalled} + continue + } + + if !errors.Is(err, unix.EEXIST) { + retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err) + if !logErrors { + return retErr + } + retErr.Log(t.l) + rl.Failed() + states[p] = routeState{kind: routeFailed} + continue + } + + // Every route we added already exists on a reload, only warn about someone else's + if !fetched { + msgs, msgsErr = fetchRoutes() + fetched = true + } + if msgsErr != nil { + t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", msgsErr) + states[p] = prev + continue + } + + switch holder := routeInterfaceIn(msgs, p); holder { + case linkAddr.Index: + states[p] = routeState{kind: routeInstalled} + case 0: + // Removed since the add, the watcher will see to it + states[p] = prev + default: + if prev.kind != routeYielded || prev.holder != holder { + t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface, it will be added once that route is removed", "route", r.Cidr, "interface", interfaceName(holder)) + } + states[p] = routeState{kind: routeYielded, holder: holder} + } + } + + t.routeStates = states + return nil +} + +func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error { + if t.hostOwned { + return nil + } + + t.routesMu.Lock() + defer t.routesMu.Unlock() + + linkAddr := t.linkAddr.Load() + if linkAddr == nil || t.closed { + return nil + } + + // The kernel deletes a route by its prefix alone, whichever interface holds it. Only delete the ones that are ours + msgs, err := fetchRoutes() + if err != nil { + var left []netip.Prefix + for _, r := range installRoutes(routes) { + left = append(left, r.Cidr) + } + return fmt.Errorf("failed to read the system routing table, leaving routes %v installed: %w", left, err) + } + + for _, r := range installRoutes(routes) { + p := r.Cidr.Masked() + // Whatever happens next, the prefix is no longer one we know about + delete(t.routeStates, p) + if routeInterfaceIn(msgs, p) != linkAddr.Index { + continue + } + + err = delRoute(p, linkAddr) + if errors.Is(err, unix.ESRCH) { + // Already gone, which is what we wanted + t.l.Info("Route was already removed", "route", r) + } else if err != nil { + t.l.Error("Failed to remove route", "error", err, "route", r) + rl.Failed() + } else { + t.l.Info("Removed route", "route", r) + } + } + return nil +} + +// reconcileRoutes brings the unsafe_routes in line with who holds their prefix now. It is quiet unless something +// changed since it last looked, it runs every routeSweepEvery. +func (t *tun) reconcileRoutes() { + t.routesMu.Lock() + defer t.routesMu.Unlock() + + linkAddr := t.linkAddr.Load() + if linkAddr == nil || t.closed { + return + } + + t.routeMoversMu.Lock() + movers := t.routeMovers + t.routeMovers = nil + t.routeMoversMu.Unlock() + + msgs, err := fetchRoutes() + if err != nil { + t.l.Error("Failed to read the system routing table", "error", err) + return + } + + states := make(map[netip.Prefix]routeState) + for _, r := range installRoutes(*t.Routes.Load()) { + p := r.Cidr.Masked() + prev := t.routeStates[p] + by := movers[p] + + holder := routeInterfaceIn(msgs, p) + switch { + case holder == linkAddr.Index: + states[p] = routeState{kind: routeInstalled} + + case holder != 0: + // Another interface taking a prefix we left alone puts ours back in line behind theirs + if prev.kind != routeYielded || prev.holder != holder { + t.l.Info("unsafe_route is routed through another interface, it will be added back when that route is removed", by.logArgs("route", r.Cidr, "interface", interfaceName(holder))...) + } + states[p] = routeState{kind: routeYielded, holder: holder} + + case prev.kind == routeInstalled: + t.l.Warn("unsafe_route was removed outside nebula, leaving it until a reload or another interface takes its destination", by.logArgs("route", r.Cidr)...) + states[p] = routeState{kind: routeRemoved} + + case prev.kind == routeRemoved: + states[p] = prev + + default: + err := addRoute(p, linkAddr) + switch { + case err == nil: + if prev.kind == routeYielded { + t.l.Info("unsafe_route's destination is free again, added it", by.logArgs("route", r)...) + } else { + t.l.Info("Added route", "route", r) + } + states[p] = routeState{kind: routeInstalled} + case errors.Is(err, unix.EEXIST): + // Taken since the fetch, the next look will see by whom + states[p] = prev + default: + if prev.kind != routeFailed { + util.NewContextualError("Failed to add route", map[string]any{"route": r}, err).Log(t.l) + } + states[p] = routeState{kind: routeFailed} + } + } + } + t.routeStates = states +} + +// startRouteWatch starts reconciling the unsafe_routes, every routeSweepEvery and, when watch isn't nil, once the +// routing table settles after anything that may have moved a prefix +func (t *tun) startRouteWatch(watch *os.File) { + t.routesMu.Lock() + defer t.routesMu.Unlock() + if t.closed { + if watch != nil { + _ = watch.Close() + } + return + } + + t.routeSweep = time.AfterFunc(t.routeSweepEvery, t.sweepRoutes) + if watch != nil { + t.routeWatch = watch + go t.watchRoutes(watch) + } +} + +func (t *tun) sweepRoutes() { + // A table still changing gets reconciled once it settles + if time.Since(time.Unix(0, t.lastRouteEvent.Load())) >= routeRetryDelay { + t.reconcileRoutes() + } + + t.routesMu.Lock() + defer t.routesMu.Unlock() + if !t.closed { + t.routeSweep.Reset(t.routeSweepEvery) + } +} + +func openRouteSocket() (*os.File, error) { + fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) + if err != nil { + return nil, fmt.Errorf("unable to create AF_ROUTE socket: %w", err) + } + // Fewer drops, the sweep catches the rest + _ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, routeSocketRcvBuf) + // Non-blocking puts it on the poller, so Close interrupts a Read + if err = unix.SetNonblock(fd, true); err != nil { + _ = unix.Close(fd) + return nil, fmt.Errorf("failed to set the AF_ROUTE socket to non-blocking mode: %w", err) + } + return os.NewFile(uintptr(fd), "route"), nil +} + +// watchRoutes reconciles the unsafe_routes once the routing table has been quiet for routeRetryDelay after a message +// that may have moved a prefix. A table that keeps changing is reconciled every routeSweepEvery regardless. +func (t *tun) watchRoutes(f *os.File) { + var reconciled atomic.Int64 + settle := time.AfterFunc(time.Hour, func() { + reconciled.Store(time.Now().UnixNano()) + t.reconcileRoutes() + }) + settle.Stop() + defer settle.Stop() + + var burst time.Time + self := int32(os.Getpid()) + b := make([]byte, os.Getpagesize()) + for { + n, err := f.Read(b) + if err != nil { + if !errors.Is(err, os.ErrClosed) { + t.l.Error("Stopped watching the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err) + } + return + } + + moved, p, pid := routesMayHaveMoved(b[:n], *t.Routes.Load()) + if !moved { + continue + } + // Our own are already in routeStates + if p.IsValid() && pid != self { + t.routeMoversMu.Lock() + if t.routeMovers == nil { + t.routeMovers = make(map[netip.Prefix]routeMover) + } + t.routeMovers[p] = routeMover{pid: pid, name: processName(pid)} + t.routeMoversMu.Unlock() + } + + now := time.Now() + t.lastRouteEvent.Store(now.UnixNano()) + if burst.IsZero() || reconciled.Load() >= burst.UnixNano() { + burst = now + } + settle.Reset(max(0, min(routeRetryDelay, time.Until(burst.Add(t.routeSweepEvery))))) + } +} + +// routesMayHaveMoved reports whether b, one routing socket message, may have freed a prefix one of routes installs or +// handed it to another interface. That's a route for it added, changed or removed, or an interface that lost an address +// or changed state, since detaching an interface deletes its routes without a message of their own. The header is read +// directly, x/net/route skips the RTM_IFINFO the socket sends, which has no interface address, and misreads the +// unpadded sockaddrs in messages the kernel makes itself. A request naming one of the prefixes also returns that prefix +// and the pid of the process that made it. +func routesMayHaveMoved(b []byte, routes []Route) (moved bool, p netip.Prefix, pid int32) { + if len(b) < 4 { + return false, p, 0 + } + switch int(b[3]) { + case unix.RTM_IFINFO, unix.RTM_DELADDR: + return true, p, 0 + case unix.RTM_ADD, unix.RTM_CHANGE, unix.RTM_DELETE: + default: + return false, p, 0 + } + + if len(b) < unix.SizeofRtMsghdr { + return false, p, 0 + } + hdr := (*unix.RtMsghdr)(unsafe.Pointer(&b[0])) + // A failed request is reported too, and a scoped route never held the prefix + if hdr.Errno != 0 || hdr.Flags&unix.RTF_IFSCOPE != 0 { + return false, p, 0 + } + + if hdr.Pid == 0 { + // The kernel's own, which x/net/route can't parse. Neighbors coming and going are host routes, anything else + // could be holding a prefix + return hdr.Flags&(unix.RTF_HOST|unix.RTF_LLINFO) == 0, p, 0 + } + + // Replies to a request carry the requester's padded sockaddrs + msgs, err := netroute.ParseRIB(unix.NET_RT_DUMP, b) + if err != nil || len(msgs) != 1 { + return false, p, 0 + } + m, ok := msgs[0].(*netroute.RouteMessage) + if !ok || len(m.Addrs) <= unix.RTAX_NETMASK { + return false, p, 0 + } + p, ok = routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK]) + if !ok { + return false, netip.Prefix{}, 0 + } + p = p.Masked() + for _, r := range installRoutes(routes) { + if r.Cidr.Masked() == p { + return true, p, hdr.Pid + } + } + return false, netip.Prefix{}, 0 +} + +// interfaceName is the name of the interface with index i, for logs +func interfaceName(i int) string { + if ifc, err := net.InterfaceByIndex(i); err == nil { + return ifc.Name + } + return strconv.Itoa(i) +}