darwin: Add an unsafe_route once the route holding its prefix is removed (#1942)

overlay/tun_darwin.go

@@ -3,7 +3,6 @@
 package overlay
 
 import (
-	"errors"
 	"fmt"
 	"log/slog"
 	"math/bits"
@@ -12,6 +11,7 @@ import (
 	"sync"
 	"sync/atomic"
 	"syscall"
+	"time"
 	"unsafe"
 
 	"github.com/gaissmai/bart"
@@ -35,6 +35,26 @@ type tun struct {
 	// and routes for it. NEPacketTunnelProvider on darwin does this.
 	hostOwned bool
 	l         *slog.Logger
+
+	// routesMu serializes changes to the system routing table between reload and the route watcher, and guards the
+	// fields below it
+	routesMu sync.Mutex
+	// routeStates is who held each installable unsafe_route prefix when we last looked
+	routeStates map[netip.Prefix]routeState
+	// routeWatch is the routing socket watchRoutes reads, Close closes it
+	routeWatch *os.File
+	// routeSweep reconciles the unsafe_routes every routeSweepEvery
+	routeSweep      *time.Timer
+	routeSweepEvery time.Duration
+	closed          bool
+
+	// routeMovers is who last changed each unsafe_route prefix since the last reconcile, for its logs. It has its own
+	// lock so the route watcher never waits on a reconcile
+	routeMoversMu sync.Mutex
+	routeMovers   map[netip.Prefix]routeMover
+
+	// lastRouteEvent is when the routing socket last reported something that may have moved a prefix, in unix nanos
+	lastRouteEvent atomic.Int64
 }
 
 type ifReq struct {
@@ -139,11 +159,12 @@ func newTun(c *config.C, l *slog.Logger, vpnNetworks []netip.Prefix, _ bool) (*t
 	}
 
 	t := &tun{
-		f:           os.NewFile(uintptr(fd), ""),
-		Device:      name,
-		vpnNetworks: vpnNetworks,
-		DefaultMTU:  getMTU(l, c),
-		l:           l,
+		f:               os.NewFile(uintptr(fd), ""),
+		Device:          name,
+		vpnNetworks:     vpnNetworks,
+		DefaultMTU:      getMTU(l, c),
+		routeSweepEvery: routeSweepInterval,
+		l:               l,
 	}
 
 	err = t.reload(c, true, nil)
@@ -219,6 +240,16 @@ func utunNameFromFd(fd int) string {
 }
 
 func (t *tun) Close() error {
+	t.routesMu.Lock()
+	t.closed = true
+	if t.routeWatch != nil {
+		_ = t.routeWatch.Close()
+	}
+	if t.routeSweep != nil {
+		t.routeSweep.Stop()
+	}
+	t.routesMu.Unlock()
+
 	if t.f != nil {
 		return t.f.Close()
 	}
@@ -287,8 +318,22 @@ func (t *tun) Activate() error {
 		return fmt.Errorf("failed to run tun device: %s", err)
 	}
 
+	// Watch before adding so a route removed in between is still noticed
+	watch, err := openRouteSocket()
+	if err != nil {
+		t.l.Error("Failed to watch the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
+	}
+
 	// Unsafe path routes
-	return t.addRoutes(false, nil)
+	if err = t.addRoutes(false, nil); err != nil {
+		if watch != nil {
+			_ = watch.Close()
+		}
+		return err
+	}
+
+	t.startRouteWatch(watch)
+	return nil
 }
 
 // activate4 assigns our vpn address as a /32 and routes the vpn network to the tun explicitly.
@@ -393,7 +438,8 @@ func (t *tun) reload(c *config.C, initial bool, r *config.Reload) error {
 	}
 
 	if !initial && !change {
-		return nil
+		// Any reload puts back the unsafe_routes someone deleted
+		return t.addRoutes(true, r)
 	}
 
 	routeTree, err := makeRouteTree(t.l, routes, false)
@@ -511,82 +557,6 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
 	return nil, nil
 }
 
-func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
-	// The route tree is still ours, the system routing table is not
-	if t.hostOwned {
-		return nil
-	}
-
-	// Activate hasn't run yet, it will install the routes
-	linkAddr := t.linkAddr.Load()
-	if linkAddr == nil {
-		return nil
-	}
-
-	routes := *t.Routes.Load()
-	systemRoutes := sync.OnceValues(fetchRoutes)
-
-	for _, r := range routes {
-		if len(r.Via) == 0 || !r.Install {
-			// We don't allow route MTUs so only install routes with a via
-			continue
-		}
-
-		err := addRoute(r.Cidr, linkAddr)
-		if err != nil {
-			if errors.Is(err, unix.EEXIST) {
-				// Every route we added already exists on a reload, only warn about someone else's
-				msgs, err := systemRoutes()
-				if err != nil {
-					t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", err)
-				} else if routeInterfaceIn(msgs, r.Cidr) != linkAddr.Index {
-					t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface", "route", r.Cidr)
-				}
-			} else {
-				retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
-				if logErrors {
-					retErr.Log(t.l)
-					rl.Failed()
-				} else {
-					return retErr
-				}
-			}
-		} else {
-			t.l.Info("Added route", "route", r)
-			// A cidr listed twice would otherwise find our own route missing from an older fetch
-			systemRoutes = sync.OnceValues(fetchRoutes)
-		}
-	}
-
-	return nil
-}
-
-func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
-	linkAddr := t.linkAddr.Load()
-	if t.hostOwned || linkAddr == nil {
-		return nil
-	}
-
-	for _, r := range routes {
-		if len(r.Via) == 0 || !r.Install {
-			// addRoutes never installs these
-			continue
-		}
-
-		err := delRoute(r.Cidr, linkAddr)
-		if errors.Is(err, unix.ESRCH) {
-			// Already gone, which is what we wanted
-			t.l.Info("Route was already removed", "route", r)
-		} else if err != nil {
-			t.l.Error("Failed to remove route", "error", err, "route", r)
-			rl.Failed()
-		} else {
-			t.l.Info("Removed route", "route", r)
-		}
-	}
-	return nil
-}
-
 func addRoute(prefix netip.Prefix, gateway netroute.Addr) error {
 	sock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
 	if err != nil {

overlay/tun_darwin_route_test.go

@@ -3,12 +3,17 @@
 package overlay
 
 import (
+	"encoding/binary"
+	"encoding/hex"
+	"fmt"
 	"net"
 	"net/netip"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/slackhq/nebula/config"
 	"github.com/slackhq/nebula/routing"
@@ -104,6 +109,367 @@ func TestTunReloadLeavesRoutesItNeverInstalled(t *testing.T) {
 	assert.Equal(t, lo0.Index, routeInterfaceIn(msgs, netip.MustParsePrefix(cidr)))
 }
 
+// Captured on macOS 15.6 while adding and removing 198.51.100.77/24 on a feth
+const (
+	// The kernel deleting the address's subnet route. It packs its sockaddrs without padding, x/net/route can't parse it
+	kernelSubnetDelete = "870005022300000000010002070000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010020000c63364000000000000000000141223000600000000000000000000000000000007ffffffffffff"
+	addressRemoved     = "5000050db400000001010000230000000000000007020000ffffff00141223000606060066657468373766657468004d10020000c633644d000000000000000010020000c63364ff0000000000000000"
+	// With no interface address, which x/net/route skips
+	interfaceChanged = "7000050e000000004288000023000000060000060e000000dc05000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000073e5ce6a0000000000000000000001000000000000000000"
+)
+
+func TestRoutesMayHaveMoved(t *testing.T) {
+	cidr := netip.MustParsePrefix("192.168.1.0/24")
+	via := routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}
+	routes := []Route{{Cidr: cidr, Via: via, Install: true}}
+	// A request like route(8) sends, the kernel echoes it to every routing socket
+	request := func(typ, flags int, dst, mask [4]byte) []byte {
+		m := &netroute.RouteMessage{
+			Version: unix.RTM_VERSION,
+			Type:    typ,
+			Flags:   flags,
+			ID:      1234,
+			Addrs: []netroute.Addr{
+				unix.RTAX_DST:     &netroute.Inet4Addr{IP: dst},
+				unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
+				unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: mask},
+			},
+		}
+		b, err := m.Marshal()
+		require.NoError(t, err)
+		return b
+	}
+	raw := func(s string) []byte {
+		b, err := hex.DecodeString(s)
+		require.NoError(t, err)
+		return b
+	}
+	lan, mask := [4]byte{192, 168, 1, 0}, [4]byte{255, 255, 255, 0}
+	routesMayHaveMoved := func(b []byte, routes []Route) bool {
+		moved, _, _ := routesMayHaveMoved(b, routes)
+		return moved
+	}
+
+	assert.True(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), routes))
+	assert.True(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP, lan, mask), routes))
+	assert.True(t, routesMayHaveMoved(request(unix.RTM_CHANGE, unix.RTF_UP, lan, mask), routes))
+	assert.False(t, routesMayHaveMoved(request(unix.RTM_GET, unix.RTF_UP, lan, mask), routes))
+	assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "a scoped route never held the prefix")
+	assert.False(t, routesMayHaveMoved(request(unix.RTM_ADD, unix.RTF_UP|unix.RTF_IFSCOPE, lan, mask), routes), "configd adds a scoped copy for each interface")
+	assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, [4]byte{255, 255, 0, 0}), routes), "only an exact prefix counts")
+	assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Install: true}}), "a route with no via is never installed")
+	assert.False(t, routesMayHaveMoved(request(unix.RTM_DELETE, unix.RTF_UP, lan, mask), []Route{{Cidr: cidr, Via: via}}), "install: false")
+
+	failed := request(unix.RTM_DELETE, unix.RTF_UP, lan, mask)
+	binary.NativeEndian.PutUint32(failed[24:28], uint32(unix.ESRCH))
+	assert.False(t, routesMayHaveMoved(failed, routes), "a failed request is echoed too")
+
+	_, err := netroute.ParseRIB(unix.NET_RT_DUMP, raw(kernelSubnetDelete))
+	require.Error(t, err, "if x/net/route learns to parse these, match their prefix instead")
+	assert.True(t, routesMayHaveMoved(raw(kernelSubnetDelete), routes), "the kernel's own deletes could be any prefix")
+	neighbor := raw(kernelSubnetDelete)
+	binary.NativeEndian.PutUint32(neighbor[8:12], unix.RTF_UP|unix.RTF_HOST|unix.RTF_LLINFO)
+	assert.False(t, routesMayHaveMoved(neighbor, routes), "a neighbor expiring")
+
+	assert.True(t, routesMayHaveMoved(raw(addressRemoved), routes))
+	assert.True(t, routesMayHaveMoved(raw(interfaceChanged), routes), "detaching an interface deletes its routes without a message for each")
+}
+
+// A request names its prefix and who made it, the kernel's own and interface changes name neither
+func TestRoutesMayHaveMovedSaysWho(t *testing.T) {
+	cidr := netip.MustParsePrefix("192.168.1.0/24")
+	routes := []Route{{Cidr: cidr, Via: routing.Gateways{routing.NewGateway(netip.MustParseAddr("10.250.0.2"), 1)}, Install: true}}
+	m := &netroute.RouteMessage{
+		Version: unix.RTM_VERSION,
+		Type:    unix.RTM_CHANGE,
+		Flags:   unix.RTF_UP,
+		ID:      1234,
+		Addrs: []netroute.Addr{
+			unix.RTAX_DST:     &netroute.Inet4Addr{IP: [4]byte{192, 168, 1, 0}},
+			unix.RTAX_GATEWAY: &netroute.LinkAddr{Index: 4},
+			unix.RTAX_NETMASK: &netroute.Inet4Addr{IP: [4]byte{255, 255, 255, 0}},
+		},
+	}
+	b, err := m.Marshal()
+	require.NoError(t, err)
+	moved, p, pid := routesMayHaveMoved(b, routes)
+	assert.True(t, moved)
+	assert.Equal(t, cidr, p)
+	assert.Equal(t, int32(1234), pid)
+
+	for name, s := range map[string]string{"kernel": kernelSubnetDelete, "ifinfo": interfaceChanged} {
+		b, err := hex.DecodeString(s)
+		require.NoError(t, err)
+		moved, p, pid := routesMayHaveMoved(b, routes)
+		assert.True(t, moved, name)
+		assert.False(t, p.IsValid(), name)
+		assert.Zero(t, pid, name)
+	}
+}
+
+func TestProcessName(t *testing.T) {
+	assert.Equal(t, "launchd", processName(1))
+	self := filepath.Base(os.Args[0])
+	assert.Equal(t, self[:min(len(self), 16)], processName(int32(os.Getpid())))
+	assert.Empty(t, processName(-1))
+}
+
+// newRouteTestTun is an activated tun for the root tests below with an unsafe_route for each of cidrs. sweep replaces
+// the minute between sweeps when it isn't zero.
+func newRouteTestTun(t *testing.T, sweep time.Duration, cidrs ...string) (*tun, *config.C, string) {
+	requireRoot(t)
+
+	cfg := "tun:\n  unsafe_routes:\n"
+	for _, cidr := range cidrs {
+		via := "10.250.0.2"
+		if netip.MustParsePrefix(cidr).Addr().Is6() {
+			via = "fd00:250::2"
+		}
+		cfg += "    - route: " + cidr + "\n      via: " + via + "\n"
+	}
+
+	l := test.NewLogger()
+	c := config.NewC(l)
+	require.NoError(t, c.LoadString(cfg))
+	tn, err := newTun(c, l, []netip.Prefix{netip.MustParsePrefix("10.250.0.1/24"), netip.MustParsePrefix("fd00:250::1/64")}, false)
+	require.NoError(t, err)
+	if sweep != 0 {
+		tn.routeSweepEvery = sweep
+	}
+	t.Cleanup(func() { _ = tn.Close() })
+	require.NoError(t, tn.Activate())
+	return tn, c, cfg
+}
+
+func requireRoot(t *testing.T) {
+	if os.Geteuid() != 0 {
+		t.Skip("creating interfaces and adding routes needs root")
+	}
+}
+
+// routeVia is the index of the interface that routes cidr, 0 if none does
+func routeVia(t *testing.T, cidr string) int {
+	msgs, err := fetchRoutes()
+	require.NoError(t, err)
+	return routeInterfaceIn(msgs, netip.MustParsePrefix(cidr))
+}
+
+// routeStateOf is what tn made of cidr when it last looked
+func routeStateOf(tn *tun, cidr string) routeState {
+	tn.routesMu.Lock()
+	defer tn.routesMu.Unlock()
+	return tn.routeStates[netip.MustParsePrefix(cidr)]
+}
+
+func routeArgs(action, cidr string, extra ...string) []string {
+	args := []string{"-q", "-n", action}
+	if netip.MustParsePrefix(cidr).Addr().Is6() {
+		args = append(args, "-inet6")
+	}
+	return append(append(args, "-net", cidr), extra...)
+}
+
+func routeCmd(t *testing.T, action, cidr string, extra ...string) {
+	requireRouteOK(t, exec.Command("route", routeArgs(action, cidr, extra...)...))
+}
+
+// requireRouteOK runs cmd, which runs route(8). route exits 0 when the kernel rejects its request, only what it prints
+// says so
+func requireRouteOK(t *testing.T, cmd *exec.Cmd) {
+	out, err := cmd.CombinedOutput()
+	require.NoError(t, err, string(out))
+	require.NotContains(t, string(out), "writing to routing socket")
+}
+
+// cleanupRoute deletes cidr's route when the test ends, whoever holds it
+func cleanupRoute(t *testing.T, cidr string) {
+	t.Cleanup(func() { _ = exec.Command("route", routeArgs("delete", cidr)...).Run() })
+}
+
+// routeThroughLo0 routes cidr through lo0, like a lan nebula started on
+func routeThroughLo0(t *testing.T, cidr string) int {
+	requireRoot(t)
+	routeCmd(t, "add", cidr, "-interface", "lo0")
+	cleanupRoute(t, cidr)
+	lo0, err := net.InterfaceByName("lo0")
+	require.NoError(t, err)
+	return lo0.Index
+}
+
+// Leaving the lan that held an unsafe_route's prefix must not need a restart to get the unsafe_route
+func TestTunAddsRouteOnceItsPrefixIsFree(t *testing.T) {
+	for _, cidr := range []string{"10.251.0.0/24", "2001:db8:251::/64"} {
+		t.Run(cidr, func(t *testing.T) {
+			lo0 := routeThroughLo0(t, cidr)
+			tn, _, _ := newRouteTestTun(t, 0, cidr)
+			require.Equal(t, lo0, routeVia(t, cidr), "an existing route keeps its prefix")
+
+			routeCmd(t, "delete", cidr)
+			assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
+		})
+	}
+}
+
+// The kernel deletes a route by prefix alone, so dropping an unsafe_route someone else holds must not delete theirs
+func TestTunReloadLeavesRouteItYielded(t *testing.T) {
+	cidr := "10.251.1.0/24"
+	lo0 := routeThroughLo0(t, cidr)
+	_, c, _ := newRouteTestTun(t, 0, cidr)
+
+	res := c.ReloadConfigString("tun: {}\n")
+	require.NoError(t, res.Err)
+	assert.False(t, res.Failed)
+	assert.Equal(t, lo0, routeVia(t, cidr))
+}
+
+// configd takes a prefix by deleting the route holding it and adding its own. Ours goes back once theirs is removed
+func TestTunYieldsToARouteThatReplacesIt(t *testing.T) {
+	cidr := "10.251.3.0/24"
+	tn, _, _ := newRouteTestTun(t, 0, cidr)
+	ours := tn.linkAddr.Load().Index
+	require.Equal(t, ours, routeVia(t, cidr))
+
+	cleanupRoute(t, cidr)
+	requireRouteOK(t, exec.Command("sh", "-c", "route -q -n delete -net "+cidr+" && route -q -n add -net "+cidr+" -interface lo0"))
+	lo0, err := net.InterfaceByName("lo0")
+	require.NoError(t, err)
+	time.Sleep(2 * routeRetryDelay)
+	require.Equal(t, lo0.Index, routeVia(t, cidr))
+
+	routeCmd(t, "delete", cidr)
+	assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
+}
+
+// A route someone deleted, with nothing in its place, stays gone until a reload, so they can put their own there
+func TestTunLeavesRouteDeletedByHand(t *testing.T) {
+	cidr := "10.251.4.0/24"
+	tn, c, cfg := newRouteTestTun(t, 200*time.Millisecond, cidr)
+	ours := tn.linkAddr.Load().Index
+	require.Equal(t, ours, routeVia(t, cidr))
+
+	routeCmd(t, "delete", cidr)
+	assert.Never(t, func() bool { return routeVia(t, cidr) != 0 }, 3*routeRetryDelay, 50*time.Millisecond)
+
+	// A reload with nothing changed is how to ask for it back
+	res := c.ReloadConfigString(cfg)
+	require.NoError(t, res.Err)
+	assert.False(t, res.Failed)
+	assert.Equal(t, ours, routeVia(t, cidr))
+}
+
+// configd deleting ours and adding the lan's more than routeRetryDelay apart, or a sweep landing in between, looks like
+// a hand delete. Seeing the lan hold the prefix puts ours back in line, it goes back once theirs is removed
+func TestTunYieldsAfterLeavingARouteDeletedByHand(t *testing.T) {
+	for name, tc := range map[string]struct {
+		cidr    string
+		sweep   time.Duration
+		misfile func(t *testing.T, tn *tun, cidr string)
+	}{
+		// The table went quiet with nothing in its place. The minute sweep leaves noticing the lan's route to the watcher
+		"settled": {"10.251.6.0/24", 0, func(t *testing.T, tn *tun, cidr string) {
+			require.Eventually(t, func() bool { return routeStateOf(tn, cidr).kind == routeRemoved }, 5*time.Second, 50*time.Millisecond)
+		}},
+		// The sweep that caps a table that never settles
+		"swept": {"10.251.7.0/24", 200 * time.Millisecond, func(t *testing.T, tn *tun, cidr string) {
+			tn.reconcileRoutes()
+			require.Equal(t, routeRemoved, routeStateOf(tn, cidr).kind)
+		}},
+	} {
+		t.Run(name, func(t *testing.T) {
+			tn, _, _ := newRouteTestTun(t, tc.sweep, tc.cidr)
+			ours := tn.linkAddr.Load().Index
+			require.Equal(t, ours, routeVia(t, tc.cidr))
+
+			routeCmd(t, "delete", tc.cidr)
+			tc.misfile(t, tn, tc.cidr)
+
+			lo0 := routeThroughLo0(t, tc.cidr)
+			require.Eventually(t, func() bool {
+				s := routeStateOf(tn, tc.cidr)
+				return s.kind == routeYielded && s.holder == lo0
+			}, 5*time.Second, 50*time.Millisecond)
+
+			routeCmd(t, "delete", tc.cidr)
+			assert.Eventually(t, func() bool { return routeVia(t, tc.cidr) == ours }, 5*time.Second, 50*time.Millisecond)
+		})
+	}
+}
+
+// Changing our route to another interface hands it the prefix, ours goes back once theirs is removed rather than being
+// taken for a hand delete
+func TestTunYieldsToARouteChangedAway(t *testing.T) {
+	for _, cidr := range []string{"10.251.8.0/24", "2001:db8:258::/64"} {
+		t.Run(cidr, func(t *testing.T) {
+			tn, _, _ := newRouteTestTun(t, 0, cidr)
+			ours := tn.linkAddr.Load().Index
+			require.Equal(t, ours, routeVia(t, cidr))
+
+			cleanupRoute(t, cidr)
+			routeCmd(t, "change", cidr, "-interface", "lo0")
+			lo0, err := net.InterfaceByName("lo0")
+			require.NoError(t, err)
+			require.Equal(t, lo0.Index, routeVia(t, cidr))
+			require.Eventually(t, func() bool {
+				s := routeStateOf(tn, cidr)
+				return s.kind == routeYielded && s.holder == lo0.Index
+			}, 5*time.Second, 50*time.Millisecond)
+
+			routeCmd(t, "delete", cidr)
+			assert.Eventually(t, func() bool { return routeVia(t, cidr) == ours }, 5*time.Second, 50*time.Millisecond)
+		})
+	}
+}
+
+// Detaching an interface deletes its routes without a routing socket message for each, like unplugging the dock whose
+// ethernet configd routed the lan through
+func TestTunAddsRouteWhenItsInterfaceDetaches(t *testing.T) {
+	requireRoot(t)
+	var feth string
+	for i := 100; i < 200 && feth == ""; i++ {
+		if _, err := net.InterfaceByName(fmt.Sprintf("feth%d", i)); err != nil {
+			feth = fmt.Sprintf("feth%d", i)
+		}
+	}
+	require.NotEmpty(t, feth)
+	require.NoError(t, exec.Command("ifconfig", feth, "create").Run())
+	t.Cleanup(func() { _ = exec.Command("ifconfig", feth, "destroy").Run() })
+	require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5/24", "up").Run())
+
+	// Static and cloning with no gateway, like the lan routes configd installs. Named by the feth's address, macOS 27
+	// refuses one named by the feth itself
+	cidr := "10.251.2.0/24"
+	routeCmd(t, "add", cidr, "-interface", "198.51.100.5")
+	cleanupRoute(t, cidr)
+	ifc, err := net.InterfaceByName(feth)
+	require.NoError(t, err)
+
+	tn, _, _ := newRouteTestTun(t, 0, cidr)
+	require.Equal(t, ifc.Index, routeVia(t, cidr))
+
+	// Losing the address first leaves the route, the reconcile finds it still held
+	require.NoError(t, exec.Command("ifconfig", feth, "inet", "198.51.100.5", "-alias").Run())
+	time.Sleep(2 * routeRetryDelay)
+	require.Equal(t, ifc.Index, routeVia(t, cidr))
+
+	// Only RTM_IFINFO says the interface went, its route goes with no message of its own
+	require.NoError(t, exec.Command("ifconfig", feth, "destroy").Run())
+	assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
+}
+
+// The routing socket drops what doesn't fit its buffer without a word, the sweep finds what it missed
+func TestTunSweepAddsRouteTheWatcherMissed(t *testing.T) {
+	cidr := "10.251.5.0/24"
+	routeThroughLo0(t, cidr)
+	tn, _, _ := newRouteTestTun(t, 200*time.Millisecond, cidr)
+
+	tn.routesMu.Lock()
+	require.NoError(t, tn.routeWatch.Close())
+	tn.routesMu.Unlock()
+
+	routeCmd(t, "delete", cidr)
+	assert.Eventually(t, func() bool { return routeVia(t, cidr) == tn.linkAddr.Load().Index }, 5*time.Second, 50*time.Millisecond)
+}
+
 // Only naming a different utun asks for a restart. Empty, plain utun and names that aren't utunN take whichever is free
 func TestTunDevNeedsRestart(t *testing.T) {
 	for name, want := range map[string]bool{

overlay/tun_routes_darwin.go

@@ -0,0 +1,451 @@
+//go:build !ios && !e2e_testing
+
+package overlay
+
+import (
+	"errors"
+	"fmt"
+	"net"
+	"net/netip"
+	"os"
+	"strconv"
+	"sync/atomic"
+	"time"
+	"unsafe"
+
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/util"
+	netroute "golang.org/x/net/route"
+	"golang.org/x/sys/unix"
+)
+
+// Darwin has no route metrics, the routing table holds one unscoped route per prefix. When another interface already
+// routes an unsafe_route's prefix, like the lan we started on, theirs stays and ours is added once theirs is removed.
+// Our own route removed by someone else, with nothing in its place, was removed on purpose and stays gone until a reload
+// or until another interface takes the prefix, after which ours follows theirs like any other.
+
+const (
+	// routeRetryDelay is how long the system routing table must be quiet before we judge who holds a prefix. configd
+	// replaces a route by deleting and adding, looking in between would find the prefix free
+	routeRetryDelay = time.Second
+
+	// routeSweepInterval bounds how long a freed unsafe_route can go unnoticed. The routing socket drops messages that
+	// don't fit its receive buffer without telling the reader, and detaching an interface deletes its routes without
+	// a message for each
+	routeSweepInterval = time.Minute
+
+	// routeSocketRcvBuf is well past the 8KiB default, which a burst of neighbor routes expiring can fill
+	routeSocketRcvBuf = 1 << 20
+)
+
+type routeStateKind int
+
+const (
+	routeUnknown routeStateKind = iota
+	// routeInstalled is our route holding the prefix
+	routeInstalled
+	// routeYielded is another interface holding the prefix, ours is added once it lets go
+	routeYielded
+	// routeRemoved is our route deleted by someone else with nothing in its place, it is left alone until a reload or
+	// another interface takes the prefix
+	routeRemoved
+	// routeFailed is adding our route failing for some other reason, it is retried
+	routeFailed
+)
+
+type routeState struct {
+	kind routeStateKind
+	// holder is the index of the interface holding the prefix when kind is routeYielded
+	holder int
+}
+
+// routeMover is the process that last added, changed or deleted a route for an unsafe_route's prefix, for logs. Its name
+// is looked up as the message arrives, route(8) has exited by the time the table settles
+type routeMover struct {
+	pid  int32
+	name string
+}
+
+// logArgs appends m to a log line's args, when we know who it was
+func (m routeMover) logArgs(args ...any) []any {
+	if m.pid == 0 {
+		return args
+	}
+	args = append(args, "pid", m.pid)
+	if m.name != "" {
+		args = append(args, "process", m.name)
+	}
+	return args
+}
+
+// processName is the name of process pid, truncated to 16 bytes, or empty when it has already exited
+func processName(pid int32) string {
+	kp, err := unix.SysctlKinfoProc("kern.proc.pid", int(pid))
+	if err != nil || kp.Proc.P_pid != pid {
+		return ""
+	}
+	return unix.ByteSliceToString(kp.Proc.P_comm[:])
+}
+
+// installRoutes returns the routes we put in the system routing table, one per prefix
+func installRoutes(routes []Route) []Route {
+	seen := make(map[netip.Prefix]struct{}, len(routes))
+	var out []Route
+	for _, r := range routes {
+		if len(r.Via) == 0 || !r.Install {
+			// We don't allow route MTUs so only install routes with a via
+			continue
+		}
+		p := r.Cidr.Masked()
+		if _, ok := seen[p]; ok {
+			continue
+		}
+		seen[p] = struct{}{}
+		out = append(out, r)
+	}
+	return out
+}
+
+// addRoutes puts every unsafe_route in the system routing table, including ones someone deleted, unless another
+// interface holds its prefix. It runs on start and on every reload.
+func (t *tun) addRoutes(logErrors bool, rl *config.Reload) error {
+	// The route tree is still ours, the system routing table is not
+	if t.hostOwned {
+		return nil
+	}
+
+	t.routesMu.Lock()
+	defer t.routesMu.Unlock()
+
+	// Activate hasn't run yet, it will install the routes
+	linkAddr := t.linkAddr.Load()
+	if linkAddr == nil || t.closed {
+		return nil
+	}
+
+	var msgs []netroute.Message
+	var msgsErr error
+	fetched := false
+	states := make(map[netip.Prefix]routeState)
+	for _, r := range installRoutes(*t.Routes.Load()) {
+		p := r.Cidr.Masked()
+		prev := t.routeStates[p]
+
+		err := addRoute(p, linkAddr)
+		if err == nil {
+			t.l.Info("Added route", "route", r)
+			states[p] = routeState{kind: routeInstalled}
+			continue
+		}
+
+		if !errors.Is(err, unix.EEXIST) {
+			retErr := util.NewContextualError("Failed to add route", map[string]any{"route": r}, err)
+			if !logErrors {
+				return retErr
+			}
+			retErr.Log(t.l)
+			rl.Failed()
+			states[p] = routeState{kind: routeFailed}
+			continue
+		}
+
+		// Every route we added already exists on a reload, only warn about someone else's
+		if !fetched {
+			msgs, msgsErr = fetchRoutes()
+			fetched = true
+		}
+		if msgsErr != nil {
+			t.l.Warn("unable to add unsafe_route, a route already exists but its interface could not be determined", "route", r.Cidr, "error", msgsErr)
+			states[p] = prev
+			continue
+		}
+
+		switch holder := routeInterfaceIn(msgs, p); holder {
+		case linkAddr.Index:
+			states[p] = routeState{kind: routeInstalled}
+		case 0:
+			// Removed since the add, the watcher will see to it
+			states[p] = prev
+		default:
+			if prev.kind != routeYielded || prev.holder != holder {
+				t.l.Warn("unable to add unsafe_route, the destination is already routed through another interface, it will be added once that route is removed", "route", r.Cidr, "interface", interfaceName(holder))
+			}
+			states[p] = routeState{kind: routeYielded, holder: holder}
+		}
+	}
+
+	t.routeStates = states
+	return nil
+}
+
+func (t *tun) removeRoutes(routes []Route, rl *config.Reload) error {
+	if t.hostOwned {
+		return nil
+	}
+
+	t.routesMu.Lock()
+	defer t.routesMu.Unlock()
+
+	linkAddr := t.linkAddr.Load()
+	if linkAddr == nil || t.closed {
+		return nil
+	}
+
+	// The kernel deletes a route by its prefix alone, whichever interface holds it. Only delete the ones that are ours
+	msgs, err := fetchRoutes()
+	if err != nil {
+		var left []netip.Prefix
+		for _, r := range installRoutes(routes) {
+			left = append(left, r.Cidr)
+		}
+		return fmt.Errorf("failed to read the system routing table, leaving routes %v installed: %w", left, err)
+	}
+
+	for _, r := range installRoutes(routes) {
+		p := r.Cidr.Masked()
+		// Whatever happens next, the prefix is no longer one we know about
+		delete(t.routeStates, p)
+		if routeInterfaceIn(msgs, p) != linkAddr.Index {
+			continue
+		}
+
+		err = delRoute(p, linkAddr)
+		if errors.Is(err, unix.ESRCH) {
+			// Already gone, which is what we wanted
+			t.l.Info("Route was already removed", "route", r)
+		} else if err != nil {
+			t.l.Error("Failed to remove route", "error", err, "route", r)
+			rl.Failed()
+		} else {
+			t.l.Info("Removed route", "route", r)
+		}
+	}
+	return nil
+}
+
+// reconcileRoutes brings the unsafe_routes in line with who holds their prefix now. It is quiet unless something
+// changed since it last looked, it runs every routeSweepEvery.
+func (t *tun) reconcileRoutes() {
+	t.routesMu.Lock()
+	defer t.routesMu.Unlock()
+
+	linkAddr := t.linkAddr.Load()
+	if linkAddr == nil || t.closed {
+		return
+	}
+
+	t.routeMoversMu.Lock()
+	movers := t.routeMovers
+	t.routeMovers = nil
+	t.routeMoversMu.Unlock()
+
+	msgs, err := fetchRoutes()
+	if err != nil {
+		t.l.Error("Failed to read the system routing table", "error", err)
+		return
+	}
+
+	states := make(map[netip.Prefix]routeState)
+	for _, r := range installRoutes(*t.Routes.Load()) {
+		p := r.Cidr.Masked()
+		prev := t.routeStates[p]
+		by := movers[p]
+
+		holder := routeInterfaceIn(msgs, p)
+		switch {
+		case holder == linkAddr.Index:
+			states[p] = routeState{kind: routeInstalled}
+
+		case holder != 0:
+			// Another interface taking a prefix we left alone puts ours back in line behind theirs
+			if prev.kind != routeYielded || prev.holder != holder {
+				t.l.Info("unsafe_route is routed through another interface, it will be added back when that route is removed", by.logArgs("route", r.Cidr, "interface", interfaceName(holder))...)
+			}
+			states[p] = routeState{kind: routeYielded, holder: holder}
+
+		case prev.kind == routeInstalled:
+			t.l.Warn("unsafe_route was removed outside nebula, leaving it until a reload or another interface takes its destination", by.logArgs("route", r.Cidr)...)
+			states[p] = routeState{kind: routeRemoved}
+
+		case prev.kind == routeRemoved:
+			states[p] = prev
+
+		default:
+			err := addRoute(p, linkAddr)
+			switch {
+			case err == nil:
+				if prev.kind == routeYielded {
+					t.l.Info("unsafe_route's destination is free again, added it", by.logArgs("route", r)...)
+				} else {
+					t.l.Info("Added route", "route", r)
+				}
+				states[p] = routeState{kind: routeInstalled}
+			case errors.Is(err, unix.EEXIST):
+				// Taken since the fetch, the next look will see by whom
+				states[p] = prev
+			default:
+				if prev.kind != routeFailed {
+					util.NewContextualError("Failed to add route", map[string]any{"route": r}, err).Log(t.l)
+				}
+				states[p] = routeState{kind: routeFailed}
+			}
+		}
+	}
+	t.routeStates = states
+}
+
+// startRouteWatch starts reconciling the unsafe_routes, every routeSweepEvery and, when watch isn't nil, once the
+// routing table settles after anything that may have moved a prefix
+func (t *tun) startRouteWatch(watch *os.File) {
+	t.routesMu.Lock()
+	defer t.routesMu.Unlock()
+	if t.closed {
+		if watch != nil {
+			_ = watch.Close()
+		}
+		return
+	}
+
+	t.routeSweep = time.AfterFunc(t.routeSweepEvery, t.sweepRoutes)
+	if watch != nil {
+		t.routeWatch = watch
+		go t.watchRoutes(watch)
+	}
+}
+
+func (t *tun) sweepRoutes() {
+	// A table still changing gets reconciled once it settles
+	if time.Since(time.Unix(0, t.lastRouteEvent.Load())) >= routeRetryDelay {
+		t.reconcileRoutes()
+	}
+
+	t.routesMu.Lock()
+	defer t.routesMu.Unlock()
+	if !t.closed {
+		t.routeSweep.Reset(t.routeSweepEvery)
+	}
+}
+
+func openRouteSocket() (*os.File, error) {
+	fd, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create AF_ROUTE socket: %w", err)
+	}
+	// Fewer drops, the sweep catches the rest
+	_ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, routeSocketRcvBuf)
+	// Non-blocking puts it on the poller, so Close interrupts a Read
+	if err = unix.SetNonblock(fd, true); err != nil {
+		_ = unix.Close(fd)
+		return nil, fmt.Errorf("failed to set the AF_ROUTE socket to non-blocking mode: %w", err)
+	}
+	return os.NewFile(uintptr(fd), "route"), nil
+}
+
+// watchRoutes reconciles the unsafe_routes once the routing table has been quiet for routeRetryDelay after a message
+// that may have moved a prefix. A table that keeps changing is reconciled every routeSweepEvery regardless.
+func (t *tun) watchRoutes(f *os.File) {
+	var reconciled atomic.Int64
+	settle := time.AfterFunc(time.Hour, func() {
+		reconciled.Store(time.Now().UnixNano())
+		t.reconcileRoutes()
+	})
+	settle.Stop()
+	defer settle.Stop()
+
+	var burst time.Time
+	self := int32(os.Getpid())
+	b := make([]byte, os.Getpagesize())
+	for {
+		n, err := f.Read(b)
+		if err != nil {
+			if !errors.Is(err, os.ErrClosed) {
+				t.l.Error("Stopped watching the system routing table, unsafe_routes held by another interface are only retried every minute", "error", err)
+			}
+			return
+		}
+
+		moved, p, pid := routesMayHaveMoved(b[:n], *t.Routes.Load())
+		if !moved {
+			continue
+		}
+		// Our own are already in routeStates
+		if p.IsValid() && pid != self {
+			t.routeMoversMu.Lock()
+			if t.routeMovers == nil {
+				t.routeMovers = make(map[netip.Prefix]routeMover)
+			}
+			t.routeMovers[p] = routeMover{pid: pid, name: processName(pid)}
+			t.routeMoversMu.Unlock()
+		}
+
+		now := time.Now()
+		t.lastRouteEvent.Store(now.UnixNano())
+		if burst.IsZero() || reconciled.Load() >= burst.UnixNano() {
+			burst = now
+		}
+		settle.Reset(max(0, min(routeRetryDelay, time.Until(burst.Add(t.routeSweepEvery)))))
+	}
+}
+
+// routesMayHaveMoved reports whether b, one routing socket message, may have freed a prefix one of routes installs or
+// handed it to another interface. That's a route for it added, changed or removed, or an interface that lost an address
+// or changed state, since detaching an interface deletes its routes without a message of their own. The header is read
+// directly, x/net/route skips the RTM_IFINFO the socket sends, which has no interface address, and misreads the
+// unpadded sockaddrs in messages the kernel makes itself. A request naming one of the prefixes also returns that prefix
+// and the pid of the process that made it.
+func routesMayHaveMoved(b []byte, routes []Route) (moved bool, p netip.Prefix, pid int32) {
+	if len(b) < 4 {
+		return false, p, 0
+	}
+	switch int(b[3]) {
+	case unix.RTM_IFINFO, unix.RTM_DELADDR:
+		return true, p, 0
+	case unix.RTM_ADD, unix.RTM_CHANGE, unix.RTM_DELETE:
+	default:
+		return false, p, 0
+	}
+
+	if len(b) < unix.SizeofRtMsghdr {
+		return false, p, 0
+	}
+	hdr := (*unix.RtMsghdr)(unsafe.Pointer(&b[0]))
+	// A failed request is reported too, and a scoped route never held the prefix
+	if hdr.Errno != 0 || hdr.Flags&unix.RTF_IFSCOPE != 0 {
+		return false, p, 0
+	}
+
+	if hdr.Pid == 0 {
+		// The kernel's own, which x/net/route can't parse. Neighbors coming and going are host routes, anything else
+		// could be holding a prefix
+		return hdr.Flags&(unix.RTF_HOST|unix.RTF_LLINFO) == 0, p, 0
+	}
+
+	// Replies to a request carry the requester's padded sockaddrs
+	msgs, err := netroute.ParseRIB(unix.NET_RT_DUMP, b)
+	if err != nil || len(msgs) != 1 {
+		return false, p, 0
+	}
+	m, ok := msgs[0].(*netroute.RouteMessage)
+	if !ok || len(m.Addrs) <= unix.RTAX_NETMASK {
+		return false, p, 0
+	}
+	p, ok = routePrefix(m.Addrs[unix.RTAX_DST], m.Addrs[unix.RTAX_NETMASK])
+	if !ok {
+		return false, netip.Prefix{}, 0
+	}
+	p = p.Masked()
+	for _, r := range installRoutes(routes) {
+		if r.Cidr.Masked() == p {
+			return true, p, hdr.Pid
+		}
+	}
+	return false, netip.Prefix{}, 0
+}
+
+// interfaceName is the name of the interface with index i, for logs
+func interfaceName(i int) string {
+	if ifc, err := net.InterfaceByIndex(i); err == nil {
+		return ifc.Name
+	}
+	return strconv.Itoa(i)
+}