Let the darwin tun adopt an fd the host already configured

Makefile

@@ -222,14 +222,11 @@ test-cov-html:
 	go test -coverprofile=coverage.out
 	go tool cover -html=coverage.out
 
-# The package builds only compile. The final line links an android binary so a linker-only failure,
-# such as the //go:linkname reference anet makes, cannot pass CI.
 build-test-mobile:
 	GOARCH=amd64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
 	GOARCH=arm64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
 	GOARCH=amd64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
 	GOARCH=arm64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
-	GOARCH=arm64 GOOS=android go build -ldflags=-checklinkname=0 -o /dev/null ${NEBULA_CMD_PATH}
 
 bench:
 	go test -bench=.

go.mod

@@ -22,7 +22,6 @@ require (
 	github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
 	github.com/stretchr/testify v1.11.1
 	github.com/vishvananda/netlink v1.3.1
-	github.com/wlynxg/anet v0.0.5
 	go.uber.org/goleak v1.3.0
 	go.yaml.in/yaml/v3 v3.0.4
 	golang.org/x/crypto v0.54.0

go.sum

@@ -149,8 +149,6 @@ github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW
 github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
 github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
 github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
-github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU=
-github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=

hostmap.go

@@ -868,28 +868,10 @@ func (i *HostInfo) logger(l *slog.Logger) *slog.Logger {
 
 // Utility functions
 
-func localAddrs(l *slog.Logger, allowList *LocalAllowList) ([]netip.Addr, error) {
-	return collectLocalAddrs(l, allowList, localInterfaces, localInterfaceAddrs)
-}
-
-// collectLocalAddrs takes its enumerators as arguments so tests can drive the filtering and the
-// failure branches without depending on the addresses of whatever host they run on. It reports
-// failures to the caller rather than logging them, because it runs on every lighthouse update and
-// only the caller can tell a new failure from a repeat of the same one.
-func collectLocalAddrs(
-	l *slog.Logger,
-	allowList *LocalAllowList,
-	interfaces func() ([]net.Interface, error),
-	interfaceAddrs func(*net.Interface) ([]net.Addr, error),
-) ([]netip.Addr, error) {
+func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
 	//FIXME: This function is pretty garbage
 	var finalAddrs []netip.Addr
-	var errs []error
-	ifaces, err := interfaces()
-	if err != nil {
-		return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err)
-	}
-
+	ifaces, _ := net.Interfaces()
 	for _, i := range ifaces {
 		allow := allowList.AllowName(i.Name)
 		if l.Enabled(context.Background(), logging.LevelTrace) {
@@ -902,12 +884,7 @@ func collectLocalAddrs(
 		if !allow {
 			continue
 		}
-		addrs, err := interfaceAddrs(&i)
-		if err != nil {
-			errs = append(errs, fmt.Errorf("failed to get addresses for %s: %w", i.Name, err))
-			continue
-		}
-
+		addrs, _ := i.Addrs()
 		for _, rawAddr := range addrs {
 			var addr netip.Addr
 			switch v := rawAddr.(type) {
@@ -942,5 +919,5 @@ func collectLocalAddrs(
 			}
 		}
 	}
-	return finalAddrs, errors.Join(errs...)
+	return finalAddrs
 }

hostmap_test.go

@@ -1,8 +1,6 @@
 package nebula
 
 import (
-	"errors"
-	"net"
 	"net/netip"
 	"slices"
 	"testing"
@@ -403,83 +401,3 @@ func TestHostMap_RelayState(t *testing.T) {
 	assert.Equal(t, []netip.Addr{}, h1.relayState.relays)
 
 }
-
-func TestCollectLocalAddrs(t *testing.T) {
-	ifaces := []net.Interface{
-		{Index: 1, Name: "lo"},
-		{Index: 2, Name: "eth0"},
-		{Index: 3, Name: "docker0"},
-	}
-	addrs := map[string][]net.Addr{
-		"lo": {
-			&net.IPNet{IP: net.ParseIP("127.0.0.1"), Mask: net.CIDRMask(8, 32)},
-			&net.IPNet{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)},
-		},
-		"eth0": {
-			&net.IPNet{IP: net.ParseIP("10.0.0.5"), Mask: net.CIDRMask(24, 32)},
-			&net.IPNet{IP: net.ParseIP("fe80::1"), Mask: net.CIDRMask(64, 128)},
-			&net.IPAddr{IP: net.ParseIP("fd00::5")},
-		},
-		"docker0": {
-			&net.IPNet{IP: net.ParseIP("172.17.0.1"), Mask: net.CIDRMask(16, 32)},
-		},
-	}
-
-	enumerate := func() ([]net.Interface, error) { return ifaces, nil }
-	addrsFor := func(i *net.Interface) ([]net.Addr, error) { return addrs[i.Name], nil }
-
-	// Loopback and link local are dropped, everything else on every interface is kept.
-	out, err := collectLocalAddrs(test.NewLogger(), nil, enumerate, addrsFor)
-	require.NoError(t, err)
-	assert.Equal(t, []netip.Addr{
-		netip.MustParseAddr("10.0.0.5"),
-		netip.MustParseAddr("fd00::5"),
-		netip.MustParseAddr("172.17.0.1"),
-	}, out)
-
-	// An interface the allow list rejects by name is never asked for its addresses.
-	c := config.NewC(test.NewLogger())
-	c.Settings["allowlist"] = map[string]any{
-		"interfaces": map[string]any{`docker.*`: false},
-	}
-	al, err := NewLocalAllowListFromConfig(c, "allowlist")
-	require.NoError(t, err)
-
-	asked := make(map[string]struct{})
-	countingAddrsFor := func(i *net.Interface) ([]net.Addr, error) {
-		asked[i.Name] = struct{}{}
-		return addrs[i.Name], nil
-	}
-	out, err = collectLocalAddrs(test.NewLogger(), al, enumerate, countingAddrsFor)
-	require.NoError(t, err)
-	assert.Equal(t, []netip.Addr{
-		netip.MustParseAddr("10.0.0.5"),
-		netip.MustParseAddr("fd00::5"),
-	}, out)
-	assert.NotContains(t, asked, "docker0")
-
-	// A failure to enumerate interfaces at all is reported rather than silently advertising nothing.
-	out, err = collectLocalAddrs(
-		test.NewLogger(),
-		nil,
-		func() ([]net.Interface, error) { return nil, errors.New("netlinkrib: permission denied") },
-		addrsFor,
-	)
-	assert.Nil(t, out)
-	require.EqualError(t, err, "failed to enumerate local interfaces: netlinkrib: permission denied")
-
-	// One interface failing is reported and skipped, the rest are still collected.
-	out, err = collectLocalAddrs(
-		test.NewLogger(),
-		nil,
-		enumerate,
-		func(i *net.Interface) ([]net.Addr, error) {
-			if i.Name == "eth0" {
-				return nil, errors.New("nope")
-			}
-			return addrs[i.Name], nil
-		},
-	)
-	assert.Equal(t, []netip.Addr{netip.MustParseAddr("172.17.0.1")}, out)
-	require.EqualError(t, err, "failed to get addresses for eth0: nope")
-}

lighthouse.go

@@ -40,10 +40,6 @@ type LightHouse struct {
 	// addresses rather than whatever this machine's NICs happen to be. Set it before Start.
 	localAddrsFn func(*LocalAllowList) []netip.Addr
 
-	// lastLocalAddrsErr is the previous localAddrsFn failure. Enumeration runs on every update, so an
-	// unchanged failure is demoted to Debug rather than warning every lighthouse.interval forever.
-	lastLocalAddrsErr atomic.Pointer[string]
-
 	// Local cache of answers from light houses
 	// map of vpn addr to answers
 	addrMap map[netip.Addr]*RemoteList
@@ -116,9 +112,7 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
 		l:                  l,
 	}
 	h.localAddrsFn = func(al *LocalAllowList) []netip.Addr {
-		addrs, err := localAddrs(h.l, al)
-		h.logLocalAddrsErr(err)
-		return addrs
+		return localAddrs(h.l, al)
 	}
 
 	lighthouses := make([]netip.Addr, 0)
@@ -919,26 +913,6 @@ func (lh *LightHouse) TriggerUpdate() {
 	}
 }
 
-// logLocalAddrsErr reports a localAddrs failure at Warn the first time it is seen and at Debug while
-// it persists unchanged, so a permanent failure does not warn on every update forever.
-func (lh *LightHouse) logLocalAddrsErr(err error) {
-	if err == nil {
-		lh.lastLocalAddrsErr.Store(nil)
-		return
-	}
-
-	msg := err.Error()
-	prev := lh.lastLocalAddrsErr.Swap(&msg)
-	if prev != nil && *prev == msg {
-		if lh.l.Enabled(context.Background(), slog.LevelDebug) {
-			lh.l.Debug("Failed to collect local addresses to advertise", "error", err)
-		}
-		return
-	}
-
-	lh.l.Warn("Failed to collect local addresses to advertise", "error", err)
-}
-
 func (lh *LightHouse) SendUpdate() {
 	var v4 []*V4AddrPort
 	var v6 []*V6AddrPort

lighthouse_test.go

@@ -1,9 +1,7 @@
 package nebula
 
 import (
-	"bytes"
 	"encoding/binary"
-	"errors"
 	"fmt"
 	"net/netip"
 	"testing"
@@ -740,32 +738,3 @@ func TestLighthouse_DeletesWork(t *testing.T) {
 	out = lh.Query(testHost)
 	assert.Nil(t, out)
 }
-
-func TestLightHouse_logLocalAddrsErr(t *testing.T) {
-	out := &bytes.Buffer{}
-	lh := &LightHouse{l: test.NewLoggerWithOutput(out)}
-
-	// The first sighting of a failure warns.
-	lh.logLocalAddrsErr(errors.New("permission denied"))
-	assert.Contains(t, out.String(), "level=WARN")
-	assert.Contains(t, out.String(), "permission denied")
-
-	// Repeating unchanged does not warn again, which is what keeps a permanent failure from warning
-	// on every lighthouse.interval for the life of the process.
-	out.Reset()
-	lh.logLocalAddrsErr(errors.New("permission denied"))
-	assert.NotContains(t, out.String(), "level=WARN")
-
-	// A different failure is a new event and warns.
-	out.Reset()
-	lh.logLocalAddrsErr(errors.New("something else"))
-	assert.Contains(t, out.String(), "level=WARN")
-	assert.Contains(t, out.String(), "something else")
-
-	// Recovering resets, so the same failure returning later warns again.
-	out.Reset()
-	lh.logLocalAddrsErr(nil)
-	assert.Empty(t, out.String())
-	lh.logLocalAddrsErr(errors.New("something else"))
-	assert.Contains(t, out.String(), "level=WARN")
-}

localaddrs.go

@@ -1,13 +0,0 @@
-//go:build !android
-
-package nebula
-
-import "net"
-
-func localInterfaces() ([]net.Interface, error) {
-	return net.Interfaces()
-}
-
-func localInterfaceAddrs(i *net.Interface) ([]net.Addr, error) {
-	return i.Addrs()
-}

localaddrs_android.go

@@ -1,32 +0,0 @@
-//go:build android
-
-package nebula
-
-import (
-	"net"
-
-	"github.com/wlynxg/anet"
-)
-
-// anet relies on //go:linkname and so needs -ldflags=-checklinkname=0 on Go 1.23+. Nebula ships no
-// Android binaries of its own, so that burden falls on consumers linking Android artifacts.
-
-func init() {
-	// anet only takes its bind-free path when it believes it is on API 30+, and detecting the running
-	// device's level requires cgo. Pin it so a CGO_ENABLED=0 build cannot quietly fall back to the
-	// denied path. The bind-free path is correct on older releases too, just unnecessary there.
-	anet.SetAndroidVersion(11)
-}
-
-// The app sandbox denies bind() on netlink_route_socket, so the stdlib's RTM_GETLINK enumeration
-// fails with EACCES and we advertise no underlay addresses at all. anet reads RTM_GETADDR from an
-// unbound socket instead, so this must not be collapsed back into net.Interfaces.
-func localInterfaces() ([]net.Interface, error) {
-	return anet.Interfaces()
-}
-
-// net.Interface.Addrs goes back through the denied netlink path, so addresses have to come from anet
-// as well. anet cannot report HardwareAddr, which localAddrs does not read.
-func localInterfaceAddrs(i *net.Interface) ([]net.Addr, error) {
-	return anet.InterfaceAddrsByInterface(i)
-}

main.go

@@ -22,9 +22,12 @@ type m = map[string]any
 
 func Main(c *config.C, configTest bool, buildVersion string, l *slog.Logger, deviceFactory overlay.DeviceFactory) (retcon *Control, reterr error) {
 	ctx, cancel := context.WithCancel(context.Background())
-	// Automatically cancel the context if Main returns an error, to signal all created goroutines to quit.
+	// The goroutines started below stop only when this context does, and only a caller holding the
+	// Control can arrange that. Cancel whenever we are not handing one back, which covers an error
+	// and a config test alike: a config test used to leave the lighthouse query worker, and a
+	// hostname resolver per dns named static host, running for the life of the process.
 	defer func() {
-		if reterr != nil {
+		if retcon == nil {
 			cancel()
 		}
 	}()

main_test.go

@@ -0,0 +1,82 @@
+package nebula
+
+import (
+	"fmt"
+	"net/netip"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/slackhq/nebula/cert"
+	cert_test "github.com/slackhq/nebula/cert_test"
+	"github.com/slackhq/nebula/config"
+	"github.com/slackhq/nebula/test"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
+)
+
+// TestMain_ConfigTestReleasesItsGoroutines pins the rule that Main only leaves goroutines running
+// when it hands back a Control to stop them with.
+//
+// A config test gets no Control, so anything it started had nothing to stop it: the lighthouse
+// query worker, and a hostname resolver per dns named static host, ran for the life of the
+// process. That matters to every embedder that validates a config in process rather than by
+// exec'ing, dnclient and the apple clients included, because they do it on each config load and
+// the leak accumulates.
+func TestMain_ConfigTestReleasesItsGoroutines(t *testing.T) {
+	defer goleak.VerifyNone(t, goleak.IgnoreCurrent())
+
+	l := test.NewLogger()
+	dir := t.TempDir()
+
+	before := time.Now().Add(-time.Hour)
+	after := time.Now().Add(time.Hour)
+	ca, _, caKey, caPEM := cert_test.NewTestCaCert(cert.Version2, cert.Curve_CURVE25519, before, after, nil, nil, nil)
+	networks := []netip.Prefix{netip.MustParsePrefix("10.0.0.1/24")}
+	_, _, keyPEM, certPEM := cert_test.NewTestCert(
+		cert.Version2, cert.Curve_CURVE25519, ca, caKey, "config-test", before, after, networks, nil, nil)
+
+	caPath := filepath.Join(dir, "ca.pem")
+	certPath := filepath.Join(dir, "cert.pem")
+	keyPath := filepath.Join(dir, "key.pem")
+	require.NoError(t, os.WriteFile(caPath, caPEM, 0o600))
+	require.NoError(t, os.WriteFile(certPath, certPEM, 0o600))
+	require.NoError(t, os.WriteFile(keyPath, keyPEM, 0o600))
+
+	// A static host by address, not by name: the query worker is the goroutine under test and a
+	// hostname would drag a real dns lookup into a unit test.
+	configBody := fmt.Sprintf(`
+pki:
+  ca: %s
+  cert: %s
+  key: %s
+static_host_map:
+  "10.0.0.2": ["192.0.2.1:4242"]
+lighthouse:
+  hosts:
+    - "10.0.0.2"
+listen:
+  host: 127.0.0.1
+  port: 0
+tun:
+  disabled: true
+firewall:
+  outbound:
+    - port: any
+      proto: any
+      host: any
+  inbound:
+    - port: any
+      proto: any
+      host: any
+`, caPath, certPath, keyPath)
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "config.yml"), []byte(configBody), 0o600))
+
+	c := config.NewC(l)
+	require.NoError(t, c.Load(dir))
+
+	ctrl, err := Main(c, true, "config-test", l, nil)
+	require.NoError(t, err)
+	require.Nil(t, ctrl, "a config test hands back nothing to stop, so it must stop itself")
+}

overlay/tun_darwin.go

@@ -30,7 +30,10 @@ type tun struct {
 	Routes      atomic.Pointer[[]Route]
 	routeTree   atomic.Pointer[bart.Table[routing.Gateways]]
 	linkAddr    *netroute.LinkAddr
-	l           *slog.Logger
+	// hostOwned means the fd arrived from the OS, which has already configured addressing, mtu
+	// and routes for it. NEPacketTunnelProvider on darwin does this.
+	hostOwned bool
+	l         *slog.Logger
 }
 
 type ifReq struct {
@@ -150,8 +153,48 @@ func (t *tun) deviceBytes() (o [16]byte) {
 	return
 }
 
-func newTunFromFd(_ *config.C, _ *slog.Logger, _ int, _ []netip.Prefix) (*tun, error) {
-	return nil, fmt.Errorf("newTunFromFd not supported in Darwin")
+// newTunFromFd adopts a utun the host already created and configured, which is how a darwin
+// network extension is handed its device. Everything about moving packets is shared with newTun,
+// only the setup differs: the host owns addressing and routing here.
+func newTunFromFd(c *config.C, l *slog.Logger, deviceFd int, vpnNetworks []netip.Prefix) (*tun, error) {
+	if err := unix.SetNonblock(deviceFd, true); err != nil {
+		// We own the fd from the moment it is handed to us
+		_ = unix.Close(deviceFd)
+		return nil, fmt.Errorf("failed to set the tun fd to non-blocking mode: %w", err)
+	}
+
+	file := os.NewFile(uintptr(deviceFd), "/dev/tun")
+	t := &tun{
+		f:           file,
+		Device:      utunNameFromFd(deviceFd),
+		vpnNetworks: vpnNetworks,
+		DefaultMTU:  c.GetInt("tun.mtu", DefaultMTU),
+		hostOwned:   true,
+		l:           l,
+	}
+
+	if err := t.reload(c, true); err != nil {
+		_ = file.Close()
+		return nil, err
+	}
+
+	c.RegisterReloadCallback(func(c *config.C) {
+		if err := t.reload(c, false); err != nil {
+			util.LogWithContextIfNeeded("failed to reload tun device", err, t.l)
+		}
+	})
+
+	return t, nil
+}
+
+// utunNameFromFd asks the socket what interface it is, for logs. A blank name is not worth
+// failing a tunnel over, so an error just leaves it empty.
+func utunNameFromFd(fd int) string {
+	name, err := unix.GetsockoptString(fd, unix.AF_SYS_CONTROL, _UTUN_OPT_IFNAME)
+	if err != nil {
+		return ""
+	}
+	return name
 }
 
 func (t *tun) Close() error {
@@ -162,6 +205,12 @@ func (t *tun) Close() error {
 }
 
 func (t *tun) Activate() error {
+	// The host handed us a configured device. Its addresses, mtu and routes come from the network
+	// settings it applied, and a sandboxed extension cannot change them anyway.
+	if t.hostOwned {
+		return nil
+	}
+
 	devName := t.deviceBytes()
 
 	s, err := unix.Socket(
@@ -375,6 +424,11 @@ func getLinkAddr(name string) (*netroute.LinkAddr, error) {
 }
 
 func (t *tun) addRoutes(logErrors bool) error {
+	// The route tree is still ours, the system routing table is not
+	if t.hostOwned {
+		return nil
+	}
+
 	routes := *t.Routes.Load()
 
 	for _, r := range routes {
@@ -404,6 +458,10 @@ func (t *tun) addRoutes(logErrors bool) error {
 }
 
 func (t *tun) removeRoutes(routes []Route) error {
+	if t.hostOwned {
+		return nil
+	}
+
 	for _, r := range routes {
 		if !r.Install {
 			continue