Do not warn about local address failures on every update

localAddrs runs inside every SendUpdate, which fires on lighthouse.interval
and again on every network change via RebindUDPServer. Logging the failure
there meant a persistent failure warned every interval for the life of the
process, once per failing interface.

collectLocalAddrs now returns its failures instead of logging them, which
keeps it stateless and lets the tests assert on errors rather than log
output. The lighthouse holds the previous error and warns only when it
changes, demoting repeats to Debug, matching how handshake_manager handles
repeated send failures.

Makefile

@@ -222,11 +222,14 @@ test-cov-html:
 	go test -coverprofile=coverage.out
 	go tool cover -html=coverage.out
 
+# The package builds only compile. The final line links an android binary so a linker-only failure,
+# such as the //go:linkname reference anet makes, cannot pass CI.
 build-test-mobile:
 	GOARCH=amd64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
 	GOARCH=arm64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
 	GOARCH=amd64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
 	GOARCH=arm64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/')
+	GOARCH=arm64 GOOS=android go build -ldflags=-checklinkname=0 -o /dev/null ${NEBULA_CMD_PATH}
 
 bench:
 	go test -bench=.

go.mod

@@ -22,8 +22,9 @@ require (
 	github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6
 	github.com/stretchr/testify v1.11.1
 	github.com/vishvananda/netlink v1.3.1
+	github.com/wlynxg/anet v0.0.5
 	go.uber.org/goleak v1.3.0
-	go.yaml.in/yaml/v3 v3.0.5
+	go.yaml.in/yaml/v3 v3.0.4
 	golang.org/x/crypto v0.54.0
 	golang.org/x/exp v0.0.0-20230725093048-515e97ebf090
 	golang.org/x/net v0.57.0

go.sum

@@ -149,14 +149,16 @@ github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW
 github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4=
 github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY=
 github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM=
+github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU=
+github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI=
 go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU=
-go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw=
-go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg=
+go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc=
+go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=

hostmap.go

@@ -868,10 +868,28 @@ func (i *HostInfo) logger(l *slog.Logger) *slog.Logger {
 
 // Utility functions
 
-func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
+func localAddrs(l *slog.Logger, allowList *LocalAllowList) ([]netip.Addr, error) {
+	return collectLocalAddrs(l, allowList, localInterfaces, localInterfaceAddrs)
+}
+
+// collectLocalAddrs takes its enumerators as arguments so tests can drive the filtering and the
+// failure branches without depending on the addresses of whatever host they run on. It reports
+// failures to the caller rather than logging them, because it runs on every lighthouse update and
+// only the caller can tell a new failure from a repeat of the same one.
+func collectLocalAddrs(
+	l *slog.Logger,
+	allowList *LocalAllowList,
+	interfaces func() ([]net.Interface, error),
+	interfaceAddrs func(*net.Interface) ([]net.Addr, error),
+) ([]netip.Addr, error) {
 	//FIXME: This function is pretty garbage
 	var finalAddrs []netip.Addr
-	ifaces, _ := net.Interfaces()
+	var errs []error
+	ifaces, err := interfaces()
+	if err != nil {
+		return nil, fmt.Errorf("failed to enumerate local interfaces: %w", err)
+	}
+
 	for _, i := range ifaces {
 		allow := allowList.AllowName(i.Name)
 		if l.Enabled(context.Background(), logging.LevelTrace) {
@@ -884,7 +902,12 @@ func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
 		if !allow {
 			continue
 		}
-		addrs, _ := i.Addrs()
+		addrs, err := interfaceAddrs(&i)
+		if err != nil {
+			errs = append(errs, fmt.Errorf("failed to get addresses for %s: %w", i.Name, err))
+			continue
+		}
+
 		for _, rawAddr := range addrs {
 			var addr netip.Addr
 			switch v := rawAddr.(type) {
@@ -919,5 +942,5 @@ func localAddrs(l *slog.Logger, allowList *LocalAllowList) []netip.Addr {
 			}
 		}
 	}
-	return finalAddrs
+	return finalAddrs, errors.Join(errs...)
 }

hostmap_test.go

@@ -1,6 +1,8 @@
 package nebula
 
 import (
+	"errors"
+	"net"
 	"net/netip"
 	"slices"
 	"testing"
@@ -401,3 +403,83 @@ func TestHostMap_RelayState(t *testing.T) {
 	assert.Equal(t, []netip.Addr{}, h1.relayState.relays)
 
 }
+
+func TestCollectLocalAddrs(t *testing.T) {
+	ifaces := []net.Interface{
+		{Index: 1, Name: "lo"},
+		{Index: 2, Name: "eth0"},
+		{Index: 3, Name: "docker0"},
+	}
+	addrs := map[string][]net.Addr{
+		"lo": {
+			&net.IPNet{IP: net.ParseIP("127.0.0.1"), Mask: net.CIDRMask(8, 32)},
+			&net.IPNet{IP: net.ParseIP("::1"), Mask: net.CIDRMask(128, 128)},
+		},
+		"eth0": {
+			&net.IPNet{IP: net.ParseIP("10.0.0.5"), Mask: net.CIDRMask(24, 32)},
+			&net.IPNet{IP: net.ParseIP("fe80::1"), Mask: net.CIDRMask(64, 128)},
+			&net.IPAddr{IP: net.ParseIP("fd00::5")},
+		},
+		"docker0": {
+			&net.IPNet{IP: net.ParseIP("172.17.0.1"), Mask: net.CIDRMask(16, 32)},
+		},
+	}
+
+	enumerate := func() ([]net.Interface, error) { return ifaces, nil }
+	addrsFor := func(i *net.Interface) ([]net.Addr, error) { return addrs[i.Name], nil }
+
+	// Loopback and link local are dropped, everything else on every interface is kept.
+	out, err := collectLocalAddrs(test.NewLogger(), nil, enumerate, addrsFor)
+	require.NoError(t, err)
+	assert.Equal(t, []netip.Addr{
+		netip.MustParseAddr("10.0.0.5"),
+		netip.MustParseAddr("fd00::5"),
+		netip.MustParseAddr("172.17.0.1"),
+	}, out)
+
+	// An interface the allow list rejects by name is never asked for its addresses.
+	c := config.NewC(test.NewLogger())
+	c.Settings["allowlist"] = map[string]any{
+		"interfaces": map[string]any{`docker.*`: false},
+	}
+	al, err := NewLocalAllowListFromConfig(c, "allowlist")
+	require.NoError(t, err)
+
+	asked := make(map[string]struct{})
+	countingAddrsFor := func(i *net.Interface) ([]net.Addr, error) {
+		asked[i.Name] = struct{}{}
+		return addrs[i.Name], nil
+	}
+	out, err = collectLocalAddrs(test.NewLogger(), al, enumerate, countingAddrsFor)
+	require.NoError(t, err)
+	assert.Equal(t, []netip.Addr{
+		netip.MustParseAddr("10.0.0.5"),
+		netip.MustParseAddr("fd00::5"),
+	}, out)
+	assert.NotContains(t, asked, "docker0")
+
+	// A failure to enumerate interfaces at all is reported rather than silently advertising nothing.
+	out, err = collectLocalAddrs(
+		test.NewLogger(),
+		nil,
+		func() ([]net.Interface, error) { return nil, errors.New("netlinkrib: permission denied") },
+		addrsFor,
+	)
+	assert.Nil(t, out)
+	require.EqualError(t, err, "failed to enumerate local interfaces: netlinkrib: permission denied")
+
+	// One interface failing is reported and skipped, the rest are still collected.
+	out, err = collectLocalAddrs(
+		test.NewLogger(),
+		nil,
+		enumerate,
+		func(i *net.Interface) ([]net.Addr, error) {
+			if i.Name == "eth0" {
+				return nil, errors.New("nope")
+			}
+			return addrs[i.Name], nil
+		},
+	)
+	assert.Equal(t, []netip.Addr{netip.MustParseAddr("172.17.0.1")}, out)
+	require.EqualError(t, err, "failed to get addresses for eth0: nope")
+}

lighthouse.go

@@ -40,6 +40,10 @@ type LightHouse struct {
 	// addresses rather than whatever this machine's NICs happen to be. Set it before Start.
 	localAddrsFn func(*LocalAllowList) []netip.Addr
 
+	// lastLocalAddrsErr is the previous localAddrsFn failure. Enumeration runs on every update, so an
+	// unchanged failure is demoted to Debug rather than warning every lighthouse.interval forever.
+	lastLocalAddrsErr atomic.Pointer[string]
+
 	// Local cache of answers from light houses
 	// map of vpn addr to answers
 	addrMap map[netip.Addr]*RemoteList
@@ -112,7 +116,9 @@ func NewLightHouseFromConfig(ctx context.Context, l *slog.Logger, c *config.C, c
 		l:                  l,
 	}
 	h.localAddrsFn = func(al *LocalAllowList) []netip.Addr {
-		return localAddrs(h.l, al)
+		addrs, err := localAddrs(h.l, al)
+		h.logLocalAddrsErr(err)
+		return addrs
 	}
 
 	lighthouses := make([]netip.Addr, 0)
@@ -913,6 +919,26 @@ func (lh *LightHouse) TriggerUpdate() {
 	}
 }
 
+// logLocalAddrsErr reports a localAddrs failure at Warn the first time it is seen and at Debug while
+// it persists unchanged, so a permanent failure does not warn on every update forever.
+func (lh *LightHouse) logLocalAddrsErr(err error) {
+	if err == nil {
+		lh.lastLocalAddrsErr.Store(nil)
+		return
+	}
+
+	msg := err.Error()
+	prev := lh.lastLocalAddrsErr.Swap(&msg)
+	if prev != nil && *prev == msg {
+		if lh.l.Enabled(context.Background(), slog.LevelDebug) {
+			lh.l.Debug("Failed to collect local addresses to advertise", "error", err)
+		}
+		return
+	}
+
+	lh.l.Warn("Failed to collect local addresses to advertise", "error", err)
+}
+
 func (lh *LightHouse) SendUpdate() {
 	var v4 []*V4AddrPort
 	var v6 []*V6AddrPort

lighthouse_test.go

@@ -1,7 +1,9 @@
 package nebula
 
 import (
+	"bytes"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"net/netip"
 	"testing"
@@ -738,3 +740,32 @@ func TestLighthouse_DeletesWork(t *testing.T) {
 	out = lh.Query(testHost)
 	assert.Nil(t, out)
 }
+
+func TestLightHouse_logLocalAddrsErr(t *testing.T) {
+	out := &bytes.Buffer{}
+	lh := &LightHouse{l: test.NewLoggerWithOutput(out)}
+
+	// The first sighting of a failure warns.
+	lh.logLocalAddrsErr(errors.New("permission denied"))
+	assert.Contains(t, out.String(), "level=WARN")
+	assert.Contains(t, out.String(), "permission denied")
+
+	// Repeating unchanged does not warn again, which is what keeps a permanent failure from warning
+	// on every lighthouse.interval for the life of the process.
+	out.Reset()
+	lh.logLocalAddrsErr(errors.New("permission denied"))
+	assert.NotContains(t, out.String(), "level=WARN")
+
+	// A different failure is a new event and warns.
+	out.Reset()
+	lh.logLocalAddrsErr(errors.New("something else"))
+	assert.Contains(t, out.String(), "level=WARN")
+	assert.Contains(t, out.String(), "something else")
+
+	// Recovering resets, so the same failure returning later warns again.
+	out.Reset()
+	lh.logLocalAddrsErr(nil)
+	assert.Empty(t, out.String())
+	lh.logLocalAddrsErr(errors.New("something else"))
+	assert.Contains(t, out.String(), "level=WARN")
+}

localaddrs.go

@@ -0,0 +1,13 @@
+//go:build !android
+
+package nebula
+
+import "net"
+
+func localInterfaces() ([]net.Interface, error) {
+	return net.Interfaces()
+}
+
+func localInterfaceAddrs(i *net.Interface) ([]net.Addr, error) {
+	return i.Addrs()
+}

localaddrs_android.go

@@ -0,0 +1,32 @@
+//go:build android
+
+package nebula
+
+import (
+	"net"
+
+	"github.com/wlynxg/anet"
+)
+
+// anet relies on //go:linkname and so needs -ldflags=-checklinkname=0 on Go 1.23+. Nebula ships no
+// Android binaries of its own, so that burden falls on consumers linking Android artifacts.
+
+func init() {
+	// anet only takes its bind-free path when it believes it is on API 30+, and detecting the running
+	// device's level requires cgo. Pin it so a CGO_ENABLED=0 build cannot quietly fall back to the
+	// denied path. The bind-free path is correct on older releases too, just unnecessary there.
+	anet.SetAndroidVersion(11)
+}
+
+// The app sandbox denies bind() on netlink_route_socket, so the stdlib's RTM_GETLINK enumeration
+// fails with EACCES and we advertise no underlay addresses at all. anet reads RTM_GETADDR from an
+// unbound socket instead, so this must not be collapsed back into net.Interfaces.
+func localInterfaces() ([]net.Interface, error) {
+	return anet.Interfaces()
+}
+
+// net.Interface.Addrs goes back through the denied netlink path, so addresses have to come from anet
+// as well. anet cannot report HardwareAddr, which localAddrs does not read.
+func localInterfaceAddrs(i *net.Interface) ([]net.Addr, error) {
+	return anet.InterfaceAddrsByInterface(i)
+}