From fcc6008a7854d492fb669bfc8dad024a9e3587f6 Mon Sep 17 00:00:00 2001 From: JackDoan Date: Mon, 14 Sep 2026 13:19:18 -0500 Subject: [PATCH] Enumerate Android interfaces without anet anet's only reason to need -ldflags=-checklinkname=0 is a pair of //go:linkname pulls that refresh the IPv6 zone caches inside net and x/net after it enumerates interfaces. localAddrs never resolves a zone and drops link-local addresses anyway, so nebula was paying for that flag, and disabling the linker's check for the whole binary, for nothing it used. Replace the dependency with the part of anet that matters: an RTM_GETADDR dump on a netlink socket that is never bound, and SIOCGIFNAME to name each index. Both are permitted to Android apps, and it is the same path Bionic's getifaddrs takes for app UIDs. It is built only for android; every other platform, Linux included, keeps net.Interfaces. One dump yields every interface with its addresses, so reshape the seam to a single enumerator returning localInterface values with a deferred Addrs. Android hands back what it already has; every other platform still only asks for the addresses of interfaces the allow list accepts. Take the bind-free path unconditionally on android. The sepolicy is keyed to the app's target SDK rather than the OS release, detecting either needs cgo, and the path is valid on every release. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01B15ivF2aJUtLFdqJ4iz6zs --- Makefile | 6 +- go.mod | 1 - go.sum | 2 - hostmap.go | 19 ++-- hostmap_test.go | 39 +++++--- localaddrs.go | 15 ++- localaddrs_android.go | 217 ++++++++++++++++++++++++++++++++++++++---- 7 files changed, 250 insertions(+), 49 deletions(-) diff --git a/Makefile b/Makefile index b7697047..448e6e08 100644 --- a/Makefile +++ b/Makefile @@ -222,14 +222,14 @@ test-cov-html: go test -coverprofile=coverage.out go tool cover -html=coverage.out -# The package builds only compile. The final line links an android binary so a linker-only failure, -# such as the //go:linkname reference anet makes, cannot pass CI. +# The package builds only compile. The final line links an android binary so a linker-only failure +# cannot pass CI. build-test-mobile: GOARCH=amd64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=arm64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=amd64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=arm64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/') - GOARCH=arm64 GOOS=android go build -ldflags=-checklinkname=0 -o /dev/null ${NEBULA_CMD_PATH} + GOARCH=arm64 GOOS=android go build -o /dev/null ${NEBULA_CMD_PATH} bench: go test -bench=. diff --git a/go.mod b/go.mod index aa754e1a..9748492d 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,6 @@ require ( github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 github.com/stretchr/testify v1.11.1 github.com/vishvananda/netlink v1.3.1 - github.com/wlynxg/anet v0.0.5 go.uber.org/goleak v1.3.0 go.yaml.in/yaml/v3 v3.0.4 golang.org/x/crypto v0.54.0 diff --git a/go.sum b/go.sum index f9ef15a6..29d68429 100644 --- a/go.sum +++ b/go.sum @@ -149,8 +149,6 @@ github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4= github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= -github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= -github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= diff --git a/hostmap.go b/hostmap.go index b82e3f14..fd8d31e7 100644 --- a/hostmap.go +++ b/hostmap.go @@ -868,19 +868,26 @@ func (i *HostInfo) logger(l *slog.Logger) *slog.Logger { // Utility functions -func localAddrs(l *slog.Logger, allowList *LocalAllowList) ([]netip.Addr, error) { - return collectLocalAddrs(l, allowList, localInterfaces, localInterfaceAddrs) +// localInterface is what localAddrs needs from a net.Interface, provided per platform by +// localInterfaces. Addrs is deferred so platforms where address lookup is a separate kernel round +// trip only pay for interfaces the allow list accepts. +type localInterface struct { + Name string + Addrs func() ([]net.Addr, error) } -// collectLocalAddrs takes its enumerators as arguments so tests can drive the filtering and the +func localAddrs(l *slog.Logger, allowList *LocalAllowList) ([]netip.Addr, error) { + return collectLocalAddrs(l, allowList, localInterfaces) +} + +// collectLocalAddrs takes its enumerator as an argument so tests can drive the filtering and the // failure branches without depending on the addresses of whatever host they run on. It reports // failures to the caller rather than logging them, because it runs on every lighthouse update and // only the caller can tell a new failure from a repeat of the same one. func collectLocalAddrs( l *slog.Logger, allowList *LocalAllowList, - interfaces func() ([]net.Interface, error), - interfaceAddrs func(*net.Interface) ([]net.Addr, error), + interfaces func() ([]localInterface, error), ) ([]netip.Addr, error) { //FIXME: This function is pretty garbage var finalAddrs []netip.Addr @@ -902,7 +909,7 @@ func collectLocalAddrs( if !allow { continue } - addrs, err := interfaceAddrs(&i) + addrs, err := i.Addrs() if err != nil { errs = append(errs, fmt.Errorf("failed to get addresses for %s: %w", i.Name, err)) continue diff --git a/hostmap_test.go b/hostmap_test.go index 9e9c8553..9236c063 100644 --- a/hostmap_test.go +++ b/hostmap_test.go @@ -425,11 +425,24 @@ func TestCollectLocalAddrs(t *testing.T) { }, } - enumerate := func() ([]net.Interface, error) { return ifaces, nil } - addrsFor := func(i *net.Interface) ([]net.Addr, error) { return addrs[i.Name], nil } + // enumerate wraps ifaces with the given address lookup. + enumerate := func(addrsFor func(name string) ([]net.Addr, error)) func() ([]localInterface, error) { + return func() ([]localInterface, error) { + out := make([]localInterface, 0, len(ifaces)) + for _, i := range ifaces { + name := i.Name + out = append(out, localInterface{ + Name: name, + Addrs: func() ([]net.Addr, error) { return addrsFor(name) }, + }) + } + return out, nil + } + } + addrsFor := func(name string) ([]net.Addr, error) { return addrs[name], nil } // Loopback and link local are dropped, everything else on every interface is kept. - out, err := collectLocalAddrs(test.NewLogger(), nil, enumerate, addrsFor) + out, err := collectLocalAddrs(test.NewLogger(), nil, enumerate(addrsFor)) require.NoError(t, err) assert.Equal(t, []netip.Addr{ netip.MustParseAddr("10.0.0.5"), @@ -446,11 +459,11 @@ func TestCollectLocalAddrs(t *testing.T) { require.NoError(t, err) asked := make(map[string]struct{}) - countingAddrsFor := func(i *net.Interface) ([]net.Addr, error) { - asked[i.Name] = struct{}{} - return addrs[i.Name], nil + countingAddrsFor := func(name string) ([]net.Addr, error) { + asked[name] = struct{}{} + return addrs[name], nil } - out, err = collectLocalAddrs(test.NewLogger(), al, enumerate, countingAddrsFor) + out, err = collectLocalAddrs(test.NewLogger(), al, enumerate(countingAddrsFor)) require.NoError(t, err) assert.Equal(t, []netip.Addr{ netip.MustParseAddr("10.0.0.5"), @@ -462,8 +475,7 @@ func TestCollectLocalAddrs(t *testing.T) { out, err = collectLocalAddrs( test.NewLogger(), nil, - func() ([]net.Interface, error) { return nil, errors.New("netlinkrib: permission denied") }, - addrsFor, + func() ([]localInterface, error) { return nil, errors.New("netlinkrib: permission denied") }, ) assert.Nil(t, out) require.EqualError(t, err, "failed to enumerate local interfaces: netlinkrib: permission denied") @@ -472,13 +484,12 @@ func TestCollectLocalAddrs(t *testing.T) { out, err = collectLocalAddrs( test.NewLogger(), nil, - enumerate, - func(i *net.Interface) ([]net.Addr, error) { - if i.Name == "eth0" { + enumerate(func(name string) ([]net.Addr, error) { + if name == "eth0" { return nil, errors.New("nope") } - return addrs[i.Name], nil - }, + return addrs[name], nil + }), ) assert.Equal(t, []netip.Addr{netip.MustParseAddr("172.17.0.1")}, out) require.EqualError(t, err, "failed to get addresses for eth0: nope") diff --git a/localaddrs.go b/localaddrs.go index 50ce83be..c00f9cc2 100644 --- a/localaddrs.go +++ b/localaddrs.go @@ -4,10 +4,15 @@ package nebula import "net" -func localInterfaces() ([]net.Interface, error) { - return net.Interfaces() -} +func localInterfaces() ([]localInterface, error) { + ifaces, err := net.Interfaces() + if err != nil { + return nil, err + } -func localInterfaceAddrs(i *net.Interface) ([]net.Addr, error) { - return i.Addrs() + out := make([]localInterface, len(ifaces)) + for n, i := range ifaces { + out[n] = localInterface{Name: i.Name, Addrs: i.Addrs} + } + return out, nil } diff --git a/localaddrs_android.go b/localaddrs_android.go index de43f981..a3b9b076 100644 --- a/localaddrs_android.go +++ b/localaddrs_android.go @@ -3,30 +3,211 @@ package nebula import ( + "errors" + "fmt" "net" + "os" + "slices" + "syscall" + "unsafe" - "github.com/wlynxg/anet" + "golang.org/x/sys/unix" ) -// anet relies on //go:linkname and so needs -ldflags=-checklinkname=0 on Go 1.23+. Nebula ships no -// Android binaries of its own, so that burden falls on consumers linking Android artifacts. - -func init() { - // anet only takes its bind-free path when it believes it is on API 30+, and detecting the running - // device's level requires cgo. Pin it so a CGO_ENABLED=0 build cannot quietly fall back to the - // denied path. The bind-free path is correct on older releases too, just unnecessary there. - anet.SetAndroidVersion(11) +// The Android app sandbox denies bind() on netlink route sockets and RTM_GETLINK, so +// net.Interfaces fails with EACCES. Enumerate with an RTM_GETADDR dump on an unbound socket instead. +func localInterfaces() ([]localInterface, error) { + return netlinkInterfaceAddrs() } -// The app sandbox denies bind() on netlink_route_socket, so the stdlib's RTM_GETLINK enumeration -// fails with EACCES and we advertise no underlay addresses at all. anet reads RTM_GETADDR from an -// unbound socket instead, so this must not be collapsed back into net.Interfaces. -func localInterfaces() ([]net.Interface, error) { - return anet.Interfaces() +// netlinkInterfaceAddrs returns every interface that has an address, with its addresses. It dumps +// RTM_GETADDR on a netlink socket that is never bound and names each index with SIOCGIFNAME, the +// same operations Bionic's getifaddrs uses for app UIDs, which the sandbox permits. +func netlinkInterfaceAddrs() ([]localInterface, error) { + dump, err := netlinkAddrDump() + if err != nil { + return nil, err + } + msgs, err := syscall.ParseNetlinkMessage(dump) + if err != nil { + return nil, err + } + + // Any socket will do for SIOCGIFNAME. + sfd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0) + if err != nil { + return nil, os.NewSyscallError("socket", err) + } + defer unix.Close(sfd) + + type iface struct { + index uint32 + name string + addrs []net.Addr + } + var ifaces []*iface + byIndex := make(map[uint32]*iface) // nil marks an index that vanished before we could name it + for _, m := range msgs { + if m.Header.Type != unix.RTM_NEWADDR { + continue + } + if len(m.Data) < unix.SizeofIfAddrmsg { + return nil, fmt.Errorf("short RTM_NEWADDR message: %d bytes", len(m.Data)) + } + ifam := (*unix.IfAddrmsg)(unsafe.Pointer(&m.Data[0])) + + i, seen := byIndex[ifam.Index] + if !seen { + name, err := interfaceNameByIndex(sfd, ifam.Index) + if err != nil && !errors.Is(err, unix.ENODEV) { + return nil, fmt.Errorf("failed to resolve name of interface %d: %w", ifam.Index, err) + } + if err == nil { + i = &iface{index: ifam.Index, name: name} + ifaces = append(ifaces, i) + } + byIndex[ifam.Index] = i + } + if i == nil { + // The interface went away between the dump and the ioctl. + continue + } + + attrs, err := syscall.ParseNetlinkRouteAttr(&m) + if err != nil { + return nil, err + } + if a := newNetlinkAddr(ifam, attrs); a != nil { + i.addrs = append(i.addrs, a) + } + } + + // The dump is grouped by address family. Sort by index to match net.Interfaces. + slices.SortFunc(ifaces, func(a, b *iface) int { return int(a.index) - int(b.index) }) + out := make([]localInterface, 0, len(ifaces)) + for _, i := range ifaces { + addrs := i.addrs + out = append(out, localInterface{ + Name: i.name, + Addrs: func() ([]net.Addr, error) { return addrs, nil }, + }) + } + return out, nil } -// net.Interface.Addrs goes back through the denied netlink path, so addresses have to come from anet -// as well. anet cannot report HardwareAddr, which localAddrs does not read. -func localInterfaceAddrs(i *net.Interface) ([]net.Addr, error) { - return anet.InterfaceAddrsByInterface(i) +// netlinkAddrDump is syscall.NetlinkRIB(RTM_GETADDR, AF_UNSPEC) without the bind() call. The +// kernel autobinds the socket on the first send, which the sandbox permits. +func netlinkAddrDump() ([]byte, error) { + fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE) + if err != nil { + return nil, os.NewSyscallError("socket", err) + } + defer unix.Close(fd) + + const seq = 1 + req := make([]byte, unix.SizeofNlMsghdr+unix.SizeofRtGenmsg) + h := (*unix.NlMsghdr)(unsafe.Pointer(&req[0])) + h.Len = uint32(len(req)) + h.Type = unix.RTM_GETADDR + h.Flags = unix.NLM_F_DUMP | unix.NLM_F_REQUEST + h.Seq = seq + req[unix.SizeofNlMsghdr] = unix.AF_UNSPEC // rtgenmsg.rtgen_family + + if err := unix.Sendto(fd, req, 0, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}); err != nil { + return nil, os.NewSyscallError("sendto", err) + } + + // Read back the port id the kernel assigned on send so replies can be matched. + lsa, err := unix.Getsockname(fd) + if err != nil { + return nil, os.NewSyscallError("getsockname", err) + } + nl, ok := lsa.(*unix.SockaddrNetlink) + if !ok { + return nil, errors.New("netlink socket returned a non-netlink local address") + } + + var out []byte + buf := make([]byte, os.Getpagesize()) + for { + n, _, err := unix.Recvfrom(fd, buf, 0) + if err != nil { + return nil, os.NewSyscallError("recvfrom", err) + } + if n < unix.NLMSG_HDRLEN { + return nil, fmt.Errorf("short netlink read: %d bytes", n) + } + msgs, err := syscall.ParseNetlinkMessage(buf[:n]) + if err != nil { + return nil, err + } + for _, m := range msgs { + if m.Header.Seq != seq || m.Header.Pid != nl.Pid { + return nil, fmt.Errorf("netlink reply for another request: seq %d pid %d", m.Header.Seq, m.Header.Pid) + } + switch m.Header.Type { + case unix.NLMSG_DONE: + return out, nil + case unix.NLMSG_ERROR: + if len(m.Data) < 4 { + return nil, errors.New("short NLMSG_ERROR") + } + errno := *(*int32)(unsafe.Pointer(&m.Data[0])) + return nil, os.NewSyscallError("RTM_GETADDR", unix.Errno(-errno)) + } + } + out = append(out, buf[:n]...) + } +} + +// interfaceNameByIndex resolves an interface index with SIOCGIFNAME. +func interfaceNameByIndex(fd int, index uint32) (string, error) { + var ifr unix.Ifreq + ifr.SetUint32(index) // ifr_ifindex is in the ifr_ifru union + if err := unix.IoctlIfreq(fd, unix.SIOCGIFNAME, &ifr); err != nil { + return "", err + } + return ifr.Name(), nil +} + +// newNetlinkAddr builds the address for one RTM_NEWADDR message. On a point-to-point link +// IFA_LOCAL is our side and IFA_ADDRESS the peer, so IFA_LOCAL wins when both are present. +func newNetlinkAddr(ifam *unix.IfAddrmsg, attrs []syscall.NetlinkRouteAttr) net.Addr { + var pointToPoint bool + for _, a := range attrs { + if a.Attr.Type == unix.IFA_LOCAL { + pointToPoint = true + break + } + } + + for _, a := range attrs { + if pointToPoint && a.Attr.Type == unix.IFA_ADDRESS { + continue + } + switch ifam.Family { + case unix.AF_INET: + if a.Attr.Type != unix.IFA_LOCAL && a.Attr.Type != unix.IFA_ADDRESS { + continue + } + if len(a.Value) < net.IPv4len { + return nil + } + return &net.IPNet{ + IP: net.IPv4(a.Value[0], a.Value[1], a.Value[2], a.Value[3]), + Mask: net.CIDRMask(int(ifam.Prefixlen), 8*net.IPv4len), + } + case unix.AF_INET6: + if a.Attr.Type != unix.IFA_LOCAL && a.Attr.Type != unix.IFA_ADDRESS { + continue + } + if len(a.Value) < net.IPv6len { + return nil + } + ifa := &net.IPNet{IP: make(net.IP, net.IPv6len), Mask: net.CIDRMask(int(ifam.Prefixlen), 8*net.IPv6len)} + copy(ifa.IP, a.Value[:net.IPv6len]) + return ifa + } + } + return nil }