diff --git a/Makefile b/Makefile index b7697047..448e6e08 100644 --- a/Makefile +++ b/Makefile @@ -222,14 +222,14 @@ test-cov-html: go test -coverprofile=coverage.out go tool cover -html=coverage.out -# The package builds only compile. The final line links an android binary so a linker-only failure, -# such as the //go:linkname reference anet makes, cannot pass CI. +# The package builds only compile. The final line links an android binary so a linker-only failure +# cannot pass CI. build-test-mobile: GOARCH=amd64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=arm64 GOOS=ios go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=amd64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/') GOARCH=arm64 GOOS=android go build $(shell go list ./... | grep -v '/cmd/\|/examples/') - GOARCH=arm64 GOOS=android go build -ldflags=-checklinkname=0 -o /dev/null ${NEBULA_CMD_PATH} + GOARCH=arm64 GOOS=android go build -o /dev/null ${NEBULA_CMD_PATH} bench: go test -bench=. diff --git a/go.mod b/go.mod index aa754e1a..9748492d 100644 --- a/go.mod +++ b/go.mod @@ -22,7 +22,6 @@ require ( github.com/stefanberger/go-pkcs11uri v0.0.0-20230803200340-78284954bff6 github.com/stretchr/testify v1.11.1 github.com/vishvananda/netlink v1.3.1 - github.com/wlynxg/anet v0.0.5 go.uber.org/goleak v1.3.0 go.yaml.in/yaml/v3 v3.0.4 golang.org/x/crypto v0.54.0 diff --git a/go.sum b/go.sum index f9ef15a6..29d68429 100644 --- a/go.sum +++ b/go.sum @@ -149,8 +149,6 @@ github.com/vishvananda/netlink v1.3.1 h1:3AEMt62VKqz90r0tmNhog0r/PpWKmrEShJU0wJW github.com/vishvananda/netlink v1.3.1/go.mod h1:ARtKouGSTGchR8aMwmkzC0qiNPrrWO5JS/XMVl45+b4= github.com/vishvananda/netns v0.0.5 h1:DfiHV+j8bA32MFM7bfEunvT8IAqQ/NzSJHtcmW5zdEY= github.com/vishvananda/netns v0.0.5/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= -github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= -github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= diff --git a/hostmap.go b/hostmap.go index b82e3f14..fd8d31e7 100644 --- a/hostmap.go +++ b/hostmap.go @@ -868,19 +868,26 @@ func (i *HostInfo) logger(l *slog.Logger) *slog.Logger { // Utility functions -func localAddrs(l *slog.Logger, allowList *LocalAllowList) ([]netip.Addr, error) { - return collectLocalAddrs(l, allowList, localInterfaces, localInterfaceAddrs) +// localInterface is what localAddrs needs from a net.Interface, provided per platform by +// localInterfaces. Addrs is deferred so platforms where address lookup is a separate kernel round +// trip only pay for interfaces the allow list accepts. +type localInterface struct { + Name string + Addrs func() ([]net.Addr, error) } -// collectLocalAddrs takes its enumerators as arguments so tests can drive the filtering and the +func localAddrs(l *slog.Logger, allowList *LocalAllowList) ([]netip.Addr, error) { + return collectLocalAddrs(l, allowList, localInterfaces) +} + +// collectLocalAddrs takes its enumerator as an argument so tests can drive the filtering and the // failure branches without depending on the addresses of whatever host they run on. It reports // failures to the caller rather than logging them, because it runs on every lighthouse update and // only the caller can tell a new failure from a repeat of the same one. func collectLocalAddrs( l *slog.Logger, allowList *LocalAllowList, - interfaces func() ([]net.Interface, error), - interfaceAddrs func(*net.Interface) ([]net.Addr, error), + interfaces func() ([]localInterface, error), ) ([]netip.Addr, error) { //FIXME: This function is pretty garbage var finalAddrs []netip.Addr @@ -902,7 +909,7 @@ func collectLocalAddrs( if !allow { continue } - addrs, err := interfaceAddrs(&i) + addrs, err := i.Addrs() if err != nil { errs = append(errs, fmt.Errorf("failed to get addresses for %s: %w", i.Name, err)) continue diff --git a/hostmap_test.go b/hostmap_test.go index 9e9c8553..9236c063 100644 --- a/hostmap_test.go +++ b/hostmap_test.go @@ -425,11 +425,24 @@ func TestCollectLocalAddrs(t *testing.T) { }, } - enumerate := func() ([]net.Interface, error) { return ifaces, nil } - addrsFor := func(i *net.Interface) ([]net.Addr, error) { return addrs[i.Name], nil } + // enumerate wraps ifaces with the given address lookup. + enumerate := func(addrsFor func(name string) ([]net.Addr, error)) func() ([]localInterface, error) { + return func() ([]localInterface, error) { + out := make([]localInterface, 0, len(ifaces)) + for _, i := range ifaces { + name := i.Name + out = append(out, localInterface{ + Name: name, + Addrs: func() ([]net.Addr, error) { return addrsFor(name) }, + }) + } + return out, nil + } + } + addrsFor := func(name string) ([]net.Addr, error) { return addrs[name], nil } // Loopback and link local are dropped, everything else on every interface is kept. - out, err := collectLocalAddrs(test.NewLogger(), nil, enumerate, addrsFor) + out, err := collectLocalAddrs(test.NewLogger(), nil, enumerate(addrsFor)) require.NoError(t, err) assert.Equal(t, []netip.Addr{ netip.MustParseAddr("10.0.0.5"), @@ -446,11 +459,11 @@ func TestCollectLocalAddrs(t *testing.T) { require.NoError(t, err) asked := make(map[string]struct{}) - countingAddrsFor := func(i *net.Interface) ([]net.Addr, error) { - asked[i.Name] = struct{}{} - return addrs[i.Name], nil + countingAddrsFor := func(name string) ([]net.Addr, error) { + asked[name] = struct{}{} + return addrs[name], nil } - out, err = collectLocalAddrs(test.NewLogger(), al, enumerate, countingAddrsFor) + out, err = collectLocalAddrs(test.NewLogger(), al, enumerate(countingAddrsFor)) require.NoError(t, err) assert.Equal(t, []netip.Addr{ netip.MustParseAddr("10.0.0.5"), @@ -462,8 +475,7 @@ func TestCollectLocalAddrs(t *testing.T) { out, err = collectLocalAddrs( test.NewLogger(), nil, - func() ([]net.Interface, error) { return nil, errors.New("netlinkrib: permission denied") }, - addrsFor, + func() ([]localInterface, error) { return nil, errors.New("netlinkrib: permission denied") }, ) assert.Nil(t, out) require.EqualError(t, err, "failed to enumerate local interfaces: netlinkrib: permission denied") @@ -472,13 +484,12 @@ func TestCollectLocalAddrs(t *testing.T) { out, err = collectLocalAddrs( test.NewLogger(), nil, - enumerate, - func(i *net.Interface) ([]net.Addr, error) { - if i.Name == "eth0" { + enumerate(func(name string) ([]net.Addr, error) { + if name == "eth0" { return nil, errors.New("nope") } - return addrs[i.Name], nil - }, + return addrs[name], nil + }), ) assert.Equal(t, []netip.Addr{netip.MustParseAddr("172.17.0.1")}, out) require.EqualError(t, err, "failed to get addresses for eth0: nope") diff --git a/localaddrs.go b/localaddrs.go index 50ce83be..c00f9cc2 100644 --- a/localaddrs.go +++ b/localaddrs.go @@ -4,10 +4,15 @@ package nebula import "net" -func localInterfaces() ([]net.Interface, error) { - return net.Interfaces() -} +func localInterfaces() ([]localInterface, error) { + ifaces, err := net.Interfaces() + if err != nil { + return nil, err + } -func localInterfaceAddrs(i *net.Interface) ([]net.Addr, error) { - return i.Addrs() + out := make([]localInterface, len(ifaces)) + for n, i := range ifaces { + out[n] = localInterface{Name: i.Name, Addrs: i.Addrs} + } + return out, nil } diff --git a/localaddrs_android.go b/localaddrs_android.go index de43f981..a3b9b076 100644 --- a/localaddrs_android.go +++ b/localaddrs_android.go @@ -3,30 +3,211 @@ package nebula import ( + "errors" + "fmt" "net" + "os" + "slices" + "syscall" + "unsafe" - "github.com/wlynxg/anet" + "golang.org/x/sys/unix" ) -// anet relies on //go:linkname and so needs -ldflags=-checklinkname=0 on Go 1.23+. Nebula ships no -// Android binaries of its own, so that burden falls on consumers linking Android artifacts. - -func init() { - // anet only takes its bind-free path when it believes it is on API 30+, and detecting the running - // device's level requires cgo. Pin it so a CGO_ENABLED=0 build cannot quietly fall back to the - // denied path. The bind-free path is correct on older releases too, just unnecessary there. - anet.SetAndroidVersion(11) +// The Android app sandbox denies bind() on netlink route sockets and RTM_GETLINK, so +// net.Interfaces fails with EACCES. Enumerate with an RTM_GETADDR dump on an unbound socket instead. +func localInterfaces() ([]localInterface, error) { + return netlinkInterfaceAddrs() } -// The app sandbox denies bind() on netlink_route_socket, so the stdlib's RTM_GETLINK enumeration -// fails with EACCES and we advertise no underlay addresses at all. anet reads RTM_GETADDR from an -// unbound socket instead, so this must not be collapsed back into net.Interfaces. -func localInterfaces() ([]net.Interface, error) { - return anet.Interfaces() +// netlinkInterfaceAddrs returns every interface that has an address, with its addresses. It dumps +// RTM_GETADDR on a netlink socket that is never bound and names each index with SIOCGIFNAME, the +// same operations Bionic's getifaddrs uses for app UIDs, which the sandbox permits. +func netlinkInterfaceAddrs() ([]localInterface, error) { + dump, err := netlinkAddrDump() + if err != nil { + return nil, err + } + msgs, err := syscall.ParseNetlinkMessage(dump) + if err != nil { + return nil, err + } + + // Any socket will do for SIOCGIFNAME. + sfd, err := unix.Socket(unix.AF_INET, unix.SOCK_DGRAM|unix.SOCK_CLOEXEC, 0) + if err != nil { + return nil, os.NewSyscallError("socket", err) + } + defer unix.Close(sfd) + + type iface struct { + index uint32 + name string + addrs []net.Addr + } + var ifaces []*iface + byIndex := make(map[uint32]*iface) // nil marks an index that vanished before we could name it + for _, m := range msgs { + if m.Header.Type != unix.RTM_NEWADDR { + continue + } + if len(m.Data) < unix.SizeofIfAddrmsg { + return nil, fmt.Errorf("short RTM_NEWADDR message: %d bytes", len(m.Data)) + } + ifam := (*unix.IfAddrmsg)(unsafe.Pointer(&m.Data[0])) + + i, seen := byIndex[ifam.Index] + if !seen { + name, err := interfaceNameByIndex(sfd, ifam.Index) + if err != nil && !errors.Is(err, unix.ENODEV) { + return nil, fmt.Errorf("failed to resolve name of interface %d: %w", ifam.Index, err) + } + if err == nil { + i = &iface{index: ifam.Index, name: name} + ifaces = append(ifaces, i) + } + byIndex[ifam.Index] = i + } + if i == nil { + // The interface went away between the dump and the ioctl. + continue + } + + attrs, err := syscall.ParseNetlinkRouteAttr(&m) + if err != nil { + return nil, err + } + if a := newNetlinkAddr(ifam, attrs); a != nil { + i.addrs = append(i.addrs, a) + } + } + + // The dump is grouped by address family. Sort by index to match net.Interfaces. + slices.SortFunc(ifaces, func(a, b *iface) int { return int(a.index) - int(b.index) }) + out := make([]localInterface, 0, len(ifaces)) + for _, i := range ifaces { + addrs := i.addrs + out = append(out, localInterface{ + Name: i.name, + Addrs: func() ([]net.Addr, error) { return addrs, nil }, + }) + } + return out, nil } -// net.Interface.Addrs goes back through the denied netlink path, so addresses have to come from anet -// as well. anet cannot report HardwareAddr, which localAddrs does not read. -func localInterfaceAddrs(i *net.Interface) ([]net.Addr, error) { - return anet.InterfaceAddrsByInterface(i) +// netlinkAddrDump is syscall.NetlinkRIB(RTM_GETADDR, AF_UNSPEC) without the bind() call. The +// kernel autobinds the socket on the first send, which the sandbox permits. +func netlinkAddrDump() ([]byte, error) { + fd, err := unix.Socket(unix.AF_NETLINK, unix.SOCK_RAW|unix.SOCK_CLOEXEC, unix.NETLINK_ROUTE) + if err != nil { + return nil, os.NewSyscallError("socket", err) + } + defer unix.Close(fd) + + const seq = 1 + req := make([]byte, unix.SizeofNlMsghdr+unix.SizeofRtGenmsg) + h := (*unix.NlMsghdr)(unsafe.Pointer(&req[0])) + h.Len = uint32(len(req)) + h.Type = unix.RTM_GETADDR + h.Flags = unix.NLM_F_DUMP | unix.NLM_F_REQUEST + h.Seq = seq + req[unix.SizeofNlMsghdr] = unix.AF_UNSPEC // rtgenmsg.rtgen_family + + if err := unix.Sendto(fd, req, 0, &unix.SockaddrNetlink{Family: unix.AF_NETLINK}); err != nil { + return nil, os.NewSyscallError("sendto", err) + } + + // Read back the port id the kernel assigned on send so replies can be matched. + lsa, err := unix.Getsockname(fd) + if err != nil { + return nil, os.NewSyscallError("getsockname", err) + } + nl, ok := lsa.(*unix.SockaddrNetlink) + if !ok { + return nil, errors.New("netlink socket returned a non-netlink local address") + } + + var out []byte + buf := make([]byte, os.Getpagesize()) + for { + n, _, err := unix.Recvfrom(fd, buf, 0) + if err != nil { + return nil, os.NewSyscallError("recvfrom", err) + } + if n < unix.NLMSG_HDRLEN { + return nil, fmt.Errorf("short netlink read: %d bytes", n) + } + msgs, err := syscall.ParseNetlinkMessage(buf[:n]) + if err != nil { + return nil, err + } + for _, m := range msgs { + if m.Header.Seq != seq || m.Header.Pid != nl.Pid { + return nil, fmt.Errorf("netlink reply for another request: seq %d pid %d", m.Header.Seq, m.Header.Pid) + } + switch m.Header.Type { + case unix.NLMSG_DONE: + return out, nil + case unix.NLMSG_ERROR: + if len(m.Data) < 4 { + return nil, errors.New("short NLMSG_ERROR") + } + errno := *(*int32)(unsafe.Pointer(&m.Data[0])) + return nil, os.NewSyscallError("RTM_GETADDR", unix.Errno(-errno)) + } + } + out = append(out, buf[:n]...) + } +} + +// interfaceNameByIndex resolves an interface index with SIOCGIFNAME. +func interfaceNameByIndex(fd int, index uint32) (string, error) { + var ifr unix.Ifreq + ifr.SetUint32(index) // ifr_ifindex is in the ifr_ifru union + if err := unix.IoctlIfreq(fd, unix.SIOCGIFNAME, &ifr); err != nil { + return "", err + } + return ifr.Name(), nil +} + +// newNetlinkAddr builds the address for one RTM_NEWADDR message. On a point-to-point link +// IFA_LOCAL is our side and IFA_ADDRESS the peer, so IFA_LOCAL wins when both are present. +func newNetlinkAddr(ifam *unix.IfAddrmsg, attrs []syscall.NetlinkRouteAttr) net.Addr { + var pointToPoint bool + for _, a := range attrs { + if a.Attr.Type == unix.IFA_LOCAL { + pointToPoint = true + break + } + } + + for _, a := range attrs { + if pointToPoint && a.Attr.Type == unix.IFA_ADDRESS { + continue + } + switch ifam.Family { + case unix.AF_INET: + if a.Attr.Type != unix.IFA_LOCAL && a.Attr.Type != unix.IFA_ADDRESS { + continue + } + if len(a.Value) < net.IPv4len { + return nil + } + return &net.IPNet{ + IP: net.IPv4(a.Value[0], a.Value[1], a.Value[2], a.Value[3]), + Mask: net.CIDRMask(int(ifam.Prefixlen), 8*net.IPv4len), + } + case unix.AF_INET6: + if a.Attr.Type != unix.IFA_LOCAL && a.Attr.Type != unix.IFA_ADDRESS { + continue + } + if len(a.Value) < net.IPv6len { + return nil + } + ifa := &net.IPNet{IP: make(net.IP, net.IPv6len), Mask: net.CIDRMask(int(ifam.Prefixlen), 8*net.IPv6len)} + copy(ifa.IP, a.Value[:net.IPv6len]) + return ifa + } + } + return nil }