From fd29ebdacd058bf5d7494f238ad3fec26bd60486 Mon Sep 17 00:00:00 2001 From: JackDoan Date: Fri, 25 Sep 2026 13:01:50 -0500 Subject: [PATCH] use a trampoline to get to the spicy _x functions --- internal/msgx/doc.go | 5 ++ internal/msgx/hdr_darwin.go | 16 +++++++ internal/msgx/msgx_darwin.go | 77 +++++++++++++++++++++++++++++++ internal/msgx/msgx_darwin.s | 9 ++++ internal/msgx/msgx_darwin_test.go | 69 +++++++++++++++++++++++++++ internal/msgx/msgx_ios.go | 12 +++++ overlay/tio/tio_utun_darwin.go | 34 +++++--------- udp/udp_darwin.go | 41 ++++++---------- udp/udp_recvmsgx_darwin_test.go | 26 +++++------ 9 files changed, 227 insertions(+), 62 deletions(-) create mode 100644 internal/msgx/doc.go create mode 100644 internal/msgx/hdr_darwin.go create mode 100644 internal/msgx/msgx_darwin.go create mode 100644 internal/msgx/msgx_darwin.s create mode 100644 internal/msgx/msgx_darwin_test.go create mode 100644 internal/msgx/msgx_ios.go diff --git a/internal/msgx/doc.go b/internal/msgx/doc.go new file mode 100644 index 00000000..735cd901 --- /dev/null +++ b/internal/msgx/doc.go @@ -0,0 +1,5 @@ +// Package msgx calls xnu's private batched socket calls, sendmsg_x and recvmsg_x, through libSystem. +// +// The calls are not public API, so the package never links them: it looks them up with dlsym on first use. +// The package is darwin-only, and on iOS Available is always false, so App Store builds keep to public API. +package msgx diff --git a/internal/msgx/hdr_darwin.go b/internal/msgx/hdr_darwin.go new file mode 100644 index 00000000..104897cc --- /dev/null +++ b/internal/msgx/hdr_darwin.go @@ -0,0 +1,16 @@ +package msgx + +import "golang.org/x/sys/unix" + +// Hdr mirrors xnu's struct msghdr_x (bsd/sys/socket_private.h), which the SDK doesn't ship. +// recvmsg_x reports each message's length in Datalen rather than in its return value. +type Hdr struct { + Name *byte + Namelen uint32 + Iov *unix.Iovec + Iovlen int32 + Control *byte + Controllen uint32 + Flags int32 + Datalen uint64 +} diff --git a/internal/msgx/msgx_darwin.go b/internal/msgx/msgx_darwin.go new file mode 100644 index 00000000..4b788183 --- /dev/null +++ b/internal/msgx/msgx_darwin.go @@ -0,0 +1,77 @@ +//go:build !ios + +package msgx + +import ( + "sync" + "syscall" + "unsafe" +) + +// dlsym is public libSystem API, imported the way x/sys/unix imports its libSystem stubs. The local name is +// distinct from runtime/race's dlsym import. +// +//go:cgo_import_dynamic msgx_dlsym dlsym "/usr/lib/libSystem.B.dylib" + +// dlsymTrampolineAddr is the address of msgx_darwin.s's jump to dlsym. +var dlsymTrampolineAddr uintptr + +// syscall6X calls the C function at fn on the system stack and returns errno when it returns a 64-bit -1. +// It is how x/sys/unix calls libSystem, and the runtime keeps it linkable for that. +// +//go:linkname syscall6X syscall.syscall6X +func syscall6X(fn, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err syscall.Errno) + +// rtldDefault is dlfcn.h's RTLD_DEFAULT, ((void *)-2): search every image loaded in the process. +const rtldDefault = ^uintptr(1) + +var ( + sendmsgXName = []byte("sendmsg_x\x00") + recvmsgXName = []byte("recvmsg_x\x00") +) + +type funcs struct { + sendmsgX uintptr + recvmsgX uintptr +} + +var resolve = sync.OnceValue(func() funcs { + return funcs{sendmsgX: lookup(sendmsgXName), recvmsgX: lookup(recvmsgXName)} +}) + +// lookup returns the address of the NUL-terminated C symbol name, or 0 when no loaded image exports it. +func lookup(name []byte) uintptr { + p, _, _ := syscall6X(dlsymTrampolineAddr, rtldDefault, uintptr(unsafe.Pointer(&name[0])), 0, 0, 0, 0) + return p +} + +// Available reports whether libSystem exports both sendmsg_x and recvmsg_x. When it is false, Send and Recv +// return ENOSYS. +func Available() bool { + f := resolve() + return f.sendmsgX != 0 && f.recvmsgX != 0 +} + +// Send passes hdrs to sendmsg_x on fd and returns how many messages the kernel took. +func Send(fd uintptr, hdrs []Hdr, flags int) (int, syscall.Errno) { + return call(resolve().sendmsgX, fd, hdrs, flags) +} + +// Recv passes hdrs to recvmsg_x on fd and returns how many messages it filled. +func Recv(fd uintptr, hdrs []Hdr, flags int) (int, syscall.Errno) { + return call(resolve().recvmsgX, fd, hdrs, flags) +} + +func call(fn, fd uintptr, hdrs []Hdr, flags int) (int, syscall.Errno) { + if fn == 0 { + return 0, syscall.ENOSYS + } + if len(hdrs) == 0 { + return 0, syscall.EINVAL + } + r, _, errno := syscall6X(fn, fd, uintptr(unsafe.Pointer(&hdrs[0])), uintptr(len(hdrs)), uintptr(flags), 0, 0) + if errno != 0 { + return 0, errno + } + return int(r), 0 +} diff --git a/internal/msgx/msgx_darwin.s b/internal/msgx/msgx_darwin.s new file mode 100644 index 00000000..869a3a6a --- /dev/null +++ b/internal/msgx/msgx_darwin.s @@ -0,0 +1,9 @@ +//go:build !ios + +#include "textflag.h" + +// The same trampoline shape x/sys/unix uses for its libSystem stubs; it assembles for amd64 and arm64 alike. +TEXT msgx_dlsym_trampoline<>(SB),NOSPLIT,$0-0 + JMP msgx_dlsym(SB) +GLOBL ·dlsymTrampolineAddr(SB), RODATA, $8 +DATA ·dlsymTrampolineAddr(SB)/8, $msgx_dlsym_trampoline<>(SB) diff --git a/internal/msgx/msgx_darwin_test.go b/internal/msgx/msgx_darwin_test.go new file mode 100644 index 00000000..3441c53c --- /dev/null +++ b/internal/msgx/msgx_darwin_test.go @@ -0,0 +1,69 @@ +//go:build !ios + +package msgx + +import ( + "bytes" + "syscall" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +// TestAvailable pins that dlsym finds both calls in this macOS's libSystem, so the batched paths are live. +func TestAvailable(t *testing.T) { + assert.True(t, Available()) +} + +// TestSendRecvRoundTrip pins the libSystem calling path end to end: a batch goes out through sendmsg_x in one +// call and comes back through recvmsg_x in one call, each message whole, in order, with Datalen set. +func TestSendRecvRoundTrip(t *testing.T) { + fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_DGRAM, 0) + require.NoError(t, err) + t.Cleanup(func() { + _ = unix.Close(fds[0]) + _ = unix.Close(fds[1]) + }) + // A dropped datagram fails the receive instead of hanging it. + require.NoError(t, unix.SetsockoptTimeval(fds[1], unix.SOL_SOCKET, unix.SO_RCVTIMEO, &unix.Timeval{Sec: 2})) + + const n = 5 + var want [n][]byte + var iovs [n]unix.Iovec + hdrs := make([]Hdr, n) + for i := range n { + want[i] = bytes.Repeat([]byte{byte(i + 1)}, 10+i) + iovs[i] = unix.Iovec{Base: &want[i][0]} + iovs[i].SetLen(len(want[i])) + hdrs[i] = Hdr{Iov: &iovs[i], Iovlen: 1} + } + sent, errno := Send(uintptr(fds[0]), hdrs, 0) + require.Zero(t, errno) + require.Equal(t, n, sent) + + var bufs [n + 1][64]byte + var riovs [n + 1]unix.Iovec + rhdrs := make([]Hdr, n+1) + for i := range rhdrs { + riovs[i] = unix.Iovec{Base: &bufs[i][0]} + riovs[i].SetLen(len(bufs[i])) + rhdrs[i] = Hdr{Iov: &riovs[i], Iovlen: 1} + } + got, errno := Recv(uintptr(fds[1]), rhdrs, 0) + require.Zero(t, errno) + require.Equal(t, n, got) + for i := range n { + l := int(rhdrs[i].Datalen) + assert.Equal(t, want[i], bufs[i][:l], "message %d", i) + } +} + +// TestEmptyBatch pins that an empty batch is refused before it reaches the kernel. +func TestEmptyBatch(t *testing.T) { + _, errno := Send(0, nil, 0) + assert.Equal(t, syscall.EINVAL, errno) + _, errno = Recv(0, nil, 0) + assert.Equal(t, syscall.EINVAL, errno) +} diff --git a/internal/msgx/msgx_ios.go b/internal/msgx/msgx_ios.go new file mode 100644 index 00000000..58238dad --- /dev/null +++ b/internal/msgx/msgx_ios.go @@ -0,0 +1,12 @@ +package msgx + +import "syscall" + +// Available is always false on iOS, so App Store builds never look up private symbols. +func Available() bool { return false } + +// Send always returns ENOSYS on iOS. +func Send(uintptr, []Hdr, int) (int, syscall.Errno) { return 0, syscall.ENOSYS } + +// Recv always returns ENOSYS on iOS. +func Recv(uintptr, []Hdr, int) (int, syscall.Errno) { return 0, syscall.ENOSYS } diff --git a/overlay/tio/tio_utun_darwin.go b/overlay/tio/tio_utun_darwin.go index d3fe72dd..46c9e5d2 100644 --- a/overlay/tio/tio_utun_darwin.go +++ b/overlay/tio/tio_utun_darwin.go @@ -9,8 +9,9 @@ import ( "sync" "sync/atomic" "syscall" - "unsafe" + _ "unsafe" // for go:linkname + "github.com/slackhq/nebula/internal/msgx" "golang.org/x/sys/unix" ) @@ -40,7 +41,7 @@ type Utun struct { readBuf []byte readRet [tunReadBatch]Packet - // noSendmsgX is set once sendmsg_x is refused; WriteBatch then writes one packet at a time. + // noSendmsgX is set when libSystem lacks sendmsg_x or the kernel refuses it; WriteBatch then writes one packet at a time. noSendmsgX atomic.Bool batchMu sync.Mutex batch tunBatch @@ -50,23 +51,10 @@ type Utun struct { type tunBatch struct { heads [tunWriteBatch][4]byte iovs [tunWriteBatch][2]unix.Iovec - hdrs [tunWriteBatch]msghdrX + hdrs [tunWriteBatch]msgx.Hdr pkts [tunWriteBatch][]byte } -// msghdrX mirrors xnu's struct msghdr_x (bsd/sys/socket_private.h). -// Keep in sync with msghdrX in udp/udp_darwin.go. -type msghdrX struct { - Name *byte - Namelen uint32 - Iov *unix.Iovec - Iovlen int32 - Control *byte - Controllen uint32 - Flags int32 - Datalen uint64 -} - // NewUtun wraps a utun control socket fd and makes it non-blocking. // On failure it does NOT close fd: the caller owns fd until NewUtun succeeds, after which Close closes it. func NewUtun(fd int, l *slog.Logger) (*Utun, error) { @@ -74,11 +62,13 @@ func NewUtun(fd int, l *slog.Logger) (*Utun, error) { if err := unix.SetNonblock(fd, true); err != nil { return nil, fmt.Errorf("failed to set the tun fd to non-blocking mode: %w", err) } - return &Utun{ + u := &Utun{ f: os.NewFile(uintptr(fd), "utun"), l: l, readBuf: make([]byte, tunReadArena), - }, nil + } + u.noSendmsgX.Store(!msgx.Available()) + return u, nil } // tunWritev and tunReadv are linkname'd to x/sys/unix's libc-routed writev/readv stubs so the @@ -246,7 +236,7 @@ func (u *Utun) WriteBatch(pkts [][]byte) error { {Base: &b.heads[n][0], Len: 4}, {Base: &p[0], Len: uint64(len(p))}, } - b.hdrs[n] = msghdrX{Iov: &b.iovs[n][0], Iovlen: 2} + b.hdrs[n] = msgx.Hdr{Iov: &b.iovs[n][0], Iovlen: 2} b.pkts[n] = p n++ } @@ -306,10 +296,10 @@ func (u *Utun) sendmsgX(off, n int) (int, error) { if err != nil { return 0, err } - var sent uintptr + var sent int var errno syscall.Errno err = rc.Write(func(fd uintptr) bool { - sent, _, errno = unix.Syscall6(unix.SYS_SENDMSG_X, fd, uintptr(unsafe.Pointer(&u.batch.hdrs[off])), uintptr(n-off), 0, 0, 0) + sent, errno = msgx.Send(fd, u.batch.hdrs[off:n], 0) // sendmsg_x reports EAGAIN only when it took nothing; a partial batch returns its count. return !errno.Temporary() }) @@ -319,7 +309,7 @@ func (u *Utun) sendmsgX(off, n int) (int, error) { if errno != 0 { return 0, errno } - return int(sent), nil + return sent, nil } // writeEach writes pkts one at a time with Write. diff --git a/udp/udp_darwin.go b/udp/udp_darwin.go index d40f18c4..47180268 100644 --- a/udp/udp_darwin.go +++ b/udp/udp_darwin.go @@ -15,6 +15,7 @@ import ( "unsafe" "github.com/slackhq/nebula/config" + "github.com/slackhq/nebula/internal/msgx" "golang.org/x/sys/unix" ) @@ -39,6 +40,7 @@ func NewListener(l *slog.Logger, s Settings) (Conn, error) { if uc, ok := pc.(*net.UDPConn); ok { c := &StdConn{UDPConn: uc, l: l} + c.noSendmsgX.Store(!msgx.Available()) rc, err := uc.SyscallConn() if err != nil { @@ -150,7 +152,7 @@ const sendBatch = 64 type sendScratch struct { names [sendBatch]unix.RawSockaddrInet6 iovs [sendBatch]unix.Iovec - hdrs [sendBatch]msghdrX + hdrs [sendBatch]msgx.Hdr // idx maps each entry back to its index in WriteBatch's bufs. idx [sendBatch]int } @@ -190,7 +192,7 @@ func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) b.iovs[n].Base = &bufs[i][0] b.iovs[n].SetLen(len(bufs[i])) } - b.hdrs[n] = msghdrX{Name: (*byte)(unsafe.Pointer(&b.names[n])), Namelen: namelen, Iov: &b.iovs[n], Iovlen: 1} + b.hdrs[n] = msgx.Hdr{Name: (*byte)(unsafe.Pointer(&b.names[n])), Namelen: namelen, Iov: &b.iovs[n], Iovlen: 1} b.idx[n] = i n++ } @@ -252,11 +254,11 @@ func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) // The socket is non-blocking, so a full send buffer returns EAGAIN rather than waiting. func (u *StdConn) sendmsgX(off, end int) (int, syscall.Errno) { for { - r0, _, errno := unix.Syscall6(unix.SYS_SENDMSG_X, u.sysFd, uintptr(unsafe.Pointer(&u.sb.hdrs[off])), uintptr(end-off), 0, 0, 0) + sent, errno := msgx.Send(u.sysFd, u.sb.hdrs[off:end], 0) if errno == unix.EINTR { continue } - return int(r0), errno + return sent, errno } } @@ -330,23 +332,12 @@ func NewUDPStatsEmitter(udpConns []Conn) func() { // recvBatch is how many datagrams one recvmsg_x call may return. const recvBatch = 64 -// msghdrX mirrors xnu's struct msghdr_x (bsd/sys/socket_private.h). recvmsg_x reports each -// datagram's length in Datalen rather than in its return value. -// Keep in sync with msghdrX in overlay/tio/tio_utun_darwin.go. -type msghdrX struct { - Name *byte - Namelen uint32 - Iov *unix.Iovec - Iovlen int32 - Control *byte - Controllen uint32 - Flags int32 - Datalen uint64 -} - // ListenOut drains the socket with recvmsg_x, darwin's private batched recvmsg, so one syscall reads // up to recvBatch datagrams instead of one. func (u *StdConn) ListenOut(r EncReader, flush func()) error { + if !msgx.Available() { + return u.listenOutSingle(r, flush) + } rc, err := u.UDPConn.SyscallConn() if err != nil { return err @@ -355,7 +346,7 @@ func (u *StdConn) ListenOut(r EncReader, flush func()) error { bufs := make([][]byte, recvBatch) names := make([]unix.RawSockaddrInet6, recvBatch) iovs := make([]unix.Iovec, recvBatch) - hdrs := make([]msghdrX, recvBatch) + hdrs := make([]msgx.Hdr, recvBatch) for i := range hdrs { bufs[i] = make([]byte, MTU) iovs[i].Base = &bufs[i][0] @@ -374,12 +365,8 @@ func (u *StdConn) ListenOut(r EncReader, flush func()) error { hdrs[i].Flags = 0 hdrs[i].Datalen = 0 } - r0, _, e := unix.Syscall6(unix.SYS_RECVMSG_X, fd, uintptr(unsafe.Pointer(&hdrs[0])), recvBatch, 0, 0, 0) - if e == unix.EAGAIN { - return false - } - n, errno = int(r0), e - return true + n, errno = msgx.Recv(fd, hdrs, 0) + return errno != unix.EAGAIN }) if err != nil { if errors.Is(err, net.ErrClosed) { @@ -389,7 +376,7 @@ func (u *StdConn) ListenOut(r EncReader, flush func()) error { continue } if errno == unix.ENOSYS || errno == unix.EPERM || errno == unix.EOPNOTSUPP { - // recvmsg_x is private, so a kernel or sandbox (such as an iOS extension's) may refuse it. + // recvmsg_x is private, so a kernel or sandbox may refuse it. u.l.Warn("recvmsg_x unavailable, reading one datagram per syscall", "error", errno) return u.listenOutSingle(r, flush) } @@ -419,7 +406,7 @@ func (u *StdConn) ListenOut(r EncReader, flush func()) error { // checkMsghdrX rejects a recvmsg_x result that intact msghdr_x entries can't produce, which is how a change // to xnu's private struct layout would show up. -func checkMsghdrX(hdrs []msghdrX, names []unix.RawSockaddrInet6, n int) error { +func checkMsghdrX(hdrs []msgx.Hdr, names []unix.RawSockaddrInet6, n int) error { if n > len(hdrs) { return fmt.Errorf("returned %d datagrams for %d headers", n, len(hdrs)) } diff --git a/udp/udp_recvmsgx_darwin_test.go b/udp/udp_recvmsgx_darwin_test.go index 357d5a45..09fbfcf1 100644 --- a/udp/udp_recvmsgx_darwin_test.go +++ b/udp/udp_recvmsgx_darwin_test.go @@ -14,6 +14,7 @@ import ( "time" "unsafe" + "github.com/slackhq/nebula/internal/msgx" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" @@ -81,7 +82,7 @@ func TestListenOutBatchesBurst(t *testing.T) { } } -// msghdrX round-trips through xnu's recvmsg_x on a loopback socket, v4 and v6. The kernel reads Name, Namelen, +// msgx.Hdr round-trips through xnu's recvmsg_x on a loopback socket, v4 and v6. The kernel reads Name, Namelen, // Iov and Iovlen from each entry and writes Namelen, Flags and Datalen back, so a field at the wrong offset or a // wrong stride between entries shows up as a wrong length, sender, or payload. // TestListenOutSingleFallback pins the path ListenOut takes when the kernel refuses recvmsg_x. @@ -134,8 +135,8 @@ func TestListenOutSingleFallback(t *testing.T) { } func TestCheckMsghdrX(t *testing.T) { - valid := func() ([]msghdrX, []unix.RawSockaddrInet6) { - hdrs := make([]msghdrX, 2) + valid := func() ([]msgx.Hdr, []unix.RawSockaddrInet6) { + hdrs := make([]msgx.Hdr, 2) names := make([]unix.RawSockaddrInet6, 2) hdrs[0].Namelen, hdrs[0].Datalen = unix.SizeofSockaddrInet4, 1 names[0].Family = unix.AF_INET @@ -151,12 +152,12 @@ func TestCheckMsghdrX(t *testing.T) { for _, tc := range []struct { name string n int - mutate func([]msghdrX, []unix.RawSockaddrInet6) + mutate func([]msgx.Hdr, []unix.RawSockaddrInet6) }{ - {"count past headers", 3, func([]msghdrX, []unix.RawSockaddrInet6) {}}, - {"datalen past buffer", 2, func(h []msghdrX, _ []unix.RawSockaddrInet6) { h[1].Datalen = MTU + 1 }}, - {"unknown family", 2, func(_ []msghdrX, a []unix.RawSockaddrInet6) { a[0].Family = unix.AF_UNIX }}, - {"family and length disagree", 2, func(h []msghdrX, _ []unix.RawSockaddrInet6) { h[0].Namelen = unix.SizeofSockaddrInet6 }}, + {"count past headers", 3, func([]msgx.Hdr, []unix.RawSockaddrInet6) {}}, + {"datalen past buffer", 2, func(h []msgx.Hdr, _ []unix.RawSockaddrInet6) { h[1].Datalen = MTU + 1 }}, + {"unknown family", 2, func(_ []msgx.Hdr, a []unix.RawSockaddrInet6) { a[0].Family = unix.AF_UNIX }}, + {"family and length disagree", 2, func(h []msgx.Hdr, _ []unix.RawSockaddrInet6) { h[0].Namelen = unix.SizeofSockaddrInet6 }}, } { t.Run(tc.name, func(t *testing.T) { hdrs, names := valid() @@ -238,7 +239,7 @@ func recvmsgXRoundTrip(t *testing.T, v6 bool, lens []int, nh int, settle time.Du bufs := make([][]byte, nh) names := make([]unix.RawSockaddrInet6, nh) iovs := make([]unix.Iovec, nh) - hdrs := make([]msghdrX, nh) + hdrs := make([]msgx.Hdr, nh) for i := range hdrs { bufs[i] = make([]byte, MTU) iovs[i].Base = &bufs[i][0] @@ -253,7 +254,7 @@ func recvmsgXRoundTrip(t *testing.T, v6 bool, lens []int, nh int, settle time.Du for next := 0; next < len(lens); { for i := range hdrs { clear(bufs[i]) - hdrs[i] = msghdrX{ + hdrs[i] = msgx.Hdr{ Name: (*byte)(unsafe.Pointer(&names[i])), Namelen: unix.SizeofSockaddrInet6, Iov: &iovs[i], @@ -265,9 +266,8 @@ func recvmsgXRoundTrip(t *testing.T, v6 bool, lens []int, nh int, settle time.Du var n int var errno syscall.Errno require.NoError(t, rc.Read(func(fd uintptr) bool { - r0, _, e := unix.Syscall6(unix.SYS_RECVMSG_X, fd, uintptr(unsafe.Pointer(&hdrs[0])), uintptr(nh), 0, 0, 0) - n, errno = int(r0), e - return e != unix.EAGAIN + n, errno = msgx.Recv(fd, hdrs[:nh], 0) + return errno != unix.EAGAIN }), "received %d of %d datagrams", next, len(lens)) require.Zero(t, errno) require.Positive(t, n)