mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-17 02:56:52 +01:00
udp/darwin: batch receives and connected-socket sends with recvmsg_x/sendmsg_x
internal/msgx is the one copy of xnu's private batched socket calls: the msghdr_x layout, Send/Recv through the raw syscall (EINTR retried) and Unsupported for the errnos a kernel or sandbox refuses them with. Every socket the listener owns reads with recvmsg_x: the listener and each connected socket run their own reader goroutine, and hand whole batches to ListenOut over a channel. ListenOut runs the EncReader over a batch and flushes once, so the reader stays single-threaded without a lock that readers contend on, and the next recvmsg_x overlaps the decrypt of the last batch. Batch buffers (64 x MTU) come from a per-listener free list that keeps up to 11; a reader takes one only while its socket is readable, so idle connected sockets hold none. A batch with a header intact msghdr_x entries can't produce, or ENOSYS/EPERM/EOPNOTSUPP, switches every reader to one datagram per read for good. Connected sockets send runs with sendmsg_x, which xnu batches through the stack on a connected socket. EAGAIN/ENOBUFS keep #1901's single-datagram stall handling; a refused sendmsg_x retries the datagram with write and turns sendmsg_x off once that goes through. The connected-socket knobs lose their NEBULA_CONNSOCK* environment overrides and keep #1901's defaults. Based on Caleb Jasik's #1895 (recvmsg_x) and #1897 (sendmsg_x), and Jack Doan's internal/msgx shape from #1898. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
37032a22bf
commit
21075ba32e
7 files changed
+808
-125
No files matched your search
+190
-20
@@ -10,12 +10,12 @@ import (
|
||||
"log/slog"
|
||||
"net"
|
||||
"net/netip"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"syscall"
|
||||
"unsafe"
|
||||
|
||||
"github.com/slackhq/nebula/config"
|
||||
"github.com/slackhq/nebula/internal/msgx"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
@@ -31,15 +31,13 @@ type StdConn struct {
|
||||
// rc reaches the listener's descriptor for another listen routine's Rebind.
|
||||
rc syscall.RawConn
|
||||
socks *connSocks
|
||||
// reader is ListenOut's reader, which connected socket readers share; readMu serializes them, since an
|
||||
// EncReader isn't safe for concurrent use.
|
||||
reader atomic.Pointer[readerPair]
|
||||
readMu sync.Mutex
|
||||
}
|
||||
|
||||
type readerPair struct {
|
||||
r EncReader
|
||||
flush func()
|
||||
// listening is set once ListenOut runs. The listener's and every connected socket's readers hand their batches
|
||||
// to it on rx, so the EncReader runs on ListenOut's goroutine alone; done closes when ListenOut returns.
|
||||
listening atomic.Bool
|
||||
rx chan *rxBatch
|
||||
done chan struct{}
|
||||
// free holds the batches ListenOut has delivered, for the readers to fill again.
|
||||
free chan *rxBatch
|
||||
}
|
||||
|
||||
var _ Conn = &StdConn{}
|
||||
@@ -53,7 +51,8 @@ func NewListener(l *slog.Logger, s Settings) (Conn, error) {
|
||||
}
|
||||
|
||||
if uc, ok := pc.(*net.UDPConn); ok {
|
||||
c := &StdConn{UDPConn: uc, l: l, listenHost: s.Listen.Addr()}
|
||||
c := &StdConn{UDPConn: uc, l: l, listenHost: s.Listen.Addr(), rx: make(chan *rxBatch, 1), done: make(chan struct{}),
|
||||
free: make(chan *rxBatch, rxBatchesKept)}
|
||||
|
||||
rc, err := uc.SyscallConn()
|
||||
if err != nil {
|
||||
@@ -247,14 +246,176 @@ func NewUDPStatsEmitter(udpConns []Conn) func() {
|
||||
return func() {}
|
||||
}
|
||||
|
||||
// ListenOut delivers the batches that the listener's reader and the connected sockets' readers hand it.
|
||||
func (u *StdConn) ListenOut(r EncReader, flush func()) error {
|
||||
buffer := make([]byte, MTU)
|
||||
u.reader.Store(&readerPair{r: r, flush: flush})
|
||||
defer close(u.done)
|
||||
u.listening.Store(true)
|
||||
u.startConnSocks()
|
||||
|
||||
errc := make(chan error, 1)
|
||||
go func() { errc <- u.readListener() }()
|
||||
for {
|
||||
// Just read one packet at a time
|
||||
n, rua, err := u.ReadFromUDPAddrPort(buffer)
|
||||
select {
|
||||
case b := <-u.rx:
|
||||
for i := range b.n {
|
||||
r(b.addrs[i], b.pkts[i])
|
||||
}
|
||||
flush()
|
||||
u.putBatch(b)
|
||||
case err := <-errc:
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rxBatch is one recvmsg_x batch: n datagrams in pkts, from addrs. A reader takes one only while its socket is
|
||||
// readable, so an idle connected socket holds none.
|
||||
type rxBatch struct {
|
||||
n int
|
||||
pkts [msgx.Batch][]byte
|
||||
addrs [msgx.Batch]netip.AddrPort
|
||||
buf []byte
|
||||
names [msgx.Batch]unix.RawSockaddrInet6
|
||||
iovs [msgx.Batch]unix.Iovec
|
||||
hdrs [msgx.Batch]msgx.Hdr
|
||||
}
|
||||
|
||||
// rxBatchesKept is how many batches a listener keeps for reuse: one being delivered and one queued, plus one for
|
||||
// each socket reading at once, the listener and up to 8 connected sockets. Each is msgx.Batch*MTU, about 576KB.
|
||||
const rxBatchesKept = 11
|
||||
|
||||
func (u *StdConn) getBatch() *rxBatch {
|
||||
select {
|
||||
case b := <-u.free:
|
||||
return b
|
||||
default:
|
||||
}
|
||||
b := &rxBatch{buf: make([]byte, msgx.Batch*MTU)}
|
||||
for i := range b.hdrs {
|
||||
b.iovs[i].Base = &b.buf[i*MTU]
|
||||
b.iovs[i].SetLen(MTU)
|
||||
b.hdrs[i] = msgx.Hdr{Name: (*byte)(unsafe.Pointer(&b.names[i])), Iov: &b.iovs[i], Iovlen: 1}
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (u *StdConn) putBatch(b *rxBatch) {
|
||||
select {
|
||||
case u.free <- b:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// noRecvX is set once recvmsg_x is refused or misbehaves; every reader then reads one datagram per syscall.
|
||||
var noRecvX atomic.Bool
|
||||
|
||||
// parse checks the n entries recvmsg_x filled and sets pkts and addrs from them. It rejects a batch that intact
|
||||
// msghdr_x entries can't produce, which is how a change to xnu's private struct layout would show up.
|
||||
func (b *rxBatch) parse(n int) error {
|
||||
if n > len(b.hdrs) {
|
||||
return fmt.Errorf("returned %d datagrams for %d headers", n, len(b.hdrs))
|
||||
}
|
||||
for i := range n {
|
||||
h := &b.hdrs[i]
|
||||
if h.Datalen > MTU {
|
||||
return fmt.Errorf("datagram %d is %d bytes, past the %d byte buffer", i, h.Datalen, MTU)
|
||||
}
|
||||
sa := &b.names[i]
|
||||
switch {
|
||||
case sa.Family == unix.AF_INET && h.Namelen == unix.SizeofSockaddrInet4:
|
||||
sa4 := (*unix.RawSockaddrInet4)(unsafe.Pointer(sa))
|
||||
b.addrs[i] = netip.AddrPortFrom(netip.AddrFrom4(sa4.Addr), binary.BigEndian.Uint16((*[2]byte)(unsafe.Pointer(&sa4.Port))[:]))
|
||||
case sa.Family == unix.AF_INET6 && h.Namelen == unix.SizeofSockaddrInet6:
|
||||
b.addrs[i] = netip.AddrPortFrom(netip.AddrFrom16(sa.Addr).Unmap(), binary.BigEndian.Uint16((*[2]byte)(unsafe.Pointer(&sa.Port))[:]))
|
||||
default:
|
||||
return fmt.Errorf("datagram %d has address family %d with length %d", i, sa.Family, h.Namelen)
|
||||
}
|
||||
b.pkts[i] = b.buf[i*MTU : i*MTU+int(h.Datalen) : i*MTU+int(h.Datalen)]
|
||||
}
|
||||
b.n = n
|
||||
return nil
|
||||
}
|
||||
|
||||
// batchReader reads one socket in batches. Its fn is bound once, since a closure passed through the RawConn
|
||||
// interface escapes and would allocate on every read.
|
||||
type batchReader struct {
|
||||
u *StdConn
|
||||
conn *net.UDPConn
|
||||
rc syscall.RawConn
|
||||
b *rxBatch
|
||||
errno syscall.Errno
|
||||
fn func(fd uintptr) bool
|
||||
}
|
||||
|
||||
func (u *StdConn) newBatchReader(conn *net.UDPConn, rc syscall.RawConn) *batchReader {
|
||||
r := &batchReader{u: u, conn: conn, rc: rc}
|
||||
r.fn = r.recv
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *batchReader) recv(fd uintptr) bool {
|
||||
b := r.u.getBatch()
|
||||
for i := range b.hdrs {
|
||||
b.hdrs[i].Namelen = unix.SizeofSockaddrInet6
|
||||
b.hdrs[i].Flags = 0
|
||||
b.hdrs[i].Datalen = 0
|
||||
}
|
||||
n, errno := msgx.Recv(fd, b.hdrs[:])
|
||||
if errno == unix.EAGAIN {
|
||||
r.u.putBatch(b)
|
||||
return false
|
||||
}
|
||||
b.n, r.b, r.errno = n, b, errno
|
||||
return true
|
||||
}
|
||||
|
||||
// read returns the next batch: up to msgx.Batch datagrams from one recvmsg_x, or one datagram once recvmsg_x is off.
|
||||
func (u *StdConn) read(r *batchReader) (*rxBatch, error) {
|
||||
if !noRecvX.Load() {
|
||||
r.b = nil
|
||||
err := r.rc.Read(r.fn)
|
||||
b := r.b
|
||||
switch {
|
||||
case err != nil:
|
||||
if b != nil {
|
||||
u.putBatch(b)
|
||||
}
|
||||
return nil, err
|
||||
case r.errno == 0:
|
||||
perr := b.parse(b.n)
|
||||
if perr == nil {
|
||||
return b, nil
|
||||
}
|
||||
// The batch was read through a layout the kernel no longer writes, so none of it is trustworthy.
|
||||
u.disableRecvX("returned an unexpected header", perr)
|
||||
case msgx.Unsupported(r.errno):
|
||||
u.disableRecvX("unavailable", r.errno)
|
||||
default:
|
||||
u.putBatch(b)
|
||||
return nil, r.errno
|
||||
}
|
||||
u.putBatch(b)
|
||||
}
|
||||
b := u.getBatch()
|
||||
n, from, err := r.conn.ReadFromUDPAddrPort(b.buf[:MTU])
|
||||
if err != nil {
|
||||
u.putBatch(b)
|
||||
return nil, err
|
||||
}
|
||||
b.n, b.pkts[0], b.addrs[0] = 1, b.buf[:n:n], netip.AddrPortFrom(from.Addr().Unmap(), from.Port())
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func (u *StdConn) disableRecvX(why string, err error) {
|
||||
if noRecvX.CompareAndSwap(false, true) {
|
||||
u.l.Warn("recvmsg_x "+why+", reading one datagram per syscall", "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
// readListener hands the listener's batches to ListenOut until the listener closes.
|
||||
func (u *StdConn) readListener() error {
|
||||
r := u.newBatchReader(u.UDPConn, u.rc)
|
||||
for {
|
||||
b, err := u.read(r)
|
||||
if err != nil {
|
||||
if errors.Is(err, net.ErrClosed) {
|
||||
return err
|
||||
@@ -262,11 +423,20 @@ func (u *StdConn) ListenOut(r EncReader, flush func()) error {
|
||||
u.l.Error("unexpected udp socket receive error", "error", err)
|
||||
continue
|
||||
}
|
||||
if !u.deliver(b) {
|
||||
return net.ErrClosed
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
u.readMu.Lock()
|
||||
r(netip.AddrPortFrom(rua.Addr().Unmap(), rua.Port()), buffer[:n:n])
|
||||
flush()
|
||||
u.readMu.Unlock()
|
||||
// deliver hands b to ListenOut, and reports false once ListenOut has returned.
|
||||
func (u *StdConn) deliver(b *rxBatch) bool {
|
||||
select {
|
||||
case u.rx <- b:
|
||||
return true
|
||||
case <-u.done:
|
||||
u.putBatch(b)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user