Compare commits

..
Author SHA1 Message Date
Caleb JasikandClaude Opus 5.5 c914ea30ed overlay/darwin: drain several tun packets per read wakeup
The darwin tun returned one packet per Read, and listenIn flushes its
UDP send batch after every Read, so each outbound packet went to the
underlay as its own batch. tunQueue.Read now keeps calling readv until
the utun would block, up to tunReadBatch packets, so a burst reaches the
send batch together.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-25 13:16:50 -05:00
9 changed files with 124 additions and 891 deletions

No files matched your search

-62
View File
@@ -435,65 +435,3 @@ func testPP(pkt []byte) *firewall.ParsedPacket {
} }
return pp return pp
} }
// fakeTunBatchWriter records each WriteBatch call and each plain Write separately.
type fakeTunBatchWriter struct {
writes [][]byte
batches [][][]byte
}
func (w *fakeTunBatchWriter) Write(p []byte) (int, error) {
w.writes = append(w.writes, bytes.Clone(p))
return len(p), nil
}
func (w *fakeTunBatchWriter) WriteBatch(pkts [][]byte) error {
var b [][]byte
for _, p := range pkts {
b = append(b, bytes.Clone(p))
}
w.batches = append(w.batches, b)
return nil
}
// TestPassthroughUsesWriteBatch pins that a writer implementing tio.BatchWriter gets the whole
// passthrough lane in one WriteBatch per Flush, in order, and no plain Writes; an empty Flush
// makes no call.
func TestPassthroughUsesWriteBatch(t *testing.T) {
w := &fakeTunBatchWriter{}
m := newTestMultiCoalescer(t, w)
k := &keySeq{epoch: 1}
var want [][]byte
for i := range 5 {
icmp := make([]byte, 28)
icmp[0] = 0x45
icmp[3] = 28
icmp[9] = 1
icmp[20] = byte(i)
want = append(want, icmp)
if err := m.Commit(icmp, k.next(), testPP(icmp)); err != nil {
t.Fatal(err)
}
}
if err := m.Flush(); err != nil {
t.Fatal(err)
}
if err := m.Flush(); err != nil {
t.Fatal(err)
}
if len(w.writes) != 0 {
t.Fatalf("got %d plain Writes, want 0", len(w.writes))
}
if len(w.batches) != 1 {
t.Fatalf("got %d WriteBatch calls, want 1", len(w.batches))
}
for i, p := range w.batches[0] {
if !bytes.Equal(p, want[i]) {
t.Fatalf("packet %d: got % x, want % x", i, p, want[i])
}
}
if len(w.batches[0]) != len(want) {
t.Fatalf("got %d packets, want %d", len(w.batches[0]), len(want))
}
}
+2 -13
View File
@@ -2,23 +2,18 @@ package batch
import ( import (
"io" "io"
"github.com/slackhq/nebula/overlay/tio"
) )
// Passthrough is MultiCoalescer's verbatim lane: no coalescing, packets are written at Flush in the // Passthrough is MultiCoalescer's verbatim lane: no batching, packets are written at Flush in the
// order enqueued, in one WriteBatch when out implements tio.BatchWriter. // order enqueued.
type Passthrough struct { type Passthrough struct {
out io.Writer out io.Writer
bw tio.BatchWriter // nil: Flush writes one packet per Write
slots [][]byte slots [][]byte
} }
func NewPassthrough(w io.Writer) *Passthrough { func NewPassthrough(w io.Writer) *Passthrough {
bw, _ := w.(tio.BatchWriter)
return &Passthrough{ return &Passthrough{
out: w, out: w,
bw: bw,
slots: make([][]byte, 0, 128), slots: make([][]byte, 0, 128),
} }
} }
@@ -31,18 +26,12 @@ func (p *Passthrough) enqueue(pkt []byte) error {
func (p *Passthrough) Flush() error { func (p *Passthrough) Flush() error {
var firstErr error var firstErr error
if p.bw != nil {
if len(p.slots) > 0 {
firstErr = p.bw.WriteBatch(p.slots)
}
} else {
for _, s := range p.slots { for _, s := range p.slots {
_, err := p.out.Write(s) _, err := p.out.Write(s)
if err != nil && firstErr == nil { if err != nil && firstErr == nil {
firstErr = err firstErr = err
} }
} }
}
clear(p.slots) clear(p.slots)
p.slots = p.slots[:0] p.slots = p.slots[:0]
return firstErr return firstErr
+2 -17
View File
@@ -15,13 +15,9 @@ type singleQueue struct {
// NewSingleQueue wraps a one-datagram-per-Read ReadWriteCloser (a legacy tun device) into a Queue. // NewSingleQueue wraps a one-datagram-per-Read ReadWriteCloser (a legacy tun device) into a Queue.
// bufSize is the per-queue read scratch size and must be at least the largest datagram the source can return. // bufSize is the per-queue read scratch size and must be at least the largest datagram the source can return.
// Close closes rwc. The Queue implements BatchWriter when rwc does. // Close closes rwc.
func NewSingleQueue(rwc io.ReadWriteCloser, bufSize int) Queue { func NewSingleQueue(rwc io.ReadWriteCloser, bufSize int) Queue {
q := &singleQueue{rw: rwc, closer: rwc, buf: make([]byte, bufSize)} return &singleQueue{rw: rwc, closer: rwc, buf: make([]byte, bufSize)}
if bw, ok := rwc.(BatchWriter); ok {
return &batchSingleQueue{singleQueue: q, bw: bw}
}
return q
} }
// NewSingleQueueNoClose is NewSingleQueue for a source owned by someone else, // NewSingleQueueNoClose is NewSingleQueue for a source owned by someone else,
@@ -51,14 +47,3 @@ func (q *singleQueue) Close() error {
} }
return q.closer.Close() return q.closer.Close()
} }
// batchSingleQueue is a singleQueue whose source also implements BatchWriter.
// It is a separate type so a plain singleQueue never advertises WriteBatch.
type batchSingleQueue struct {
*singleQueue
bw BatchWriter
}
func (q *batchSingleQueue) WriteBatch(pkts [][]byte) error {
return q.bw.WriteBatch(pkts)
}
-11
View File
@@ -145,14 +145,3 @@ func SupportsGSO(w io.Writer, want GSOProto) (GSOWriter, bool) {
return gw, false return gw, false
} }
} }
// BatchWriter is implemented by Queues that can emit several already-segmented packets in one call.
// Backends without a batched write do not implement it and callers loop over Write.
//
// pkts are borrowed: WriteBatch must not retain them past its return.
// A returned error doesn't say which packets were delivered, so callers must not retry the batch.
// Safe for concurrent use, like Write.
type BatchWriter interface {
io.Writer
WriteBatch(pkts [][]byte) error
}
+75 -176
View File
@@ -8,7 +8,6 @@ import (
"log/slog" "log/slog"
"net/netip" "net/netip"
"os" "os"
"sync"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -34,35 +33,6 @@ type tun struct {
// and routes for it. NEPacketTunnelProvider on darwin does this. // and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool hostOwned bool
l *slog.Logger l *slog.Logger
// noSendmsgX is set once sendmsg_x is refused; WriteBatch then writes one packet at a time.
noSendmsgX atomic.Bool
batchMu sync.Mutex
batch tunBatch
}
// tunWriteBatch is how many packets one sendmsg_x call may carry.
const tunWriteBatch = 64
// tunBatch is WriteBatch's scratch, guarded by tun.batchMu. The kernel reads every entry in place.
type tunBatch struct {
heads [tunWriteBatch][4]byte
iovs [tunWriteBatch][2]unix.Iovec
hdrs [tunWriteBatch]msghdrX
pkts [tunWriteBatch][]byte
}
// msghdrX mirrors xnu's struct msghdr_x (bsd/sys/socket_private.h).
// Keep in sync with msghdrX in udp/udp_darwin.go.
type msghdrX struct {
Name *byte
Namelen uint32
Iov *unix.Iovec
Iovlen int32
Control *byte
Controllen uint32
Flags int32
Datalen uint64
} }
type ifReq struct { type ifReq struct {
@@ -637,6 +607,72 @@ func (t *tun) Read(to []byte) (int, error) {
return n - 4, nil return n - 4, nil
} }
// tunReadBatch is the most packets one tunQueue.Read returns.
const tunReadBatch = 64
// tunReadArena is tunQueue's receive buffer. Draining stops once less than defaultBatchBufSize of it
// is left, so every readv has room for the largest packet utun can return at any device MTU.
const tunReadArena = 4 * defaultBatchBufSize
// tunQueue is the darwin tun's Queue. Read drains every packet already queued on the utun, up to
// tunReadBatch, so the caller encrypts and sends them as one batch rather than one per wakeup.
type tunQueue struct {
t *tun
buf []byte
pkts [tunReadBatch]tio.Packet
}
// Read waits for the utun to become readable, then reads packets until it would block. An error after
// at least one packet is dropped in favor of returning those packets; a persistent one recurs on the
// next Read.
func (q *tunQueue) Read() ([]tio.Packet, error) {
rc, err := q.t.f.SyscallConn()
if err != nil {
return nil, err
}
var head [4]byte
n, off := 0, 0
var callErr error
err = rc.Read(func(fd uintptr) bool {
for n < tunReadBatch && len(q.buf)-off >= defaultBatchBufSize {
iovecs := [2]unix.Iovec{
{Base: &head[0], Len: 4},
{Base: &q.buf[off], Len: uint64(len(q.buf) - off)},
}
l, e := tunReadv(int(fd), iovecs[:])
if e != nil {
if errno, ok := e.(syscall.Errno); ok && errno.Temporary() {
// Park on the poller only while there is nothing to hand back.
return n > 0
}
callErr = e
return true
}
if l < 4 {
// A datagram too short to carry the AF prefix, or end of file; stop rather than spin on it.
return true
}
end := off + l - 4
q.pkts[n] = tio.Packet{Bytes: q.buf[off:end:end]}
n++
off = end
}
return true
})
if n > 0 {
return q.pkts[:n], nil
}
if err != nil {
return nil, err
}
return nil, callErr
}
func (q *tunQueue) Write(p []byte) (int, error) { return q.t.Write(p) }
func (q *tunQueue) Close() error { return q.t.Close() }
// Write pushes one IP packet onto the utun device. Safe for concurrent use: // Write pushes one IP packet onto the utun device. Safe for concurrent use:
// the AF prefix and iovecs are per-call stack state, and the fd write itself // the AF prefix and iovecs are per-call stack state, and the fd write itself
// serializes on the runtime's fd mutex (see the Queue contract in tio.go). // serializes on the runtime's fd mutex (see the Queue contract in tio.go).
@@ -645,12 +681,16 @@ func (t *tun) Write(from []byte) (int, error) {
return 0, syscall.EIO return 0, syscall.EIO
} }
ipVer := from[0] >> 4
var head [4]byte var head [4]byte
af, err := tunAF(from) switch ipVer {
if err != nil { case 4:
return 0, err head[3] = syscall.AF_INET
case 6:
head[3] = syscall.AF_INET6
default:
return 0, fmt.Errorf("unable to determine IP version from packet")
} }
head[3] = af
// Grab rc as a local so the compiler can devirtualize the call and keep the closure on the stack. // Grab rc as a local so the compiler can devirtualize the call and keep the closure on the stack.
rc, err := t.f.SyscallConn() rc, err := t.f.SyscallConn()
@@ -683,147 +723,6 @@ func (t *tun) Write(from []byte) (int, error) {
return n - 4, nil return n - 4, nil
} }
// tunAF returns the utun address-family prefix byte for an IP packet.
func tunAF(pkt []byte) (byte, error) {
switch pkt[0] >> 4 {
case 4:
return syscall.AF_INET, nil
case 6:
return syscall.AF_INET6, nil
default:
return 0, fmt.Errorf("unable to determine IP version from packet")
}
}
// WriteBatch writes pkts to the utun device with sendmsg_x, xnu's private batched sendmsg, up to
// tunWriteBatch packets per syscall. The kernel still hands each packet to utun on its own, so this
// saves syscalls, not per-packet kernel work. Safe for concurrent use.
//
// An empty packet or one with no IP version is skipped and reported, as Write would. Any other
// error from sendmsg_x means the kernel took an unknown prefix of that call and dropped the rest, so
// it is reported rather than retried, which could deliver packets twice.
//
// Under mbuf exhaustion, sendmsg_x silently drops packets that writev would have delivered: the
// kernel counts each packet as sent once it is copied in, and when a later allocation fails it
// frees that prefix unsent and returns its length as a short count, which is indistinguishable
// from a real one.
func (t *tun) WriteBatch(pkts [][]byte) error {
if t.noSendmsgX.Load() {
return t.writeEach(pkts)
}
t.batchMu.Lock()
defer t.batchMu.Unlock()
b := &t.batch
var firstErr error
for len(pkts) > 0 {
n := 0
for len(pkts) > 0 && n < tunWriteBatch {
p := pkts[0]
pkts = pkts[1:]
var af byte
err := error(syscall.EIO)
if len(p) > 0 {
af, err = tunAF(p)
}
if err != nil {
if firstErr == nil {
firstErr = err
}
continue
}
b.heads[n] = [4]byte{3: af}
b.iovs[n] = [2]unix.Iovec{
{Base: &b.heads[n][0], Len: 4},
{Base: &p[0], Len: uint64(len(p))},
}
b.hdrs[n] = msghdrX{Iov: &b.iovs[n][0], Iovlen: 2}
b.pkts[n] = p
n++
}
if n == 0 {
continue
}
for off := 0; off < n; {
sent, err := t.sendmsgX(off, n)
switch {
case err == nil && sent > 0:
// The kernel stops early without an error on a packet larger than the socket's send
// buffer, or when sending a later packet fails with ENOBUFS; resend the remainder.
off += sent
case err == nil:
if werr := t.writeEach(b.pkts[off:n]); werr != nil && firstErr == nil {
firstErr = werr
}
off = n
case err == unix.EMSGSIZE:
// Only the first packet being larger than the socket's send buffer fails the whole call,
// before the kernel takes any; Write reports that one and the rest go out in the next call.
if werr := t.writeEach(b.pkts[off : off+1]); werr != nil && firstErr == nil {
firstErr = werr
}
off++
case err == unix.ENOSYS || err == unix.EPERM || err == unix.EOPNOTSUPP:
// sendmsg_x is private API: fall back to writev if a kernel or sandbox refuses it.
if t.noSendmsgX.CompareAndSwap(false, true) {
t.l.Warn("sendmsg_x unavailable on the tun device, writing one packet per syscall", "error", err)
}
if werr := t.writeEach(b.pkts[off:n]); werr != nil && firstErr == nil {
firstErr = werr
}
clear(b.iovs[:n])
clear(b.pkts[:n])
if werr := t.writeEach(pkts); werr != nil && firstErr == nil {
firstErr = werr
}
return firstErr
default:
if firstErr == nil {
firstErr = err
}
off = n
}
}
clear(b.iovs[:n])
clear(b.pkts[:n])
}
return firstErr
}
// sendmsgX hands entries off through n-1 of t.batch to sendmsg_x and returns how many the kernel took.
func (t *tun) sendmsgX(off, n int) (int, error) {
rc, err := t.f.SyscallConn()
if err != nil {
return 0, err
}
var sent uintptr
var errno syscall.Errno
err = rc.Write(func(fd uintptr) bool {
sent, _, errno = unix.Syscall6(unix.SYS_SENDMSG_X, fd, uintptr(unsafe.Pointer(&t.batch.hdrs[off])), uintptr(n-off), 0, 0, 0)
// sendmsg_x reports EAGAIN only when it took nothing; a partial batch returns its count.
return !errno.Temporary()
})
if err != nil {
return 0, err
}
if errno != 0 {
return 0, errno
}
return int(sent), nil
}
// writeEach writes pkts one at a time with Write.
func (t *tun) writeEach(pkts [][]byte) error {
var firstErr error
for _, p := range pkts {
if _, err := t.Write(p); err != nil && firstErr == nil {
firstErr = err
}
}
return firstErr
}
func (t *tun) Networks() []netip.Prefix { func (t *tun) Networks() []netip.Prefix {
return t.vpnNetworks return t.vpnNetworks
} }
@@ -833,5 +732,5 @@ func (t *tun) Name() string {
} }
func (t *tun) Queues(int) ([]tio.Queue, error) { func (t *tun) Queues(int) ([]tio.Queue, error) {
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil return []tio.Queue{&tunQueue{t: t, buf: make([]byte, tunReadArena)}}, nil
} }
+44 -100
View File
@@ -3,7 +3,6 @@
package overlay package overlay
import ( import (
"bytes"
"os" "os"
"syscall" "syscall"
"testing" "testing"
@@ -14,9 +13,9 @@ import (
"golang.org/x/sys/unix" "golang.org/x/sys/unix"
) )
// newSocketpairTun returns a tun writing to one end of a connected AF_UNIX datagram socketpair, which // newSocketpairTun returns a tun on one end of a connected AF_UNIX datagram socketpair, which keeps
// takes sendmsg_x the way a utun control socket does, and the other end's fd for reading back. // datagram boundaries the way a utun control socket does, and the other end's fd.
// sndbuf is the writer's send buffer, which caps the largest datagram it accepts. // sndbuf is the tun end's send buffer, which caps the largest datagram it accepts.
func newSocketpairTun(t *testing.T, sndbuf int) (*tun, int) { func newSocketpairTun(t *testing.T, sndbuf int) (*tun, int) {
t.Helper() t.Helper()
fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_DGRAM, 0) fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_DGRAM, 0)
@@ -34,8 +33,8 @@ func newSocketpairTun(t *testing.T, sndbuf int) (*tun, int) {
return &tun{f: f, l: test.NewLogger()}, fds[1] return &tun{f: f, l: test.NewLogger()}, fds[1]
} }
// testTunPkt returns packet i of size bytes, alternating IPv4 and IPv6, and the datagram the utun // testTunPkt returns packet i of size bytes, alternating IPv4 and IPv6, and the same packet as a utun
// device should receive for it. // datagram, behind its AF prefix.
func testTunPkt(i, size int) (pkt, wire []byte) { func testTunPkt(i, size int) (pkt, wire []byte) {
pkt = make([]byte, size) pkt = make([]byte, size)
pkt[0] = 0x45 pkt[0] = 0x45
@@ -49,105 +48,50 @@ func testTunPkt(i, size int) (pkt, wire []byte) {
return pkt, append([]byte{0, 0, 0, af}, pkt...) return pkt, append([]byte{0, 0, 0, af}, pkt...)
} }
// readTunBack asserts r receives exactly want, in order. // TestTunQueueReadDrains pins that Read hands back every queued packet, AF prefix stripped and in
func readTunBack(t *testing.T, r int, want [][]byte) { // order, capped at tunReadBatch per call, and doesn't wait for more once the queue is empty.
t.Helper() func TestTunQueueReadDrains(t *testing.T) {
buf := make([]byte, 4096) tn, w := newSocketpairTun(t, 1<<20)
for i, w := range want { // An AF_UNIX datagram send is bounded by the receiver's buffer, here the tun's end.
n, _, err := unix.Recvfrom(r, buf, 0) rc, err := tn.f.SyscallConn()
require.NoError(t, err, "datagram %d", i) require.NoError(t, err)
if !bytes.Equal(w, buf[:n]) { require.NoError(t, rc.Control(func(fd uintptr) {
t.Fatalf("datagram %d: got % x, want % x", i, buf[:n], w) require.NoError(t, unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_RCVBUF, 1<<20))
} }))
} q := &tunQueue{t: tn, buf: make([]byte, tunReadArena)}
_, _, err := unix.Recvfrom(r, buf, unix.MSG_DONTWAIT) total := tunReadBatch + 3
assert.ErrorIs(t, err, unix.EAGAIN, "extra datagrams delivered") var want [][]byte
for i := range total {
pkt, wire := testTunPkt(i, 60+i)
_, err := unix.Write(w, wire)
require.NoError(t, err)
want = append(want, pkt)
} }
// forEachTunWritePath runs fn once over sendmsg_x and once over the per-packet writev fallback. var got [][]byte
func forEachTunWritePath(t *testing.T, fn func(t *testing.T, fallback bool)) { for _, size := range []int{tunReadBatch, 3} {
for _, fallback := range []bool{false, true} { pkts, err := q.Read()
name := "sendmsg_x" require.NoError(t, err)
if fallback { require.Len(t, pkts, size)
name = "writev" for _, p := range pkts {
got = append(got, p.Clone().Bytes)
} }
t.Run(name, func(t *testing.T) { fn(t, fallback) })
} }
assert.Equal(t, want, got)
} }
// TestTunWriteBatch pins that WriteBatch delivers every packet whole, in order and behind the right // TestTunQueueReadClipsPackets pins that each returned packet's capacity ends at its own bytes, so
// utun AF prefix, across several sendmsg_x calls, and returns nil. // appending to one can't overwrite the next.
func TestTunWriteBatch(t *testing.T) { func TestTunQueueReadClipsPackets(t *testing.T) {
forEachTunWritePath(t, func(t *testing.T, fallback bool) { tn, w := newSocketpairTun(t, 1<<20)
tn, r := newSocketpairTun(t, 1<<20) q := &tunQueue{t: tn, buf: make([]byte, tunReadArena)}
tn.noSendmsgX.Store(fallback) for i := range 2 {
_, wire := testTunPkt(i, 100)
var pkts, want [][]byte _, err := unix.Write(w, wire)
for i := range 3*tunWriteBatch + 5 { require.NoError(t, err)
p, w := testTunPkt(i, 60+i)
pkts = append(pkts, p)
want = append(want, w)
}
require.NoError(t, tn.WriteBatch(pkts))
assert.Equal(t, fallback, tn.noSendmsgX.Load())
readTunBack(t, r, want)
})
}
// TestTunWriteBatchSkipsInvalid pins that an empty packet or one with no IP version is skipped
// without shifting the packets after it, and that the first such error is the one reported.
func TestTunWriteBatchSkipsInvalid(t *testing.T) {
forEachTunWritePath(t, func(t *testing.T, fallback bool) {
tn, r := newSocketpairTun(t, 1<<20)
tn.noSendmsgX.Store(fallback)
var pkts, want [][]byte
for i := range 20 {
p, w := testTunPkt(i, 60)
pkts = append(pkts, p)
want = append(want, w)
}
pkts = append(pkts[:10], append([][]byte{{0x10, 1, 2}, {}}, pkts[10:]...)...)
err := tn.WriteBatch(pkts)
assert.ErrorContains(t, err, "IP version")
readTunBack(t, r, want)
})
}
// TestTunWriteBatchOversize pins that a packet larger than the socket's send buffer drops only
// itself and is reported, whether sendmsg_x reaches it mid-call (a short count) or first (EMSGSIZE
// for the whole call). The send buffer is 1024 bytes and the oversize packet 2000.
func TestTunWriteBatchOversize(t *testing.T) {
for _, tc := range []struct {
name string
at int
}{
{"mid-call", tunWriteBatch + 6},
{"first-in-call", tunWriteBatch},
} {
t.Run(tc.name, func(t *testing.T) {
forEachTunWritePath(t, func(t *testing.T, fallback bool) {
tn, r := newSocketpairTun(t, 1024)
tn.noSendmsgX.Store(fallback)
var pkts, want [][]byte
for i := range 2*tunWriteBatch + 10 {
size := 100
if i == tc.at {
size = 2000
}
p, w := testTunPkt(i, size)
pkts = append(pkts, p)
if i != tc.at {
want = append(want, w)
}
}
err := tn.WriteBatch(pkts)
assert.ErrorIs(t, err, unix.EMSGSIZE)
assert.Equal(t, fallback, tn.noSendmsgX.Load())
readTunBack(t, r, want)
})
})
} }
pkts, err := q.Read()
require.NoError(t, err)
require.Len(t, pkts, 2)
assert.Equal(t, len(pkts[0].Bytes), cap(pkts[0].Bytes))
} }
+1 -125
View File
@@ -180,122 +180,11 @@ func NewUDPStatsEmitter(udpConns []Conn) func() {
return func() {} return func() {}
} }
// recvBatch is how many datagrams one recvmsg_x call may return.
const recvBatch = 64
// msghdrX mirrors xnu's struct msghdr_x (bsd/sys/socket_private.h). recvmsg_x reports each
// datagram's length in Datalen rather than in its return value.
// Keep in sync with msghdrX in overlay/tun_darwin.go.
type msghdrX struct {
Name *byte
Namelen uint32
Iov *unix.Iovec
Iovlen int32
Control *byte
Controllen uint32
Flags int32
Datalen uint64
}
// ListenOut drains the socket with recvmsg_x, darwin's private batched recvmsg, so one syscall reads
// up to recvBatch datagrams instead of one. Sends stay on sendto: xnu batches sendmsg_x through
// the stack only on connected sockets, and on this unconnected one it loops sendit per message.
func (u *StdConn) ListenOut(r EncReader, flush func()) error { func (u *StdConn) ListenOut(r EncReader, flush func()) error {
rc, err := u.UDPConn.SyscallConn()
if err != nil {
return err
}
bufs := make([][]byte, recvBatch)
names := make([]unix.RawSockaddrInet6, recvBatch)
iovs := make([]unix.Iovec, recvBatch)
hdrs := make([]msghdrX, recvBatch)
for i := range hdrs {
bufs[i] = make([]byte, MTU)
iovs[i].Base = &bufs[i][0]
iovs[i].SetLen(MTU)
hdrs[i].Iov = &iovs[i]
hdrs[i].Iovlen = 1
hdrs[i].Name = (*byte)(unsafe.Pointer(&names[i]))
}
for {
var n int
var errno syscall.Errno
err := rc.Read(func(fd uintptr) bool {
for i := range hdrs {
hdrs[i].Namelen = unix.SizeofSockaddrInet6
hdrs[i].Flags = 0
hdrs[i].Datalen = 0
}
r0, _, e := unix.Syscall6(unix.SYS_RECVMSG_X, fd, uintptr(unsafe.Pointer(&hdrs[0])), recvBatch, 0, 0, 0)
if e == unix.EAGAIN {
return false
}
n, errno = int(r0), e
return true
})
if err != nil {
if errors.Is(err, net.ErrClosed) {
return err
}
u.l.Error("unexpected udp socket receive error", "error", err)
continue
}
if errno == unix.ENOSYS || errno == unix.EPERM || errno == unix.EOPNOTSUPP {
// recvmsg_x is private, so a kernel or sandbox (such as an iOS extension's) may refuse it.
u.l.Warn("recvmsg_x unavailable, reading one datagram per syscall", "error", errno)
return u.listenOutSingle(r, flush)
}
if errno != 0 {
if errno != unix.EINTR {
u.l.Error("unexpected udp socket receive error", "error", errno)
}
continue
}
if err := checkMsghdrX(hdrs, names, n); err != nil {
// The batch was read through a layout the kernel no longer writes, so none of it is trustworthy.
u.l.Warn("recvmsg_x returned an unexpected header, reading one datagram per syscall", "error", err)
return u.listenOutSingle(r, flush)
}
for i := 0; i < n; i++ {
addr, ok := sockaddrToAddrPort(&names[i])
if !ok {
continue
}
l := int(hdrs[i].Datalen)
r(addr, bufs[i][:l:l])
}
flush()
}
}
// checkMsghdrX rejects a recvmsg_x result that intact msghdr_x entries can't produce, which is how a change
// to xnu's private struct layout would show up.
func checkMsghdrX(hdrs []msghdrX, names []unix.RawSockaddrInet6, n int) error {
if n > len(hdrs) {
return fmt.Errorf("returned %d datagrams for %d headers", n, len(hdrs))
}
for i := range n {
if hdrs[i].Datalen > MTU {
return fmt.Errorf("datagram %d is %d bytes, past the %d byte buffer", i, hdrs[i].Datalen, MTU)
}
switch {
case names[i].Family == unix.AF_INET && hdrs[i].Namelen == unix.SizeofSockaddrInet4:
case names[i].Family == unix.AF_INET6 && hdrs[i].Namelen == unix.SizeofSockaddrInet6:
default:
return fmt.Errorf("datagram %d has address family %d with length %d", i, names[i].Family, hdrs[i].Namelen)
}
}
return nil
}
// listenOutSingle reads one datagram per recvfrom and flushes after each, for when recvmsg_x is refused or misbehaves.
func (u *StdConn) listenOutSingle(r EncReader, flush func()) error {
buffer := make([]byte, MTU) buffer := make([]byte, MTU)
for { for {
// Just read one packet at a time
n, rua, err := u.ReadFromUDPAddrPort(buffer) n, rua, err := u.ReadFromUDPAddrPort(buffer)
if err != nil { if err != nil {
if errors.Is(err, net.ErrClosed) { if errors.Is(err, net.ErrClosed) {
@@ -310,19 +199,6 @@ func (u *StdConn) listenOutSingle(r EncReader, flush func()) error {
} }
} }
func sockaddrToAddrPort(sa *unix.RawSockaddrInet6) (netip.AddrPort, bool) {
switch sa.Family {
case unix.AF_INET:
sa4 := (*unix.RawSockaddrInet4)(unsafe.Pointer(sa))
port := binary.BigEndian.Uint16((*[2]byte)(unsafe.Pointer(&sa4.Port))[:])
return netip.AddrPortFrom(netip.AddrFrom4(sa4.Addr), port), true
case unix.AF_INET6:
port := binary.BigEndian.Uint16((*[2]byte)(unsafe.Pointer(&sa.Port))[:])
return netip.AddrPortFrom(netip.AddrFrom16(sa.Addr).Unmap(), port), true
}
return netip.AddrPort{}, false
}
func (u *StdConn) SupportsMultipleReaders() bool { func (u *StdConn) SupportsMultipleReaders() bool {
return false return false
} }
-85
View File
@@ -1,85 +0,0 @@
//go:build !e2e_testing
package udp
import (
"log/slog"
"net"
"net/netip"
"sync"
"sync/atomic"
"testing"
)
// BenchmarkListenOut measures the receive path on loopback: senders flood datagrams sized like a full tunnel
// packet, and each op is one datagram delivered to the reader. There are enough senders that the reader, not the
// senders, sets the pace. flushes/pkt is 1 when every receive is its own syscall and falls as receives batch. The
// senders share the machine, so ns/op is only comparable between runs on the same host.
func BenchmarkListenOut(b *testing.B) {
for _, tc := range []struct{ name, listen string }{
{"v4", "127.0.0.1"},
{"v6", "::1"},
} {
b.Run(tc.name, func(b *testing.B) {
c, err := NewListener(slog.New(slog.DiscardHandler), Settings{Listen: netip.AddrPortFrom(netip.MustParseAddr(tc.listen), 0)})
if err != nil {
b.Fatal(err)
}
la, err := c.LocalAddr()
if err != nil {
b.Fatal(err)
}
const size, senders = 1300, 4
txs := make([]net.Conn, senders)
for i := range txs {
if txs[i], err = net.Dial("udp", la.String()); err != nil {
b.Fatal(err)
}
defer txs[i].Close()
}
target := int64(b.N)
var got, flushes atomic.Int64
reached := make(chan struct{})
done := make(chan struct{})
go func() {
_ = c.ListenOut(func(netip.AddrPort, []byte) {
if got.Add(1) == target {
close(reached)
}
}, func() { flushes.Add(1) })
close(done)
}()
// Writes fail with ENOBUFS while the send queue is full; the flood only has to keep the receiver busy.
stop := make(chan struct{})
var sent sync.WaitGroup
for _, tx := range txs {
sent.Go(func() {
payload := make([]byte, size)
for {
select {
case <-stop:
return
default:
_, _ = tx.Write(payload)
}
}
})
}
b.SetBytes(size)
b.ResetTimer()
<-reached
b.StopTimer()
close(stop)
sent.Wait()
if err := c.Close(); err != nil {
b.Fatal(err)
}
<-done
b.ReportMetric(float64(flushes.Load())/float64(got.Load()), "flushes/pkt")
})
}
}
-302
View File
@@ -1,302 +0,0 @@
//go:build !e2e_testing
package udp
import (
"bytes"
"fmt"
"log/slog"
"net"
"net/netip"
"sync"
"syscall"
"testing"
"time"
"unsafe"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sys/unix"
)
// A burst larger than recvBatch arrives whole, with each datagram's own bytes and source address, and
// flush runs once per recvmsg_x batch rather than once per datagram.
func TestListenOutBatchesBurst(t *testing.T) {
for _, tc := range []struct{ name, listen string }{
{"v4", "127.0.0.1"},
{"v6", "::1"},
} {
t.Run(tc.name, func(t *testing.T) {
c, err := NewListener(slog.New(slog.DiscardHandler), Settings{Listen: netip.AddrPortFrom(netip.MustParseAddr(tc.listen), 0)})
require.NoError(t, err)
la, err := c.LocalAddr()
require.NoError(t, err)
tx, err := net.Dial("udp", la.String())
require.NoError(t, err)
defer tx.Close()
txAddr := netip.MustParseAddrPort(tx.LocalAddr().String())
const total = recvBatch*3 + 7
var mu sync.Mutex
var got []string
var froms []netip.AddrPort
flushes := 0
// Queued before the reader starts, so recvmsg_x sees full batches.
for i := 0; i < total; i++ {
_, err := tx.Write([]byte(fmt.Sprintf("pkt-%03d", i)))
require.NoError(t, err)
}
done := make(chan struct{})
go func() {
_ = c.ListenOut(func(addr netip.AddrPort, payload []byte) {
mu.Lock()
got = append(got, string(payload))
froms = append(froms, addr)
mu.Unlock()
}, func() {
mu.Lock()
flushes++
mu.Unlock()
})
close(done)
}()
require.Eventually(t, func() bool {
mu.Lock()
defer mu.Unlock()
return len(got) == total
}, 5*time.Second, 10*time.Millisecond)
require.NoError(t, c.Close())
<-done
for i, p := range got {
assert.Equal(t, fmt.Sprintf("pkt-%03d", i), p)
assert.Equal(t, txAddr, froms[i])
}
assert.Equal(t, (total+recvBatch-1)/recvBatch, flushes)
})
}
}
// msghdrX round-trips through xnu's recvmsg_x on a loopback socket, v4 and v6. The kernel reads Name, Namelen,
// Iov and Iovlen from each entry and writes Namelen, Flags and Datalen back, so a field at the wrong offset or a
// wrong stride between entries shows up as a wrong length, sender, or payload.
// TestListenOutSingleFallback pins the path ListenOut takes when the kernel refuses recvmsg_x.
func TestListenOutSingleFallback(t *testing.T) {
c, err := NewListener(slog.New(slog.DiscardHandler), Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0")})
require.NoError(t, err)
la, err := c.LocalAddr()
require.NoError(t, err)
tx, err := net.Dial("udp", la.String())
require.NoError(t, err)
defer tx.Close()
txAddr := netip.MustParseAddrPort(tx.LocalAddr().String())
const total = 5
for i := 0; i < total; i++ {
_, err := tx.Write([]byte(fmt.Sprintf("pkt-%03d", i)))
require.NoError(t, err)
}
var mu sync.Mutex
var got []string
flushes := 0
done := make(chan struct{})
go func() {
_ = c.(*StdConn).listenOutSingle(func(addr netip.AddrPort, payload []byte) {
assert.Equal(t, txAddr, addr)
mu.Lock()
got = append(got, string(payload))
mu.Unlock()
}, func() {
mu.Lock()
flushes++
mu.Unlock()
})
close(done)
}()
require.Eventually(t, func() bool {
mu.Lock()
defer mu.Unlock()
return len(got) == total
}, 5*time.Second, 10*time.Millisecond)
require.NoError(t, c.Close())
<-done
for i, p := range got {
assert.Equal(t, fmt.Sprintf("pkt-%03d", i), p)
}
assert.Equal(t, total, flushes)
}
func TestCheckMsghdrX(t *testing.T) {
valid := func() ([]msghdrX, []unix.RawSockaddrInet6) {
hdrs := make([]msghdrX, 2)
names := make([]unix.RawSockaddrInet6, 2)
hdrs[0].Namelen, hdrs[0].Datalen = unix.SizeofSockaddrInet4, 1
names[0].Family = unix.AF_INET
hdrs[1].Namelen, hdrs[1].Datalen = unix.SizeofSockaddrInet6, MTU
names[1].Family = unix.AF_INET6
return hdrs, names
}
hdrs, names := valid()
require.NoError(t, checkMsghdrX(hdrs, names, 2))
require.NoError(t, checkMsghdrX(hdrs, names, 0))
for _, tc := range []struct {
name string
n int
mutate func([]msghdrX, []unix.RawSockaddrInet6)
}{
{"count past headers", 3, func([]msghdrX, []unix.RawSockaddrInet6) {}},
{"datalen past buffer", 2, func(h []msghdrX, _ []unix.RawSockaddrInet6) { h[1].Datalen = MTU + 1 }},
{"unknown family", 2, func(_ []msghdrX, a []unix.RawSockaddrInet6) { a[0].Family = unix.AF_UNIX }},
{"family and length disagree", 2, func(h []msghdrX, _ []unix.RawSockaddrInet6) { h[0].Namelen = unix.SizeofSockaddrInet6 }},
} {
t.Run(tc.name, func(t *testing.T) {
hdrs, names := valid()
tc.mutate(hdrs, names)
assert.Error(t, checkMsghdrX(hdrs, names, tc.n))
})
}
}
func TestMsghdrXKernelRoundTrip(t *testing.T) {
for _, v6 := range []bool{false, true} {
t.Run(fmt.Sprintf("v6=%v", v6), func(t *testing.T) {
// Loopback delivery goes through the dlil input thread, so the settle gives all three time to
// queue and come back from one call, which is what exercises the stride.
got := recvmsgXRoundTrip(t, v6, []int{1, 300, 1400}, 4, 50*time.Millisecond)
assert.Equal(t, 3, got, "largest batch from one recvmsg_x call")
})
}
}
// Any burst, of any datagram lengths up to the kernel's maxdgram, into any number of headers, comes back in order
// with every entry's fields intact, and headers past the returned count untouched.
func FuzzMsghdrXKernelRoundTrip(f *testing.F) {
f.Add(false, uint8(3), []byte{0, 1, 1, 44, 5, 120})
f.Add(true, uint8(0), []byte{0, 0, 0x23, 0x29}) // one empty datagram, then one at MTU
f.Add(false, uint8(1), []byte{0x23, 0x2a, 0x24, 0x00, 0}) // past MTU: cut to MTU, no MSG_TRUNC
f.Add(true, uint8(recvBatch-1), bytes.Repeat([]byte{0x05, 0xdc}, recvBatch))
f.Fuzz(func(t *testing.T, v6 bool, spare uint8, sizes []byte) {
// Two bytes per datagram, big-endian, folded into [0, maxdgram].
var lens []int
for i := 0; i+1 < len(sizes) && len(lens) < recvBatch; i += 2 {
lens = append(lens, (int(sizes[i])<<8|int(sizes[i+1]))%(maxDgram+1))
}
if len(lens) == 0 {
return
}
recvmsgXRoundTrip(t, v6, lens, 1+int(spare)%recvBatch, time.Millisecond)
})
}
// maxDgram is darwin's default net.inet.udp.maxdgram, the largest datagram a loopback sender may write.
const maxDgram = 9216
// recvmsgXRoundTrip sends one datagram per entry of lens to a fresh loopback socket, drains them with recvmsg_x
// into nh headers, and checks every field the kernel writes. Neighboring datagrams differ in sender, length and
// fill byte, so no field can pass by reading a neighbor's value. It returns the largest batch one call returned.
func recvmsgXRoundTrip(t *testing.T, v6 bool, lens []int, nh int, settle time.Duration) int {
t.Helper()
network, lo := "udp4", netip.MustParseAddr("127.0.0.1")
if v6 {
network, lo = "udp6", netip.IPv6Loopback()
}
rx, err := net.ListenUDP(network, net.UDPAddrFromAddrPort(netip.AddrPortFrom(lo, 0)))
require.NoError(t, err)
defer rx.Close()
txs := make([]*net.UDPConn, 3)
for i := range txs {
txs[i], err = net.DialUDP(network, nil, rx.LocalAddr().(*net.UDPAddr))
require.NoError(t, err)
defer txs[i].Close()
}
senders := make([]netip.AddrPort, len(lens))
for i, l := range lens {
tx := txs[i%len(txs)]
senders[i] = tx.LocalAddr().(*net.UDPAddr).AddrPort()
_, err := tx.Write(bytes.Repeat([]byte{byte(i + 1)}, l))
require.NoError(t, err)
}
time.Sleep(settle)
// Namelen goes in as the v6 size on both families, so the v4 size coming back proves the kernel wrote it.
const sentinel = 0xa5a5a5a5a5a5a5a5
wantNamelen := uint32(unix.SizeofSockaddrInet4)
if v6 {
wantNamelen = unix.SizeofSockaddrInet6
}
bufs := make([][]byte, nh)
names := make([]unix.RawSockaddrInet6, nh)
iovs := make([]unix.Iovec, nh)
hdrs := make([]msghdrX, nh)
for i := range hdrs {
bufs[i] = make([]byte, MTU)
iovs[i].Base = &bufs[i][0]
iovs[i].SetLen(MTU)
}
// A datagram dropped on loopback would otherwise block rc.Read forever.
require.NoError(t, rx.SetReadDeadline(time.Now().Add(5*time.Second)))
rc, err := rx.SyscallConn()
require.NoError(t, err)
largest := 0
for next := 0; next < len(lens); {
for i := range hdrs {
clear(bufs[i])
hdrs[i] = msghdrX{
Name: (*byte)(unsafe.Pointer(&names[i])),
Namelen: unix.SizeofSockaddrInet6,
Iov: &iovs[i],
Iovlen: 1,
Flags: -1,
Datalen: sentinel,
}
}
var n int
var errno syscall.Errno
require.NoError(t, rc.Read(func(fd uintptr) bool {
r0, _, e := unix.Syscall6(unix.SYS_RECVMSG_X, fd, uintptr(unsafe.Pointer(&hdrs[0])), uintptr(nh), 0, 0, 0)
n, errno = int(r0), e
return e != unix.EAGAIN
}), "received %d of %d datagrams", next, len(lens))
require.Zero(t, errno)
require.Positive(t, n)
require.LessOrEqual(t, n, min(nh, len(lens)-next))
largest = max(largest, n)
require.NoError(t, checkMsghdrX(hdrs[:nh], names[:nh], n), "ListenOut would fall back on this batch")
for i := range n {
d := next + i
h := &hdrs[i]
want := min(lens[d], MTU)
require.EqualValues(t, want, h.Datalen, "datagram %d Datalen", d)
require.Equal(t, wantNamelen, h.Namelen, "datagram %d Namelen", d)
from, ok := sockaddrToAddrPort(&names[i])
require.True(t, ok, "datagram %d family %d", d, names[i].Family)
require.Equal(t, senders[d], from, "datagram %d sender", d)
// Flags goes in as -1 and must come back 0. xnu's list receive (soreceive_m_list) never sets
// MSG_TRUNC, so a datagram past MTU arrives silently cut to MTU, as recvfrom through net.UDPConn does.
require.Zero(t, h.Flags, "datagram %d Flags", d)
require.Equal(t, bytes.Repeat([]byte{byte(d + 1)}, want), bufs[i][:want], "datagram %d payload", d)
if want < MTU {
require.Zero(t, bufs[i][want], "datagram %d wrote past Datalen", d)
}
}
for i := n; i < nh; i++ {
require.EqualValues(t, uint64(sentinel), hdrs[i].Datalen, "spare header %d Datalen", i)
require.EqualValues(t, unix.SizeofSockaddrInet6, hdrs[i].Namelen, "spare header %d Namelen", i)
}
next += n
}
return largest
}