overlay/darwin: read the utun with recvmsg_x

tunQueue.Read now takes up to 64 queued packets with one recvmsg_x through
internal/msgx, each into its own 64KiB slot of a buffer allocated once per
queue, and falls back to the readv drain for good when recvmsg_x is refused
or returns a batch intact msghdr_x entries can't produce, or for one Read on
a transient error. Ported from the pipeline bench build's tunRecvX, minus its
env toggle and stats.

With 64 pending packets on the utun this is the remaining send gain: on an
M2 mini, one stream, pipeline + 64 pending reads 4.9 Gbps with recvmsg_x and
2.6 with the readv loop, the same as this branch before this commit.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ryan HuberandClaude Opus 5.5 committed 2026-09-30 13:27:27 +00:00
1 parent 64bec56404
commit 21a7f1b711
2 files changed
+442 -37

No files matched your search

+304 -26
View File
@@ -5,8 +5,10 @@ package overlay
import (
"bytes"
"os"
"sync/atomic"
"syscall"
"testing"
"time"
"github.com/slackhq/nebula/internal/msgx"
"github.com/slackhq/nebula/test"
@@ -154,28 +156,59 @@ func TestTunWriteBatchOversize(t *testing.T) {
}
}
// TestTunQueueReadDrains pins that Read hands back every queued packet, AF prefix stripped and in
// order, capped at tunReadBatch per call, and doesn't wait for more once the queue is empty.
func TestTunQueueReadDrains(t *testing.T) {
// forEachTunReadPath runs fn once over recvmsg_x and once over the per-packet readv drain.
func forEachTunReadPath(t *testing.T, fn func(t *testing.T, newQ func(*tun) *tunQueue)) {
for _, recvX := range []bool{true, false} {
name := "recvmsg_x"
if !recvX {
name = "readv"
}
t.Run(name, func(t *testing.T) {
fn(t, func(tn *tun) *tunQueue {
q := newTunQueue(tn)
if !recvX {
q.x = nil
}
return q
})
})
}
}
// writeTunPkts writes n test packets of size bytes to the peer end w and returns the packets the tun should hand back
// for them.
func writeTunPkts(t *testing.T, w, n, size int) [][]byte {
t.Helper()
var want [][]byte
for i := range n {
pkt, wire := testTunPkt(i, size+i)
_, err := unix.Write(w, wire)
require.NoError(t, err)
want = append(want, pkt)
}
return want
}
// newReadTun is newSocketpairTun with the tun's receive buffer big enough for every test burst, since an AF_UNIX
// datagram send is bounded by the receiver's buffer, and the writer's send buffer big enough for the largest IP packet.
func newReadTun(t *testing.T) (*tun, int) {
t.Helper()
tn, w := newSocketpairTun(t, 1<<20)
// An AF_UNIX datagram send is bounded by the receiver's buffer, here the tun's end.
require.NoError(t, unix.SetsockoptInt(w, unix.SOL_SOCKET, unix.SO_SNDBUF, 1<<18))
rc, err := tn.f.SyscallConn()
require.NoError(t, err)
require.NoError(t, rc.Control(func(fd uintptr) {
require.NoError(t, unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_RCVBUF, 1<<20))
}))
q := &tunQueue{t: tn, buf: make([]byte, tunReadArena)}
total := tunReadBatch + 3
var want [][]byte
for i := range total {
pkt, wire := testTunPkt(i, 60+i)
_, err := unix.Write(w, wire)
require.NoError(t, err)
want = append(want, pkt)
}
return tn, w
}
// readTunPkts calls q.Read once per entry of sizes, requires each to return that many packets, and returns copies of
// them all.
func readTunPkts(t *testing.T, q *tunQueue, sizes ...int) [][]byte {
t.Helper()
var got [][]byte
for _, size := range []int{tunReadBatch, 3} {
for _, size := range sizes {
pkts, err := q.Read()
require.NoError(t, err)
require.Len(t, pkts, size)
@@ -183,23 +216,268 @@ func TestTunQueueReadDrains(t *testing.T) {
got = append(got, p.Clone().Bytes)
}
}
assert.Equal(t, want, got)
return got
}
// TestTunQueueReadClipsPackets pins that each returned packet's capacity ends at its own bytes, so
// appending to one can't overwrite the next.
// stubTunRecvmsgX swaps tunRecvmsgX for fn until the test ends and counts its calls. fn gets the real msgx.Recv to
// delegate to.
func stubTunRecvmsgX(t *testing.T, fn func(sys func(uintptr, []msgx.Hdr) (int, syscall.Errno), fd uintptr, hdrs []msgx.Hdr) (int, syscall.Errno)) *atomic.Int64 {
t.Helper()
sys := tunRecvmsgX
var calls atomic.Int64
tunRecvmsgX = func(fd uintptr, hdrs []msgx.Hdr) (int, syscall.Errno) {
calls.Add(1)
return fn(sys, fd, hdrs)
}
t.Cleanup(func() { tunRecvmsgX = sys })
return &calls
}
// TestTunQueueReadDrains pins that Read hands back every queued packet, AF prefix stripped and in order, capped at
// tunReadBatch per call, and doesn't wait for more once the queue is empty.
func TestTunQueueReadDrains(t *testing.T) {
forEachTunReadPath(t, func(t *testing.T, newQ func(*tun) *tunQueue) {
tn, w := newReadTun(t)
q := newQ(tn)
want := writeTunPkts(t, w, tunReadBatch+3, 60)
assert.Equal(t, want, readTunPkts(t, q, tunReadBatch, 3))
})
}
// TestTunQueueReadUsesRecvX pins that a burst is read with one recvmsg_x, not one call per packet.
func TestTunQueueReadUsesRecvX(t *testing.T) {
calls := stubTunRecvmsgX(t, func(sys func(uintptr, []msgx.Hdr) (int, syscall.Errno), fd uintptr, hdrs []msgx.Hdr) (int, syscall.Errno) {
return sys(fd, hdrs)
})
tn, w := newReadTun(t)
q := newTunQueue(tn)
want := writeTunPkts(t, w, 10, 60)
assert.Equal(t, want, readTunPkts(t, q, 10))
assert.EqualValues(t, 1, calls.Load())
assert.NotNil(t, q.x)
}
// TestTunQueueReadLargePackets pins that packets up to the largest IP packet come back whole, which on the recvmsg_x
// path means each lands in its own slot and none spills into the next.
func TestTunQueueReadLargePackets(t *testing.T) {
forEachTunReadPath(t, func(t *testing.T, newQ func(*tun) *tunQueue) {
tn, w := newReadTun(t)
q := newQ(tn)
var want [][]byte
for i, size := range []int{65535, 1, 9001, 65535} {
pkt, wire := testTunPkt(i, max(size, 3))
pkt, wire = pkt[:size], wire[:4+size]
_, err := unix.Write(w, wire)
require.NoError(t, err)
want = append(want, pkt)
}
assert.Equal(t, want, readTunPkts(t, q, 4))
})
}
// TestTunQueueReadClipsPackets pins that each returned packet's capacity ends at its own bytes, so appending to one
// can't overwrite the next.
func TestTunQueueReadClipsPackets(t *testing.T) {
tn, w := newSocketpairTun(t, 1<<20)
q := &tunQueue{t: tn, buf: make([]byte, tunReadArena)}
for i := range 2 {
_, wire := testTunPkt(i, 100)
_, err := unix.Write(w, wire)
forEachTunReadPath(t, func(t *testing.T, newQ func(*tun) *tunQueue) {
tn, w := newSocketpairTun(t, 1<<20)
q := newQ(tn)
writeTunPkts(t, w, 2, 100)
pkts, err := q.Read()
require.NoError(t, err)
require.Len(t, pkts, 2)
assert.Equal(t, len(pkts[0].Bytes), cap(pkts[0].Bytes))
assert.Equal(t, len(pkts[1].Bytes), cap(pkts[1].Bytes))
})
}
// TestTunQueueReadDoesNotAllocate pins both read paths to zero allocations per Read once the queue exists.
func TestTunQueueReadDoesNotAllocate(t *testing.T) {
forEachTunReadPath(t, func(t *testing.T, newQ func(*tun) *tunQueue) {
tn, w := newReadTun(t)
q := newQ(tn)
_, wire := testTunPkt(0, 1300)
allocs := testing.AllocsPerRun(1000, func() {
if _, err := unix.Write(w, wire); err != nil {
t.Fatal(err)
}
if pkts, err := q.Read(); err != nil || len(pkts) != 1 {
t.Fatalf("read %d packets: %v", len(pkts), err)
}
})
assert.Zero(t, allocs)
})
}
// TestTunQueueReadXRefused pins that recvmsg_x being refused turns it off for good, and that the same Read still
// hands back the queued packets through readv.
func TestTunQueueReadXRefused(t *testing.T) {
for _, errno := range []syscall.Errno{unix.ENOSYS, unix.EPERM, unix.EOPNOTSUPP, unix.EINVAL} {
t.Run(errno.Error(), func(t *testing.T) {
calls := stubTunRecvmsgX(t, func(func(uintptr, []msgx.Hdr) (int, syscall.Errno), uintptr, []msgx.Hdr) (int, syscall.Errno) {
return 0, errno
})
tn, w := newReadTun(t)
q := newTunQueue(tn)
want := writeTunPkts(t, w, 5, 60)
assert.Equal(t, want, readTunPkts(t, q, 5))
assert.Nil(t, q.x)
assert.EqualValues(t, 1, calls.Load())
want = writeTunPkts(t, w, 2, 60)
assert.Equal(t, want, readTunPkts(t, q, 2))
assert.EqualValues(t, 1, calls.Load(), "recvmsg_x was tried again after it was refused")
})
}
}
// TestTunQueueReadXTransientError pins that an error from recvmsg_x that doesn't mean it is refused leaves this Read
// to readv but keeps recvmsg_x on for the next.
func TestTunQueueReadXTransientError(t *testing.T) {
var failed atomic.Bool
calls := stubTunRecvmsgX(t, func(sys func(uintptr, []msgx.Hdr) (int, syscall.Errno), fd uintptr, hdrs []msgx.Hdr) (int, syscall.Errno) {
if !failed.Swap(true) {
return 0, unix.ENOBUFS
}
return sys(fd, hdrs)
})
tn, w := newReadTun(t)
q := newTunQueue(tn)
want := writeTunPkts(t, w, 3, 60)
assert.Equal(t, want, readTunPkts(t, q, 3))
assert.NotNil(t, q.x)
assert.EqualValues(t, 1, calls.Load())
want = writeTunPkts(t, w, 2, 60)
assert.Equal(t, want, readTunPkts(t, q, 2))
assert.EqualValues(t, 2, calls.Load(), "recvmsg_x wasn't tried again after a transient error")
}
// TestTunQueueReadXParks pins that EAGAIN from recvmsg_x parks the Read until the utun is readable, rather than
// falling back or returning an empty batch.
func TestTunQueueReadXParks(t *testing.T) {
var failed atomic.Bool
calls := stubTunRecvmsgX(t, func(sys func(uintptr, []msgx.Hdr) (int, syscall.Errno), fd uintptr, hdrs []msgx.Hdr) (int, syscall.Errno) {
if !failed.Swap(true) {
return 0, unix.EAGAIN
}
return sys(fd, hdrs)
})
tn, w := newReadTun(t)
q := newTunQueue(tn)
// A real EAGAIN means nothing is queued, so the packet arrives only once Read has parked. One packet, so the
// wakeup can't split it across Reads.
done := make(chan struct{})
go func() {
defer close(done)
time.Sleep(50 * time.Millisecond)
_, wire := testTunPkt(0, 60)
_, _ = unix.Write(w, wire)
}()
t.Cleanup(func() { <-done })
pkt, _ := testTunPkt(0, 60)
assert.Equal(t, [][]byte{pkt}, readTunPkts(t, q, 1))
assert.NotNil(t, q.x)
// A spurious wakeup costs another EAGAIN, so only a floor.
assert.GreaterOrEqual(t, calls.Load(), int64(2))
}
// TestTunQueueReadXBadBatch pins that a recvmsg_x batch no intact header could produce turns recvmsg_x off for good
// and hands the Read to readv. The stub reports a bad batch without taking any packets, so readv then reads them all.
func TestTunQueueReadXBadBatch(t *testing.T) {
v4 := [4]byte{3: syscall.AF_INET}
for _, tc := range []struct {
name string
mess func(x *tunRecvX) int
}{
{"too many", func(x *tunRecvX) int {
for i := range x.hdrs {
x.hdrs[i].Datalen, x.heads[i] = 64, v4
}
return len(x.hdrs) + 1
}},
{"past the buffer", func(x *tunRecvX) int {
x.hdrs[0].Datalen, x.heads[0] = 4+tunRecvXSlot+1, v4
return 1
}},
{"bad AF prefix", func(x *tunRecvX) int {
x.hdrs[0].Datalen, x.heads[0] = 64, [4]byte{syscall.AF_INET}
return 1
}},
} {
t.Run(tc.name, func(t *testing.T) {
var q *tunQueue
stubTunRecvmsgX(t, func(func(uintptr, []msgx.Hdr) (int, syscall.Errno), uintptr, []msgx.Hdr) (int, syscall.Errno) {
return tc.mess(q.x), 0
})
tn, w := newReadTun(t)
q = newTunQueue(tn)
want := writeTunPkts(t, w, 4, 60)
assert.Equal(t, want, readTunPkts(t, q, 4))
assert.Nil(t, q.x)
})
}
}
// TestTunRecvXCheck pins which recvmsg_x batches tunRecvXCheck accepts.
func TestTunRecvXCheck(t *testing.T) {
const slot = 1000
v4, v6 := [4]byte{3: syscall.AF_INET}, [4]byte{3: syscall.AF_INET6}
good := func() ([]msgx.Hdr, [][4]byte) {
hdrs := make([]msgx.Hdr, 4)
heads := make([][4]byte, 4)
for i := range hdrs {
hdrs[i].Datalen = uint64(4 + 20 + i)
heads[i] = v4
if i%2 == 1 {
heads[i] = v6
}
}
return hdrs, heads
}
for _, tc := range []struct {
name string
n int
mess func(hdrs []msgx.Hdr, heads [][4]byte)
ok bool
}{
{"all", 4, nil, true},
{"none", 0, nil, true},
{"prefix", 2, nil, true},
{"garbage past n is ignored", 2, func(h []msgx.Hdr, hd [][4]byte) { h[3].Datalen, hd[3] = 1<<40, [4]byte{9} }, true},
{"just the AF prefix", 1, func(h []msgx.Hdr, _ [][4]byte) { h[0].Datalen = 4 }, true},
{"fills the slot", 1, func(h []msgx.Hdr, _ [][4]byte) { h[0].Datalen = 4 + slot }, true},
{"more than headers", 5, nil, false},
{"negative", -1, nil, false},
{"past the slot", 2, func(h []msgx.Hdr, _ [][4]byte) { h[1].Datalen = 4 + slot + 1 }, false},
{"shorter than the AF prefix", 1, func(h []msgx.Hdr, _ [][4]byte) { h[0].Datalen = 3 }, false},
{"truncated", 3, func(h []msgx.Hdr, _ [][4]byte) { h[2].Flags = unix.MSG_TRUNC }, false},
{"zero AF prefix", 1, func(_ []msgx.Hdr, hd [][4]byte) { hd[0] = [4]byte{} }, false},
{"host order AF prefix", 1, func(_ []msgx.Hdr, hd [][4]byte) { hd[0] = [4]byte{syscall.AF_INET} }, false},
{"other AF", 2, func(_ []msgx.Hdr, hd [][4]byte) { hd[1] = [4]byte{3: syscall.AF_UNIX} }, false},
} {
t.Run(tc.name, func(t *testing.T) {
hdrs, heads := good()
if tc.mess != nil {
tc.mess(hdrs, heads)
}
err := tunRecvXCheck(tc.n, hdrs, heads, slot)
if tc.ok {
assert.NoError(t, err)
} else {
assert.Error(t, err)
}
})
}
}
// TestTunRecvXRefused pins which recvmsg_x errnos turn it off for good.
func TestTunRecvXRefused(t *testing.T) {
for _, errno := range []syscall.Errno{unix.ENOSYS, unix.EPERM, unix.EOPNOTSUPP, unix.EINVAL, unix.EMSGSIZE} {
assert.True(t, tunRecvXRefused(errno), "%v", errno)
}
for _, errno := range []syscall.Errno{0, unix.EAGAIN, unix.EINTR, unix.ENOBUFS, unix.ENOMEM, unix.EBADF, unix.ENOTCONN} {
assert.False(t, tunRecvXRefused(errno), "%v", errno)
}
pkts, err := q.Read()
require.NoError(t, err)
require.Len(t, pkts, 2)
assert.Equal(t, len(pkts[0].Bytes), cap(pkts[0].Bytes))
}
// TestTunRaisePendingPackets pins that the pending limit is set at SYSPROTO_CONTROL level after the receive buffer is