Compare commits

..
Author SHA1 Message Date
Caleb JasikandClaude Opus 5.5 10e1924489 udp/darwin: batch sends with sendmsg_x
WriteBatch hands up to 64 datagrams to one sendmsg_x call instead of
one sendto each. The socket is unconnected, so xnu still runs every
message through sendit, per-send MAC checks included; the saving is the
syscall crossings.

xnu's per-message loop stops at the first message that fails. For
EWOULDBLOCK, ENOBUFS and EMSGSIZE it reports how many went out, or the
errno alone when none did; for any other error it returns only the
errno, with the messages before it already sent. So each call carries
one run of datagrams to a single destination, and nothing is resent
after the kernel may have taken it. A datagram that fails on its own
account (a full interface queue, an oversize packet) goes to sendto,
which delivers or reports it, so it costs only itself, as without
batching; a refused destination costs its run. ENOSYS/EOPNOTSUPP, or
EPERM/EINVAL on a datagram sendto then delivers, switch the socket
back to sendto for good.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-24 16:10:26 -05:00
Caleb JasikandClaude Opus 5.5 4d191f805f overlay/darwin: drain several tun packets per read wakeup
The darwin tun returned one packet per Read, and listenIn flushes its
UDP send batch after every Read, so each outbound packet went to the
underlay as its own batch. tunQueue.Read now keeps calling readv until
the utun would block, up to tunReadBatch packets, so a burst reaches the
send batch together.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-24 12:44:00 -05:00
Caleb JasikandClaude Opus 5.5 61947ff1c2 overlay/darwin: batch tun writes with sendmsg_x
Passthrough.Flush hands its whole lane to a new tio.BatchWriter in one
WriteBatch call when the queue implements it, and NewSingleQueue
exposes WriteBatch when its source does. On darwin, tun.WriteBatch
writes up to 64 packets per sendmsg_x on the utun control socket,
which is a connected datagram socket, so xnu takes the batched
sendit_x path rather than one sendit per message. utun registers no
send_list handler, so the kernel still passes each packet to utun on
its own: this saves syscalls and fd locking, not per-packet kernel work.

Error handling follows what sendmsg_x reports:

- A short count with no error (a packet past the send buffer
  mid-call, or a later send failing with ENOBUFS): the remainder goes
  out in the next sendmsg_x.
- EMSGSIZE: only returned when the first packet is too large, before
  the kernel takes any, so that packet alone goes through Write, which
  reports it, and the rest are batched again.
- ENOSYS, EPERM, EOPNOTSUPP: sendmsg_x is private API, so WriteBatch
  logs once and falls back to writev per packet for good.
- Anything else: sendmsg_x returns the error without the count it
  took, so the call is reported rather than retried, which could
  deliver packets twice.

One loss mode has no userspace fix. sendit_x counts each packet as
sent once it is copied into an mbuf; if a later allocation fails, it
frees the copied prefix unsent, returns its length, and sendmsg_x
drops the ENOBUFS because the count is nonzero. That looks exactly
like a real short count, so under mbuf exhaustion WriteBatch silently
drops packets that writev would have delivered.

The iOS tun path (tun_ios.go) is unchanged.

Inside a macOS Network Extension, download goes from 640 to 832 Mbps
against the recvmsg_x change alone in three paired rounds, at 15% less
CPU per Mbps; the likely cause is that its utun charges more per write
than a raw one. Raw dnclient already runs download near line rate,
and there the CPU difference is within noise.

TestTunWriteBatch* round-trip through a connected AF_UNIX datagram
socketpair, which takes sendmsg_x the same way, over both the batched
and the fallback path: order, AF prefixes, skipped invalid packets,
and an oversize packet mid-call and first-in-call. Dropping Namelen
from msghdrX fails them. Mutation testing with gremlins leaves only
equivalent survivors in WriteBatch, plus which error is reported first
when several fail.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-24 10:04:03 -05:00
8 changed files with 852 additions and 20 deletions

No files matched your search

+62
View File
@@ -435,3 +435,65 @@ func testPP(pkt []byte) *firewall.ParsedPacket {
} }
return pp return pp
} }
// fakeTunBatchWriter records each WriteBatch call and each plain Write separately.
type fakeTunBatchWriter struct {
writes [][]byte
batches [][][]byte
}
func (w *fakeTunBatchWriter) Write(p []byte) (int, error) {
w.writes = append(w.writes, bytes.Clone(p))
return len(p), nil
}
func (w *fakeTunBatchWriter) WriteBatch(pkts [][]byte) error {
var b [][]byte
for _, p := range pkts {
b = append(b, bytes.Clone(p))
}
w.batches = append(w.batches, b)
return nil
}
// TestPassthroughUsesWriteBatch pins that a writer implementing tio.BatchWriter gets the whole
// passthrough lane in one WriteBatch per Flush, in order, and no plain Writes; an empty Flush
// makes no call.
func TestPassthroughUsesWriteBatch(t *testing.T) {
w := &fakeTunBatchWriter{}
m := newTestMultiCoalescer(t, w)
k := &keySeq{epoch: 1}
var want [][]byte
for i := range 5 {
icmp := make([]byte, 28)
icmp[0] = 0x45
icmp[3] = 28
icmp[9] = 1
icmp[20] = byte(i)
want = append(want, icmp)
if err := m.Commit(icmp, k.next(), testPP(icmp)); err != nil {
t.Fatal(err)
}
}
if err := m.Flush(); err != nil {
t.Fatal(err)
}
if err := m.Flush(); err != nil {
t.Fatal(err)
}
if len(w.writes) != 0 {
t.Fatalf("got %d plain Writes, want 0", len(w.writes))
}
if len(w.batches) != 1 {
t.Fatalf("got %d WriteBatch calls, want 1", len(w.batches))
}
for i, p := range w.batches[0] {
if !bytes.Equal(p, want[i]) {
t.Fatalf("packet %d: got % x, want % x", i, p, want[i])
}
}
if len(w.batches[0]) != len(want) {
t.Fatalf("got %d packets, want %d", len(w.batches[0]), len(want))
}
}
+13 -2
View File
@@ -2,18 +2,23 @@ package batch
import ( import (
"io" "io"
"github.com/slackhq/nebula/overlay/tio"
) )
// Passthrough is MultiCoalescer's verbatim lane: no batching, packets are written at Flush in the // Passthrough is MultiCoalescer's verbatim lane: no coalescing, packets are written at Flush in the
// order enqueued. // order enqueued, in one WriteBatch when out implements tio.BatchWriter.
type Passthrough struct { type Passthrough struct {
out io.Writer out io.Writer
bw tio.BatchWriter // nil: Flush writes one packet per Write
slots [][]byte slots [][]byte
} }
func NewPassthrough(w io.Writer) *Passthrough { func NewPassthrough(w io.Writer) *Passthrough {
bw, _ := w.(tio.BatchWriter)
return &Passthrough{ return &Passthrough{
out: w, out: w,
bw: bw,
slots: make([][]byte, 0, 128), slots: make([][]byte, 0, 128),
} }
} }
@@ -26,12 +31,18 @@ func (p *Passthrough) enqueue(pkt []byte) error {
func (p *Passthrough) Flush() error { func (p *Passthrough) Flush() error {
var firstErr error var firstErr error
if p.bw != nil {
if len(p.slots) > 0 {
firstErr = p.bw.WriteBatch(p.slots)
}
} else {
for _, s := range p.slots { for _, s := range p.slots {
_, err := p.out.Write(s) _, err := p.out.Write(s)
if err != nil && firstErr == nil { if err != nil && firstErr == nil {
firstErr = err firstErr = err
} }
} }
}
clear(p.slots) clear(p.slots)
p.slots = p.slots[:0] p.slots = p.slots[:0]
return firstErr return firstErr
+17 -2
View File
@@ -15,9 +15,13 @@ type singleQueue struct {
// NewSingleQueue wraps a one-datagram-per-Read ReadWriteCloser (a legacy tun device) into a Queue. // NewSingleQueue wraps a one-datagram-per-Read ReadWriteCloser (a legacy tun device) into a Queue.
// bufSize is the per-queue read scratch size and must be at least the largest datagram the source can return. // bufSize is the per-queue read scratch size and must be at least the largest datagram the source can return.
// Close closes rwc. // Close closes rwc. The Queue implements BatchWriter when rwc does.
func NewSingleQueue(rwc io.ReadWriteCloser, bufSize int) Queue { func NewSingleQueue(rwc io.ReadWriteCloser, bufSize int) Queue {
return &singleQueue{rw: rwc, closer: rwc, buf: make([]byte, bufSize)} q := &singleQueue{rw: rwc, closer: rwc, buf: make([]byte, bufSize)}
if bw, ok := rwc.(BatchWriter); ok {
return &batchSingleQueue{singleQueue: q, bw: bw}
}
return q
} }
// NewSingleQueueNoClose is NewSingleQueue for a source owned by someone else, // NewSingleQueueNoClose is NewSingleQueue for a source owned by someone else,
@@ -47,3 +51,14 @@ func (q *singleQueue) Close() error {
} }
return q.closer.Close() return q.closer.Close()
} }
// batchSingleQueue is a singleQueue whose source also implements BatchWriter.
// It is a separate type so a plain singleQueue never advertises WriteBatch.
type batchSingleQueue struct {
*singleQueue
bw BatchWriter
}
func (q *batchSingleQueue) WriteBatch(pkts [][]byte) error {
return q.bw.WriteBatch(pkts)
}
+11
View File
@@ -145,3 +145,14 @@ func SupportsGSO(w io.Writer, want GSOProto) (GSOWriter, bool) {
return gw, false return gw, false
} }
} }
// BatchWriter is implemented by Queues that can emit several already-segmented packets in one call.
// Backends without a batched write do not implement it and callers loop over Write.
//
// pkts are borrowed: WriteBatch must not retain them past its return.
// A returned error doesn't say which packets were delivered, so callers must not retry the batch.
// Safe for concurrent use, like Write.
type BatchWriter interface {
io.Writer
WriteBatch(pkts [][]byte) error
}
+244 -9
View File
@@ -8,6 +8,7 @@ import (
"log/slog" "log/slog"
"net/netip" "net/netip"
"os" "os"
"sync"
"sync/atomic" "sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -33,6 +34,35 @@ type tun struct {
// and routes for it. NEPacketTunnelProvider on darwin does this. // and routes for it. NEPacketTunnelProvider on darwin does this.
hostOwned bool hostOwned bool
l *slog.Logger l *slog.Logger
// noSendmsgX is set once sendmsg_x is refused; WriteBatch then writes one packet at a time.
noSendmsgX atomic.Bool
batchMu sync.Mutex
batch tunBatch
}
// tunWriteBatch is how many packets one sendmsg_x call may carry.
const tunWriteBatch = 64
// tunBatch is WriteBatch's scratch, guarded by tun.batchMu. The kernel reads every entry in place.
type tunBatch struct {
heads [tunWriteBatch][4]byte
iovs [tunWriteBatch][2]unix.Iovec
hdrs [tunWriteBatch]msghdrX
pkts [tunWriteBatch][]byte
}
// msghdrX mirrors xnu's struct msghdr_x (bsd/sys/socket_private.h).
// Keep in sync with msghdrX in udp/udp_darwin.go.
type msghdrX struct {
Name *byte
Namelen uint32
Iov *unix.Iovec
Iovlen int32
Control *byte
Controllen uint32
Flags int32
Datalen uint64
} }
type ifReq struct { type ifReq struct {
@@ -607,6 +637,74 @@ func (t *tun) Read(to []byte) (int, error) {
return n - 4, nil return n - 4, nil
} }
// tunReadBatch is the most packets one tunQueue.Read returns.
const tunReadBatch = 64
// tunReadArena is tunQueue's receive buffer. Draining stops once less than defaultBatchBufSize of it
// is left, so every readv has room for the largest packet utun can return at any device MTU.
const tunReadArena = 4 * defaultBatchBufSize
// tunQueue is the darwin tun's Queue. Read drains every packet already queued on the utun, up to
// tunReadBatch, so the caller encrypts and sends them as one batch rather than one per wakeup.
type tunQueue struct {
t *tun
buf []byte
pkts [tunReadBatch]tio.Packet
}
// Read waits for the utun to become readable, then reads packets until it would block. An error after
// at least one packet is dropped in favor of returning those packets; a persistent one recurs on the
// next Read.
func (q *tunQueue) Read() ([]tio.Packet, error) {
rc, err := q.t.f.SyscallConn()
if err != nil {
return nil, err
}
var head [4]byte
n, off := 0, 0
var callErr error
err = rc.Read(func(fd uintptr) bool {
for n < tunReadBatch && len(q.buf)-off >= defaultBatchBufSize {
iovecs := [2]unix.Iovec{
{Base: &head[0], Len: 4},
{Base: &q.buf[off], Len: uint64(len(q.buf) - off)},
}
l, e := tunReadv(int(fd), iovecs[:])
if e != nil {
if errno, ok := e.(syscall.Errno); ok && errno.Temporary() {
// Park on the poller only while there is nothing to hand back.
return n > 0
}
callErr = e
return true
}
if l < 4 {
// A datagram too short to carry the AF prefix, or end of file; stop rather than spin on it.
return true
}
end := off + l - 4
q.pkts[n] = tio.Packet{Bytes: q.buf[off:end:end]}
n++
off = end
}
return true
})
if n > 0 {
return q.pkts[:n], nil
}
if err != nil {
return nil, err
}
return nil, callErr
}
func (q *tunQueue) Write(p []byte) (int, error) { return q.t.Write(p) }
func (q *tunQueue) WriteBatch(pkts [][]byte) error { return q.t.WriteBatch(pkts) }
func (q *tunQueue) Close() error { return q.t.Close() }
// Write pushes one IP packet onto the utun device. Safe for concurrent use: // Write pushes one IP packet onto the utun device. Safe for concurrent use:
// the AF prefix and iovecs are per-call stack state, and the fd write itself // the AF prefix and iovecs are per-call stack state, and the fd write itself
// serializes on the runtime's fd mutex (see the Queue contract in tio.go). // serializes on the runtime's fd mutex (see the Queue contract in tio.go).
@@ -615,16 +713,12 @@ func (t *tun) Write(from []byte) (int, error) {
return 0, syscall.EIO return 0, syscall.EIO
} }
ipVer := from[0] >> 4
var head [4]byte var head [4]byte
switch ipVer { af, err := tunAF(from)
case 4: if err != nil {
head[3] = syscall.AF_INET return 0, err
case 6:
head[3] = syscall.AF_INET6
default:
return 0, fmt.Errorf("unable to determine IP version from packet")
} }
head[3] = af
// Grab rc as a local so the compiler can devirtualize the call and keep the closure on the stack. // Grab rc as a local so the compiler can devirtualize the call and keep the closure on the stack.
rc, err := t.f.SyscallConn() rc, err := t.f.SyscallConn()
@@ -657,6 +751,147 @@ func (t *tun) Write(from []byte) (int, error) {
return n - 4, nil return n - 4, nil
} }
// tunAF returns the utun address-family prefix byte for an IP packet.
func tunAF(pkt []byte) (byte, error) {
switch pkt[0] >> 4 {
case 4:
return syscall.AF_INET, nil
case 6:
return syscall.AF_INET6, nil
default:
return 0, fmt.Errorf("unable to determine IP version from packet")
}
}
// WriteBatch writes pkts to the utun device with sendmsg_x, xnu's private batched sendmsg, up to
// tunWriteBatch packets per syscall. The kernel still hands each packet to utun on its own, so this
// saves syscalls, not per-packet kernel work. Safe for concurrent use.
//
// An empty packet or one with no IP version is skipped and reported, as Write would. Any other
// error from sendmsg_x means the kernel took an unknown prefix of that call and dropped the rest, so
// it is reported rather than retried, which could deliver packets twice.
//
// Under mbuf exhaustion, sendmsg_x silently drops packets that writev would have delivered: the
// kernel counts each packet as sent once it is copied in, and when a later allocation fails it
// frees that prefix unsent and returns its length as a short count, which is indistinguishable
// from a real one.
func (t *tun) WriteBatch(pkts [][]byte) error {
if t.noSendmsgX.Load() {
return t.writeEach(pkts)
}
t.batchMu.Lock()
defer t.batchMu.Unlock()
b := &t.batch
var firstErr error
for len(pkts) > 0 {
n := 0
for len(pkts) > 0 && n < tunWriteBatch {
p := pkts[0]
pkts = pkts[1:]
var af byte
err := error(syscall.EIO)
if len(p) > 0 {
af, err = tunAF(p)
}
if err != nil {
if firstErr == nil {
firstErr = err
}
continue
}
b.heads[n] = [4]byte{3: af}
b.iovs[n] = [2]unix.Iovec{
{Base: &b.heads[n][0], Len: 4},
{Base: &p[0], Len: uint64(len(p))},
}
b.hdrs[n] = msghdrX{Iov: &b.iovs[n][0], Iovlen: 2}
b.pkts[n] = p
n++
}
if n == 0 {
continue
}
for off := 0; off < n; {
sent, err := t.sendmsgX(off, n)
switch {
case err == nil && sent > 0:
// The kernel stops early without an error on a packet larger than the socket's send
// buffer, or when sending a later packet fails with ENOBUFS; resend the remainder.
off += sent
case err == nil:
if werr := t.writeEach(b.pkts[off:n]); werr != nil && firstErr == nil {
firstErr = werr
}
off = n
case err == unix.EMSGSIZE:
// Only the first packet being larger than the socket's send buffer fails the whole call,
// before the kernel takes any; Write reports that one and the rest go out in the next call.
if werr := t.writeEach(b.pkts[off : off+1]); werr != nil && firstErr == nil {
firstErr = werr
}
off++
case err == unix.ENOSYS || err == unix.EPERM || err == unix.EOPNOTSUPP:
// sendmsg_x is private API: fall back to writev if a kernel or sandbox refuses it.
if t.noSendmsgX.CompareAndSwap(false, true) {
t.l.Warn("sendmsg_x unavailable on the tun device, writing one packet per syscall", "error", err)
}
if werr := t.writeEach(b.pkts[off:n]); werr != nil && firstErr == nil {
firstErr = werr
}
clear(b.iovs[:n])
clear(b.pkts[:n])
if werr := t.writeEach(pkts); werr != nil && firstErr == nil {
firstErr = werr
}
return firstErr
default:
if firstErr == nil {
firstErr = err
}
off = n
}
}
clear(b.iovs[:n])
clear(b.pkts[:n])
}
return firstErr
}
// sendmsgX hands entries off through n-1 of t.batch to sendmsg_x and returns how many the kernel took.
func (t *tun) sendmsgX(off, n int) (int, error) {
rc, err := t.f.SyscallConn()
if err != nil {
return 0, err
}
var sent uintptr
var errno syscall.Errno
err = rc.Write(func(fd uintptr) bool {
sent, _, errno = unix.Syscall6(unix.SYS_SENDMSG_X, fd, uintptr(unsafe.Pointer(&t.batch.hdrs[off])), uintptr(n-off), 0, 0, 0)
// sendmsg_x reports EAGAIN only when it took nothing; a partial batch returns its count.
return !errno.Temporary()
})
if err != nil {
return 0, err
}
if errno != 0 {
return 0, errno
}
return int(sent), nil
}
// writeEach writes pkts one at a time with Write.
func (t *tun) writeEach(pkts [][]byte) error {
var firstErr error
for _, p := range pkts {
if _, err := t.Write(p); err != nil && firstErr == nil {
firstErr = err
}
}
return firstErr
}
func (t *tun) Networks() []netip.Prefix { func (t *tun) Networks() []netip.Prefix {
return t.vpnNetworks return t.vpnNetworks
} }
@@ -666,5 +901,5 @@ func (t *tun) Name() string {
} }
func (t *tun) Queues(int) ([]tio.Queue, error) { func (t *tun) Queues(int) ([]tio.Queue, error) {
return []tio.Queue{tio.NewSingleQueue(t, defaultBatchBufSize)}, nil return []tio.Queue{&tunQueue{t: t, buf: make([]byte, tunReadArena)}}, nil
} }
+201
View File
@@ -0,0 +1,201 @@
//go:build !ios && !e2e_testing
package overlay
import (
"bytes"
"os"
"syscall"
"testing"
"github.com/slackhq/nebula/test"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/sys/unix"
)
// newSocketpairTun returns a tun writing to one end of a connected AF_UNIX datagram socketpair, which
// takes sendmsg_x the way a utun control socket does, and the other end's fd for reading back.
// sndbuf is the writer's send buffer, which caps the largest datagram it accepts.
func newSocketpairTun(t *testing.T, sndbuf int) (*tun, int) {
t.Helper()
fds, err := unix.Socketpair(unix.AF_UNIX, unix.SOCK_DGRAM, 0)
require.NoError(t, err)
require.NoError(t, unix.SetsockoptInt(fds[0], unix.SOL_SOCKET, unix.SO_SNDBUF, sndbuf))
require.NoError(t, unix.SetsockoptInt(fds[1], unix.SOL_SOCKET, unix.SO_RCVBUF, 1<<20))
// A dropped datagram fails the read-back instead of hanging it.
require.NoError(t, unix.SetsockoptTimeval(fds[1], unix.SOL_SOCKET, unix.SO_RCVTIMEO, &unix.Timeval{Sec: 2}))
require.NoError(t, unix.SetNonblock(fds[0], true))
f := os.NewFile(uintptr(fds[0]), "socketpair")
t.Cleanup(func() {
_ = f.Close()
_ = unix.Close(fds[1])
})
return &tun{f: f, l: test.NewLogger()}, fds[1]
}
// testTunPkt returns packet i of size bytes, alternating IPv4 and IPv6, and the datagram the utun
// device should receive for it.
func testTunPkt(i, size int) (pkt, wire []byte) {
pkt = make([]byte, size)
pkt[0] = 0x45
af := byte(syscall.AF_INET)
if i%2 == 1 {
pkt[0] = 0x60
af = syscall.AF_INET6
}
pkt[1] = byte(i)
pkt[2] = byte(i >> 8)
return pkt, append([]byte{0, 0, 0, af}, pkt...)
}
// readTunBack asserts r receives exactly want, in order.
func readTunBack(t *testing.T, r int, want [][]byte) {
t.Helper()
buf := make([]byte, 4096)
for i, w := range want {
n, _, err := unix.Recvfrom(r, buf, 0)
require.NoError(t, err, "datagram %d", i)
if !bytes.Equal(w, buf[:n]) {
t.Fatalf("datagram %d: got % x, want % x", i, buf[:n], w)
}
}
_, _, err := unix.Recvfrom(r, buf, unix.MSG_DONTWAIT)
assert.ErrorIs(t, err, unix.EAGAIN, "extra datagrams delivered")
}
// forEachTunWritePath runs fn once over sendmsg_x and once over the per-packet writev fallback.
func forEachTunWritePath(t *testing.T, fn func(t *testing.T, fallback bool)) {
for _, fallback := range []bool{false, true} {
name := "sendmsg_x"
if fallback {
name = "writev"
}
t.Run(name, func(t *testing.T) { fn(t, fallback) })
}
}
// TestTunWriteBatch pins that WriteBatch delivers every packet whole, in order and behind the right
// utun AF prefix, across several sendmsg_x calls, and returns nil.
func TestTunWriteBatch(t *testing.T) {
forEachTunWritePath(t, func(t *testing.T, fallback bool) {
tn, r := newSocketpairTun(t, 1<<20)
tn.noSendmsgX.Store(fallback)
var pkts, want [][]byte
for i := range 3*tunWriteBatch + 5 {
p, w := testTunPkt(i, 60+i)
pkts = append(pkts, p)
want = append(want, w)
}
require.NoError(t, tn.WriteBatch(pkts))
assert.Equal(t, fallback, tn.noSendmsgX.Load())
readTunBack(t, r, want)
})
}
// TestTunWriteBatchSkipsInvalid pins that an empty packet or one with no IP version is skipped
// without shifting the packets after it, and that the first such error is the one reported.
func TestTunWriteBatchSkipsInvalid(t *testing.T) {
forEachTunWritePath(t, func(t *testing.T, fallback bool) {
tn, r := newSocketpairTun(t, 1<<20)
tn.noSendmsgX.Store(fallback)
var pkts, want [][]byte
for i := range 20 {
p, w := testTunPkt(i, 60)
pkts = append(pkts, p)
want = append(want, w)
}
pkts = append(pkts[:10], append([][]byte{{0x10, 1, 2}, {}}, pkts[10:]...)...)
err := tn.WriteBatch(pkts)
assert.ErrorContains(t, err, "IP version")
readTunBack(t, r, want)
})
}
// TestTunWriteBatchOversize pins that a packet larger than the socket's send buffer drops only
// itself and is reported, whether sendmsg_x reaches it mid-call (a short count) or first (EMSGSIZE
// for the whole call). The send buffer is 1024 bytes and the oversize packet 2000.
func TestTunWriteBatchOversize(t *testing.T) {
for _, tc := range []struct {
name string
at int
}{
{"mid-call", tunWriteBatch + 6},
{"first-in-call", tunWriteBatch},
} {
t.Run(tc.name, func(t *testing.T) {
forEachTunWritePath(t, func(t *testing.T, fallback bool) {
tn, r := newSocketpairTun(t, 1024)
tn.noSendmsgX.Store(fallback)
var pkts, want [][]byte
for i := range 2*tunWriteBatch + 10 {
size := 100
if i == tc.at {
size = 2000
}
p, w := testTunPkt(i, size)
pkts = append(pkts, p)
if i != tc.at {
want = append(want, w)
}
}
err := tn.WriteBatch(pkts)
assert.ErrorIs(t, err, unix.EMSGSIZE)
assert.Equal(t, fallback, tn.noSendmsgX.Load())
readTunBack(t, r, want)
})
})
}
}
// TestTunQueueReadDrains pins that Read hands back every queued packet, AF prefix stripped and in
// order, capped at tunReadBatch per call, and doesn't wait for more once the queue is empty.
func TestTunQueueReadDrains(t *testing.T) {
tn, w := newSocketpairTun(t, 1<<20)
// An AF_UNIX datagram send is bounded by the receiver's buffer, here the tun's end.
rc, err := tn.f.SyscallConn()
require.NoError(t, err)
require.NoError(t, rc.Control(func(fd uintptr) {
require.NoError(t, unix.SetsockoptInt(int(fd), unix.SOL_SOCKET, unix.SO_RCVBUF, 1<<20))
}))
q := &tunQueue{t: tn, buf: make([]byte, tunReadArena)}
total := tunReadBatch + 3
var want [][]byte
for i := range total {
pkt, wire := testTunPkt(i, 60+i)
_, err := unix.Write(w, wire)
require.NoError(t, err)
want = append(want, pkt)
}
var got [][]byte
for _, size := range []int{tunReadBatch, 3} {
pkts, err := q.Read()
require.NoError(t, err)
require.Len(t, pkts, size)
for _, p := range pkts {
got = append(got, p.Clone().Bytes)
}
}
assert.Equal(t, want, got)
}
// TestTunQueueReadClipsPackets pins that each returned packet's capacity ends at its own bytes, so
// appending to one can't overwrite the next.
func TestTunQueueReadClipsPackets(t *testing.T) {
tn, w := newSocketpairTun(t, 1<<20)
q := &tunQueue{t: tn, buf: make([]byte, tunReadArena)}
for i := range 2 {
_, wire := testTunPkt(i, 100)
_, err := unix.Write(w, wire)
require.NoError(t, err)
}
pkts, err := q.Read()
require.NoError(t, err)
require.Len(t, pkts, 2)
assert.Equal(t, len(pkts[0].Bytes), cap(pkts[0].Bytes))
}
+154 -7
View File
@@ -10,6 +10,7 @@ import (
"log/slog" "log/slog"
"net" "net"
"net/netip" "net/netip"
"sync/atomic"
"syscall" "syscall"
"unsafe" "unsafe"
@@ -22,6 +23,9 @@ type StdConn struct {
isV4 bool isV4 bool
sysFd uintptr sysFd uintptr
l *slog.Logger l *slog.Logger
noSendmsgX atomic.Bool
sb *sendScratch
} }
var _ Conn = &StdConn{} var _ Conn = &StdConn{}
@@ -139,11 +143,138 @@ func (u *StdConn) WriteTo(b []byte, ap netip.AddrPort) error {
} }
} }
// sendBatch is how many datagrams one sendmsg_x call may carry.
const sendBatch = 64
// sendScratch is WriteBatch's scratch. The kernel reads every entry in place.
type sendScratch struct {
names [sendBatch]unix.RawSockaddrInet6
iovs [sendBatch]unix.Iovec
hdrs [sendBatch]msghdrX
// idx maps each entry back to its index in WriteBatch's bufs.
idx [sendBatch]int
}
// WriteBatch sends bufs with sendmsg_x, darwin's private batched sendmsg, so one syscall carries up to
// sendBatch datagrams. On this unconnected socket xnu still runs each message through sendit, so the
// saving is the syscall crossings, not the per-send policy checks.
//
// xnu's per-message loop stops at the first message that fails. For the errors in sentNothing it reports
// how many went out; for any other it returns only the errno, with the messages before it already sent.
// Each call therefore carries one run of datagrams to the same destination: a destination the kernel
// refuses fails on its first datagram, so a failed call costs that run and nothing else, and no datagram
// is ever resent after the kernel may have taken it.
//
// TODO: WriteTo maps EWOULDBLOCK to an error, so a full send buffer
// silently drops the rest of a burst (linux blocks instead). Poll for
// writability on EAGAIN before giving up on the remainder.
func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) { func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error) {
// An un-sendable destination costs its own packet, never the ones behind it in the batch. if u.noSendmsgX.Load() {
// TODO: WriteTo maps EWOULDBLOCK to an error, so a full send buffer return u.writeEach(bufs, addrs), nil
// silently drops the rest of a burst (linux blocks instead). Poll for }
// writability on EAGAIN before giving up on the remainder. if u.sb == nil {
u.sb = new(sendScratch)
}
b := u.sb
written := 0
for i := 0; i < len(bufs); {
n := 0
for ; i < len(bufs) && n < sendBatch; i++ {
namelen, err := u.putSockaddr(&b.names[n], addrs[i])
if err != nil {
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
continue
}
b.iovs[n] = unix.Iovec{}
if len(bufs[i]) > 0 {
b.iovs[n].Base = &bufs[i][0]
b.iovs[n].SetLen(len(bufs[i]))
}
b.hdrs[n] = msghdrX{Name: (*byte)(unsafe.Pointer(&b.names[n])), Namelen: namelen, Iov: &b.iovs[n], Iovlen: 1}
b.idx[n] = i
n++
}
for off := 0; off < n; {
dst := addrs[b.idx[off]]
end := off + 1
for end < n && addrs[b.idx[end]] == dst {
end++
}
sent, errno := u.sendmsgX(off, end)
switch {
case errno == 0 && sent > 0:
written += sent
off += sent
continue
case sentNothing(errno):
// The datagram at off failed on its own account. sendto delivers or reports it, as without
// batching, and the rest of the run goes back to sendmsg_x.
j := b.idx[off]
written += u.writeEach(bufs[j:j+1], addrs[j:j+1])
off++
continue
case errno == 0:
case errno == unix.ENOSYS || errno == unix.EOPNOTSUPP:
u.fallBack(errno)
case errno == unix.EPERM || errno == unix.EINVAL:
// Either can refuse one destination, but a datagram that sendto delivers was refused by
// sendmsg_x itself, as a sandbox or an older kernel would.
j := b.idx[off]
if u.writeEach(bufs[j:j+1], addrs[j:j+1]) == 0 {
off = end
continue
}
written++
off++
u.fallBack(errno)
default:
u.l.Debug("failed to write packets in batch", "udpAddr", dst, "packets", end-off, "error", errno)
off = end
continue
}
// The kernel took nothing from this call, so the rest can go out one datagram at a time.
for _, j := range b.idx[off:n] {
written += u.writeEach(bufs[j:j+1], addrs[j:j+1])
}
off = n
if u.noSendmsgX.Load() {
clear(b.iovs[:n])
return written + u.writeEach(bufs[i:], addrs[i:]), nil
}
}
clear(b.iovs[:n])
}
return written, nil
}
// sendmsgX hands entries off through end-1 of u.sb to sendmsg_x and returns how many the kernel took.
// The socket is non-blocking, so a full send buffer returns EAGAIN rather than waiting.
func (u *StdConn) sendmsgX(off, end int) (int, syscall.Errno) {
for {
r0, _, errno := unix.Syscall6(unix.SYS_SENDMSG_X, u.sysFd, uintptr(unsafe.Pointer(&u.sb.hdrs[off])), uintptr(end-off), 0, 0, 0)
if errno == unix.EINTR {
continue
}
return int(r0), errno
}
}
// sentNothing reports whether sendmsg_x failing with errno means no datagram went out: once any has, xnu
// turns these errors into a short count instead.
func sentNothing(errno syscall.Errno) bool {
return errno == unix.EWOULDBLOCK || errno == unix.ENOBUFS || errno == unix.EMSGSIZE
}
// fallBack switches WriteBatch to one sendto per datagram for the life of the socket.
func (u *StdConn) fallBack(errno syscall.Errno) {
if u.noSendmsgX.CompareAndSwap(false, true) {
u.l.Warn("sendmsg_x unavailable, sending one datagram per syscall", "error", errno)
}
}
// writeEach sends bufs one at a time with WriteTo and returns how many were written.
func (u *StdConn) writeEach(bufs [][]byte, addrs []netip.AddrPort) int {
written := 0 written := 0
for i, b := range bufs { for i, b := range bufs {
if err := u.WriteTo(b, addrs[i]); err == nil { if err := u.WriteTo(b, addrs[i]); err == nil {
@@ -152,7 +283,23 @@ func (u *StdConn) WriteBatch(bufs [][]byte, addrs []netip.AddrPort) (int, error)
u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err) u.l.Debug("failed to write packet in batch", "udpAddr", addrs[i], "error", err)
} }
} }
return written, nil return written
}
// putSockaddr writes ap into sa in the socket's address family, as WriteTo would, and returns its length.
func (u *StdConn) putSockaddr(sa *unix.RawSockaddrInet6, ap netip.AddrPort) (uint32, error) {
if u.isV4 {
if ap.Addr().Is6() {
return 0, ErrInvalidIPv6RemoteForSocket
}
sa4 := (*unix.RawSockaddrInet4)(unsafe.Pointer(sa))
*sa4 = unix.RawSockaddrInet4{Family: unix.AF_INET, Addr: ap.Addr().As4()}
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&sa4.Port))[:], ap.Port())
return unix.SizeofSockaddrInet4, nil
}
*sa = unix.RawSockaddrInet6{Family: unix.AF_INET6, Addr: ap.Addr().As16()}
binary.BigEndian.PutUint16((*[2]byte)(unsafe.Pointer(&sa.Port))[:], ap.Port())
return unix.SizeofSockaddrInet6, nil
} }
func (u *StdConn) LocalAddr() (netip.AddrPort, error) { func (u *StdConn) LocalAddr() (netip.AddrPort, error) {
@@ -185,6 +332,7 @@ const recvBatch = 64
// msghdrX mirrors xnu's struct msghdr_x (bsd/sys/socket_private.h). recvmsg_x reports each // msghdrX mirrors xnu's struct msghdr_x (bsd/sys/socket_private.h). recvmsg_x reports each
// datagram's length in Datalen rather than in its return value. // datagram's length in Datalen rather than in its return value.
// Keep in sync with msghdrX in overlay/tun_darwin.go.
type msghdrX struct { type msghdrX struct {
Name *byte Name *byte
Namelen uint32 Namelen uint32
@@ -197,8 +345,7 @@ type msghdrX struct {
} }
// ListenOut drains the socket with recvmsg_x, darwin's private batched recvmsg, so one syscall reads // ListenOut drains the socket with recvmsg_x, darwin's private batched recvmsg, so one syscall reads
// up to recvBatch datagrams instead of one. Sends stay on sendto: xnu batches sendmsg_x through // up to recvBatch datagrams instead of one.
// the stack only on connected sockets, and on this unconnected one it loops sendit per message.
func (u *StdConn) ListenOut(r EncReader, flush func()) error { func (u *StdConn) ListenOut(r EncReader, flush func()) error {
rc, err := u.UDPConn.SyscallConn() rc, err := u.UDPConn.SyscallConn()
if err != nil { if err != nil {
+150
View File
@@ -0,0 +1,150 @@
//go:build !e2e_testing
package udp
import (
"fmt"
"log/slog"
"net"
"net/netip"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
// recvAll reads n datagrams from pc, or fails the test after a second without one.
func recvAll(t *testing.T, pc net.PacketConn, n int) []string {
t.Helper()
var got []string
buf := make([]byte, 64)
for len(got) < n {
require.NoError(t, pc.SetReadDeadline(time.Now().Add(time.Second)))
l, _, err := pc.ReadFrom(buf)
require.NoError(t, err, "after %d of %d datagrams", len(got), n)
got = append(got, string(buf[:l]))
}
return got
}
// A batch longer than sendBatch reaches each peer whole and in order through sendmsg_x, without falling
// back to sendto: a run to one peer that spans two calls, then datagrams alternating between peers.
func TestWriteBatchSendsEachToItsPeer(t *testing.T) {
for _, tc := range []struct{ name, listen string }{
{"v4", "127.0.0.1"},
{"v6", "::1"},
} {
t.Run(tc.name, func(t *testing.T) {
c, err := NewListener(slog.New(slog.DiscardHandler), Settings{Listen: netip.AddrPortFrom(netip.MustParseAddr(tc.listen), 0)})
require.NoError(t, err)
defer c.Close()
var peers [2]net.PacketConn
var dsts [2]netip.AddrPort
for i := range peers {
peers[i], err = net.ListenPacket("udp", net.JoinHostPort(tc.listen, "0"))
require.NoError(t, err)
defer peers[i].Close()
dsts[i] = netip.MustParseAddrPort(peers[i].LocalAddr().String())
}
const total = sendBatch*2 + 5
bufs := make([][]byte, total)
addrs := make([]netip.AddrPort, total)
var want [2][]string
for i := range bufs {
bufs[i] = fmt.Appendf(nil, "pkt-%03d", i)
p := i % 2
if i < sendBatch+6 {
p = 0
}
addrs[i] = dsts[p]
want[p] = append(want[p], string(bufs[i]))
}
n, err := c.WriteBatch(bufs, addrs)
require.NoError(t, err)
assert.Equal(t, total, n)
assert.False(t, c.(*StdConn).noSendmsgX.Load(), "fell back to sendto")
for i := range peers {
assert.Equal(t, want[i], recvAll(t, peers[i], len(want[i])))
}
})
}
}
// Destinations that can't be sent to cost only their own packets: an IPv6 peer on an IPv4 socket is
// skipped before the call, and a broadcast address without SO_BROADCAST is refused by the kernel
// mid-batch. Neither stops the packets around it, delivers any twice, or turns sendmsg_x off.
func TestWriteBatchSkipsUnsendableDestinations(t *testing.T) {
c, err := NewListener(slog.New(slog.DiscardHandler), Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0")})
require.NoError(t, err)
defer c.Close()
peer, err := net.ListenPacket("udp", "127.0.0.1:0")
require.NoError(t, err)
defer peer.Close()
good := netip.MustParseAddrPort(peer.LocalAddr().String())
bcast := netip.AddrPortFrom(netip.MustParseAddr("255.255.255.255"), good.Port())
v6 := netip.MustParseAddrPort("[::1]:4242")
addrs := []netip.AddrPort{good, bcast, bcast, good, v6, good, bcast}
bufs := make([][]byte, len(addrs))
for i := range bufs {
bufs[i] = fmt.Appendf(nil, "pkt-%d", i)
}
n, err := c.WriteBatch(bufs, addrs)
require.NoError(t, err)
assert.Equal(t, 3, n)
assert.False(t, c.(*StdConn).noSendmsgX.Load(), "fell back to sendto")
assert.Equal(t, []string{"pkt-0", "pkt-3", "pkt-5"}, recvAll(t, peer, 3))
require.NoError(t, peer.SetReadDeadline(time.Now().Add(100*time.Millisecond)))
_, _, err = peer.ReadFrom(make([]byte, 64))
assert.Error(t, err, "a datagram arrived twice")
}
// A datagram the kernel refuses as its own fault (EMSGSIZE here, as ENOBUFS would be under a full
// interface queue) costs only itself, whether it fails after others in the same call (a short count) or
// first (the errno alone): the rest of its run still arrives, in order, and sendmsg_x stays on.
func TestWriteBatchLosesOnlyTheFailedDatagram(t *testing.T) {
for _, tc := range []struct {
name string
big int
}{
{"mid-call", 2},
{"first", 0},
} {
t.Run(tc.name, func(t *testing.T) {
c, err := NewListener(slog.New(slog.DiscardHandler), Settings{Listen: netip.MustParseAddrPort("127.0.0.1:0")})
require.NoError(t, err)
defer c.Close()
peer, err := net.ListenPacket("udp", "127.0.0.1:0")
require.NoError(t, err)
defer peer.Close()
dst := netip.MustParseAddrPort(peer.LocalAddr().String())
bufs := make([][]byte, 5)
addrs := make([]netip.AddrPort, len(bufs))
var want []string
for i := range bufs {
addrs[i] = dst
if i == tc.big {
// Past the 65535-byte IP datagram limit, so the kernel can only refuse it.
bufs[i] = make([]byte, 70000)
continue
}
bufs[i] = fmt.Appendf(nil, "pkt-%d", i)
want = append(want, string(bufs[i]))
}
n, err := c.WriteBatch(bufs, addrs)
require.NoError(t, err)
assert.Equal(t, len(want), n)
assert.False(t, c.(*StdConn).noSendmsgX.Load(), "fell back to sendto")
assert.Equal(t, want, recvAll(t, peer, len(want)))
})
}
}