mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-16 20:46:52 +01:00
329 lines
9.7 KiB
Go
329 lines
9.7 KiB
Go
//go:build darwin && !ios
|
|
|
|
package tio
|
|
|
|
import (
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"sync"
|
|
"sync/atomic"
|
|
"syscall"
|
|
_ "unsafe" // for go:linkname
|
|
|
|
"github.com/slackhq/nebula/internal/msgx"
|
|
"golang.org/x/sys/unix"
|
|
)
|
|
|
|
// tunMaxPacket is the largest packet utun can return at any device MTU.
|
|
const tunMaxPacket = 65535
|
|
|
|
// tunReadBatch is the most packets one Utun.Read returns.
|
|
const tunReadBatch = 64
|
|
|
|
// tunReadArena is Utun's receive buffer. Draining stops once less than tunMaxPacket of it is left,
|
|
// so every readv has room for the largest packet utun can return.
|
|
const tunReadArena = 4 * tunMaxPacket
|
|
|
|
// tunWriteBatch is how many packets one sendmsg_x call may carry.
|
|
const tunWriteBatch = 64
|
|
|
|
// Utun is a darwin utun control socket as a Queue. Every packet crosses the socket behind a 4 byte
|
|
// address family prefix, which Read strips and the writes add.
|
|
//
|
|
// The fd lives in the runtime poller, so Close wakes a Read parked on it; darwin needs no QueueSet
|
|
// shutdown fd.
|
|
type Utun struct {
|
|
f *os.File
|
|
l *slog.Logger
|
|
|
|
// readBuf and readRet are Read's scratch, owned by the single reader.
|
|
readBuf []byte
|
|
readRet [tunReadBatch]Packet
|
|
|
|
// noSendmsgX is set when libSystem lacks sendmsg_x or the kernel refuses it; WriteBatch then writes one packet at a time.
|
|
noSendmsgX atomic.Bool
|
|
batchMu sync.Mutex
|
|
batch tunBatch
|
|
}
|
|
|
|
// tunBatch is WriteBatch's scratch, guarded by Utun.batchMu. The kernel reads every entry in place.
|
|
type tunBatch struct {
|
|
heads [tunWriteBatch][4]byte
|
|
iovs [tunWriteBatch][2]unix.Iovec
|
|
hdrs [tunWriteBatch]msgx.Hdr
|
|
pkts [tunWriteBatch][]byte
|
|
}
|
|
|
|
// NewUtun wraps a utun control socket fd and makes it non-blocking.
|
|
// On failure it does NOT close fd: the caller owns fd until NewUtun succeeds, after which Close closes it.
|
|
func NewUtun(fd int, l *slog.Logger) (*Utun, error) {
|
|
// Before os.NewFile, which only hands a non-blocking fd to the runtime poller.
|
|
if err := unix.SetNonblock(fd, true); err != nil {
|
|
return nil, fmt.Errorf("failed to set the tun fd to non-blocking mode: %w", err)
|
|
}
|
|
u := &Utun{
|
|
f: os.NewFile(uintptr(fd), "utun"),
|
|
l: l,
|
|
readBuf: make([]byte, tunReadArena),
|
|
}
|
|
u.noSendmsgX.Store(!msgx.Available())
|
|
return u, nil
|
|
}
|
|
|
|
// tunWritev and tunReadv are linkname'd to x/sys/unix's libc-routed writev/readv stubs so the
|
|
// calls go through libSystem's pinned trampoline. A raw syscall.Syscall(SYS_WRITEV/SYS_READV, ...)
|
|
// on darwin/arm64 emits an SVC #0x80 trap (see $GOROOT/src/syscall/asm_darwin_arm64.s), the path
|
|
// Apple keeps warning they will eventually disallow. We pull the low-level stubs instead of calling
|
|
// unix.Writev/unix.Readv because those take [][]byte and rebuild the []Iovec every call, which
|
|
// heap-allocates the header; linkname'ing the stubs lets us hand them our own stack-allocated
|
|
// iovecs. See golang/go#78049.
|
|
|
|
//go:linkname tunWritev golang.org/x/sys/unix.writev
|
|
//go:noescape
|
|
func tunWritev(fd int, iovecs []unix.Iovec) (n int, err error)
|
|
|
|
//go:linkname tunReadv golang.org/x/sys/unix.readv
|
|
//go:noescape
|
|
func tunReadv(fd int, iovecs []unix.Iovec) (n int, err error)
|
|
|
|
// Read waits for the utun to become readable, then reads packets until it would block, up to
|
|
// tunReadBatch, so the caller encrypts and sends them as one batch rather than one per wakeup. Each
|
|
// packet is read with its AF prefix scattered away, so the payload lands directly in readBuf.
|
|
// An error after at least one packet is dropped in favor of returning those packets; a persistent
|
|
// one recurs on the next Read.
|
|
func (u *Utun) Read() ([]Packet, error) {
|
|
rc, err := u.f.SyscallConn()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var head [4]byte
|
|
n, off := 0, 0
|
|
var callErr error
|
|
err = rc.Read(func(fd uintptr) bool {
|
|
for n < tunReadBatch && len(u.readBuf)-off >= tunMaxPacket {
|
|
iovecs := [2]unix.Iovec{
|
|
{Base: &head[0], Len: 4},
|
|
{Base: &u.readBuf[off], Len: uint64(len(u.readBuf) - off)},
|
|
}
|
|
l, e := tunReadv(int(fd), iovecs[:])
|
|
if e != nil {
|
|
if errno, ok := e.(syscall.Errno); ok && errno.Temporary() {
|
|
// Park on the poller only while there is nothing to hand back.
|
|
return n > 0
|
|
}
|
|
callErr = e
|
|
return true
|
|
}
|
|
if l < 4 {
|
|
// A datagram too short to carry the AF prefix, or end of file; stop rather than spin on it.
|
|
return true
|
|
}
|
|
end := off + l - 4
|
|
u.readRet[n] = Packet{Bytes: u.readBuf[off:end:end]}
|
|
n++
|
|
off = end
|
|
}
|
|
return true
|
|
})
|
|
if n > 0 {
|
|
return u.readRet[:n], nil
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return nil, callErr
|
|
}
|
|
|
|
// Write pushes one IP packet onto the utun device. Safe for concurrent use:
|
|
// the AF prefix and iovecs are per-call stack state, and the fd write itself
|
|
// serializes on the runtime's fd mutex (see the Queue contract in tio.go).
|
|
func (u *Utun) Write(from []byte) (int, error) {
|
|
if len(from) == 0 {
|
|
return 0, syscall.EIO
|
|
}
|
|
|
|
var head [4]byte
|
|
af, err := tunAF(from)
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
head[3] = af
|
|
|
|
// Grab rc as a local so the compiler can devirtualize the call and keep the closure on the stack.
|
|
rc, err := u.f.SyscallConn()
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
|
|
var n int
|
|
var callErr error
|
|
err = rc.Write(func(fd uintptr) bool {
|
|
iovecs := []unix.Iovec{
|
|
{Base: &head[0], Len: 4},
|
|
{Base: &from[0], Len: uint64(len(from))},
|
|
}
|
|
n, callErr = tunWritev(int(fd), iovecs)
|
|
// Type-assert to syscall.Errno so the EAGAIN/EWOULDBLOCK/EINTR check doesn't box the errno
|
|
// constants into error interfaces on every call.
|
|
if errno, ok := callErr.(syscall.Errno); ok && errno.Temporary() {
|
|
return false
|
|
}
|
|
return true
|
|
})
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if callErr != nil {
|
|
return 0, callErr
|
|
}
|
|
|
|
return n - 4, nil
|
|
}
|
|
|
|
// tunAF returns the utun address-family prefix byte for an IP packet.
|
|
func tunAF(pkt []byte) (byte, error) {
|
|
switch pkt[0] >> 4 {
|
|
case 4:
|
|
return syscall.AF_INET, nil
|
|
case 6:
|
|
return syscall.AF_INET6, nil
|
|
default:
|
|
return 0, fmt.Errorf("unable to determine IP version from packet")
|
|
}
|
|
}
|
|
|
|
// WriteBatch writes pkts to the utun device with sendmsg_x, xnu's private batched sendmsg, up to
|
|
// tunWriteBatch packets per syscall. The kernel still hands each packet to utun on its own, so this
|
|
// saves syscalls, not per-packet kernel work. Safe for concurrent use.
|
|
//
|
|
// An empty packet or one with no IP version is skipped and reported, as Write would. Any other
|
|
// error from sendmsg_x means the kernel took an unknown prefix of that call and dropped the rest, so
|
|
// it is reported rather than retried, which could deliver packets twice.
|
|
//
|
|
// Under mbuf exhaustion, sendmsg_x silently drops packets that writev would have delivered: the
|
|
// kernel counts each packet as sent once it is copied in, and when a later allocation fails it
|
|
// frees that prefix unsent and returns its length as a short count, which is indistinguishable
|
|
// from a real one.
|
|
func (u *Utun) WriteBatch(pkts [][]byte) error {
|
|
if u.noSendmsgX.Load() {
|
|
return u.writeEach(pkts)
|
|
}
|
|
|
|
u.batchMu.Lock()
|
|
defer u.batchMu.Unlock()
|
|
b := &u.batch
|
|
var firstErr error
|
|
for len(pkts) > 0 {
|
|
n := 0
|
|
for len(pkts) > 0 && n < tunWriteBatch {
|
|
p := pkts[0]
|
|
pkts = pkts[1:]
|
|
var af byte
|
|
err := error(syscall.EIO)
|
|
if len(p) > 0 {
|
|
af, err = tunAF(p)
|
|
}
|
|
if err != nil {
|
|
if firstErr == nil {
|
|
firstErr = err
|
|
}
|
|
continue
|
|
}
|
|
b.heads[n] = [4]byte{3: af}
|
|
b.iovs[n] = [2]unix.Iovec{
|
|
{Base: &b.heads[n][0], Len: 4},
|
|
{Base: &p[0], Len: uint64(len(p))},
|
|
}
|
|
b.hdrs[n] = msgx.Hdr{Iov: &b.iovs[n][0], Iovlen: 2}
|
|
b.pkts[n] = p
|
|
n++
|
|
}
|
|
if n == 0 {
|
|
continue
|
|
}
|
|
|
|
for off := 0; off < n; {
|
|
sent, err := u.sendmsgX(off, n)
|
|
switch {
|
|
case err == nil && sent > 0:
|
|
// The kernel stops early without an error on a packet larger than the socket's send
|
|
// buffer, or when sending a later packet fails with ENOBUFS; resend the remainder.
|
|
off += sent
|
|
case err == nil:
|
|
if werr := u.writeEach(b.pkts[off:n]); werr != nil && firstErr == nil {
|
|
firstErr = werr
|
|
}
|
|
off = n
|
|
case err == unix.EMSGSIZE:
|
|
// Only the first packet being larger than the socket's send buffer fails the whole call,
|
|
// before the kernel takes any; Write reports that one and the rest go out in the next call.
|
|
if werr := u.writeEach(b.pkts[off : off+1]); werr != nil && firstErr == nil {
|
|
firstErr = werr
|
|
}
|
|
off++
|
|
case err == unix.ENOSYS || err == unix.EPERM || err == unix.EOPNOTSUPP:
|
|
// sendmsg_x is private API: fall back to writev if a kernel or sandbox refuses it.
|
|
if u.noSendmsgX.CompareAndSwap(false, true) {
|
|
u.l.Warn("sendmsg_x unavailable on the tun device, writing one packet per syscall", "error", err)
|
|
}
|
|
if werr := u.writeEach(b.pkts[off:n]); werr != nil && firstErr == nil {
|
|
firstErr = werr
|
|
}
|
|
clear(b.iovs[:n])
|
|
clear(b.pkts[:n])
|
|
if werr := u.writeEach(pkts); werr != nil && firstErr == nil {
|
|
firstErr = werr
|
|
}
|
|
return firstErr
|
|
default:
|
|
if firstErr == nil {
|
|
firstErr = err
|
|
}
|
|
off = n
|
|
}
|
|
}
|
|
clear(b.iovs[:n])
|
|
clear(b.pkts[:n])
|
|
}
|
|
return firstErr
|
|
}
|
|
|
|
// sendmsgX hands entries off through n-1 of u.batch to sendmsg_x and returns how many the kernel took.
|
|
func (u *Utun) sendmsgX(off, n int) (int, error) {
|
|
rc, err := u.f.SyscallConn()
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
var sent int
|
|
var errno syscall.Errno
|
|
err = rc.Write(func(fd uintptr) bool {
|
|
sent, errno = msgx.Send(fd, u.batch.hdrs[off:n], 0)
|
|
// sendmsg_x reports EAGAIN only when it took nothing; a partial batch returns its count.
|
|
return !errno.Temporary()
|
|
})
|
|
if err != nil {
|
|
return 0, err
|
|
}
|
|
if errno != 0 {
|
|
return 0, errno
|
|
}
|
|
return sent, nil
|
|
}
|
|
|
|
// writeEach writes pkts one at a time with Write.
|
|
func (u *Utun) writeEach(pkts [][]byte) error {
|
|
var firstErr error
|
|
for _, p := range pkts {
|
|
if _, err := u.Write(p); err != nil && firstErr == nil {
|
|
firstErr = err
|
|
}
|
|
}
|
|
return firstErr
|
|
}
|
|
|
|
func (u *Utun) Close() error {
|
|
return u.f.Close()
|
|
}
|