overlay/tio: KeepAlive poll-path readv/writev buffers too

The Poll fallback (used when IFF_VNET_HDR can't be enabled) has the
same unsafe-pointer-via-uintptr pattern as Offload.rawWrite:
readOne/writeOne build a [2]syscall.Iovec on the stack, pass it to
syscall.Syscall as uintptr, and the kernel then DMAs in/out of the
to/from slices whose Base pointers the iovec holds.

Escape analysis can't see that use, so under GC pressure the
backing memory could be collected or moved mid-syscall.

Add runtime.KeepAlive on the iovec and the user buffer around both
the SYS_READV and SYS_WRITEV syscalls. Same pattern and rationale
as the prior commit on the offload path.

overlay/tio/tio_gso_linux.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"runtime"
 	"sync/atomic"
 	"syscall"
 	"unsafe"
@@ -32,15 +33,6 @@ const tunDrainCap = 64 //256
 // any reallocations.
 const gsoInitialPayIovs = 66
 
-// gsoWriteBufCap is the initial per-queue coalesce scratch capacity used by
-// WriteGSO to assemble [virtio_hdr || IP/TCP hdr || pays...] into a single
-// contiguous buffer so we can emit the superpacket via a single write()
-// instead of writev(). One worst-case TSO superpacket is bounded by the
-// virtio spec at 64KiB; 128KiB gives comfortable slack for the 10-byte
-// virtio header, the IP/TCP header, and any future size bumps. Grown on
-// demand if a superpacket exceeds this.
-const gsoWriteBufCap = tunSegBufSize
-
 // validVnetHdr is the 10-byte virtio_net_hdr we prepend to every non-GSO TUN
 // write. Only flag set is VIRTIO_NET_HDR_F_DATA_VALID, which marks the skb
 // CHECKSUM_UNNECESSARY so the receiving network stack skips L4 checksum
@@ -74,20 +66,10 @@ type Offload struct {
 	// by WriteGSO. Separate from validVnetHdr so a concurrent non-GSO Write on
 	// another queue never observes a half-written header.
 	gsoHdrBuf [virtioNetHdrLen]byte
-	// gsoIovs is a legacy writev iovec scratch. No longer used by the
-	// WriteGSO path (which coalesces into gsoWriteBuf and uses a single
-	// write()) but retained for any other iovec-based path that may use it.
+	// gsoIovs is the writev iovec scratch for WriteGSO. Sized to hold the
+	// virtio header + IP/TCP header + up to gsoInitialPayIovs payload
+	// fragments; grown on demand if a coalescer pushes more.
 	gsoIovs []unix.Iovec
-
-	// gsoWriteBuf is a per-queue scratch used by WriteGSO to coalesce the
-	// virtio_net_hdr + IP/TCP header + payload fragments into a single
-	// contiguous buffer, which is then written to the TUN fd with one
-	// write() syscall. This mirrors wireguard-go's approach and avoids
-	// triggering a kernel refcount use-after-free in skb_set_owner_w /
-	// sock_wfree observed on Linux 4.19 TUN when scatter-gather writev is
-	// combined with GSO-flagged virtio_net_hdr in the tun_chr_write_iter
-	// path. Grown on demand if a superpacket exceeds the initial cap.
-	gsoWriteBuf []byte
 }
 
 func newOffload(fd int, shutdownFd int) (*Offload, error) {
@@ -109,9 +91,8 @@ func newOffload(fd int, shutdownFd int) (*Offload, error) {
 			{Fd: int32(shutdownFd), Events: unix.POLLIN},
 		},
 
-		segBuf:      make([]byte, tunSegBufCap),
-		gsoIovs:     make([]unix.Iovec, 2, 2+gsoInitialPayIovs),
-		gsoWriteBuf: make([]byte, 0, gsoWriteBufCap),
+		segBuf:  make([]byte, tunSegBufCap),
+		gsoIovs: make([]unix.Iovec, 2, 2+gsoInitialPayIovs),
 	}
 
 	out.writeIovs[0].Base = &validVnetHdr[0]
@@ -283,12 +264,24 @@ func (r *Offload) writeWithScratch(buf []byte, iovs *[2]unix.Iovec) (int, error)
 	// to validVnetHdr during Offload construction so we don't rebuild it here.
 	iovs[1].Base = &buf[0]
 	iovs[1].SetLen(len(buf))
-	return r.rawWrite(unsafe.Slice(&iovs[0], len(iovs)))
+	iovPtr := unsafe.Pointer(&iovs[0])
+	// Pin the caller's buffer AND the iovec array through the syscall.
+	return r.rawWrite(iovPtr, 2, buf, iovs)
 }
 
-func (r *Offload) rawWrite(iovs []unix.Iovec) (int, error) {
+func (r *Offload) rawWrite(iovs unsafe.Pointer, iovcnt int, keepAlive ...interface{}) (int, error) {
 	for {
-		n, _, errno := syscall.Syscall(unix.SYS_WRITEV, uintptr(r.fd), uintptr(unsafe.Pointer(&iovs[0])), uintptr(len(iovs)))
+		n, _, errno := syscall.Syscall(unix.SYS_WRITEV, uintptr(r.fd), uintptr(iovs), uintptr(iovcnt))
+		// Anchor the iovec array + every user-supplied payload slice
+		// through the syscall return. Without these, Go's GC may move or
+		// collect the underlying backing arrays while the kernel is still
+		// reading them via DMA (we pass the iovec as uintptr, so the
+		// compiler does not keep it live). Observed in practice as a
+		// kernel refcount underflow on tun_chr_write_iter / sock_wfree.
+		runtime.KeepAlive(iovs)
+		for _, ka := range keepAlive {
+			runtime.KeepAlive(ka)
+		}
 		if errno == 0 {
 			if int(n) < virtioNetHdrLen {
 				return 0, io.ErrShortWrite
@@ -311,59 +304,18 @@ func (r *Offload) rawWrite(iovs []unix.Iovec) (int, error) {
 	}
 }
 
-// rawWriteSingle writes buf to the TUN fd with a single write() syscall.
-// Unlike rawWrite (which uses writev), this avoids the kernel
-// scatter-gather path that triggers a use-after-free in
-// tun_chr_write_iter → sock_alloc_send_pskb → skb_set_owner_w on Linux
-// 4.19 TUN when the virtio_net_hdr requests TSO segmentation. The caller
-// is responsible for including the virtio_net_hdr prefix in buf.
-func (r *Offload) rawWriteSingle(buf []byte) (int, error) {
-	for {
-		n, err := unix.Write(r.fd, buf)
-		if err == nil {
-			if n < virtioNetHdrLen {
-				return 0, io.ErrShortWrite
-			}
-			return n - virtioNetHdrLen, nil
-		}
-		if err == unix.EAGAIN {
-			if werr := r.blockOnWrite(); werr != nil {
-				return 0, werr
-			}
-			continue
-		}
-		if err == unix.EINTR {
-			continue
-		}
-		if err == unix.EBADF {
-			return 0, os.ErrClosed
-		}
-		return 0, err
-	}
-}
-
 // GSOSupported reports whether this queue was opened with IFF_VNET_HDR and
 // can accept WriteGSO. When false, callers should fall back to per-segment
 // Write calls.
 func (r *Offload) GSOSupported() bool { return true }
 
-// WriteGSO emits a TCP TSO superpacket. hdr is the IPv4/IPv6 + TCP header
-// prefix (already finalized — total length, IP csum, and TCP pseudo-header
-// partial set by the caller). pays are payload fragments whose concatenation
-// forms the full coalesced payload. gsoSize is the MSS; every segment except
-// possibly the last is exactly gsoSize bytes. csumStart is the byte offset
-// where the TCP header begins within hdr.
-//
-// Implementation note: this path coalesces [virtio_hdr || hdr || pays...]
-// into a single contiguous scratch buffer (r.gsoWriteBuf) and emits it via
-// one write() syscall rather than writev() with a scatter-gather iovec.
-// The scatter-gather path triggered a kernel-side use-after-free on Linux
-// 4.19 TUN where tun_chr_write_iter → sock_alloc_send_pskb →
-// skb_set_owner_w could be invoked with a zero sk_wmem_alloc, crashing
-// the router. The single-write path mirrors wireguard-go's design (see
-// golang.zx2c4.com/wireguard/tun/tun_linux.go Write — it always coalesces
-// GRO-merged data into a single contiguous buffer before calling
-// tunFile.Write) and has no equivalent failure mode.
+// WriteGSO emits a TCP TSO superpacket in a single writev. hdr is the
+// IPv4/IPv6 + TCP header prefix (already finalized — total length, IP csum,
+// and TCP pseudo-header partial set by the caller). pays are payload
+// fragments whose concatenation forms the full coalesced payload; each
+// slice is read-only and must stay valid until return. gsoSize is the MSS;
+// every segment except possibly the last is exactly gsoSize bytes.
+// csumStart is the byte offset where the TCP header begins within hdr.
 func (r *Offload) WriteGSO(hdr []byte, pays [][]byte, gsoSize uint16, isV6 bool, csumStart uint16) error {
 	if len(hdr) == 0 || len(pays) == 0 {
 		return nil
@@ -395,26 +347,30 @@ func (r *Offload) WriteGSO(hdr []byte, pays [][]byte, gsoSize uint16, isV6 bool,
 	}
 	vhdr.encode(r.gsoHdrBuf[:])
 
-	// Coalesce [virtio_hdr || hdr || pays...] into a single contiguous
-	// buffer. This avoids the kernel scatter-gather write path entirely.
-	need := virtioNetHdrLen + len(hdr) + totalPay
-	if cap(r.gsoWriteBuf) < need {
-		// Grow geometrically to amortize reallocs.
-		newCap := cap(r.gsoWriteBuf) * 2
-		if newCap < need {
-			newCap = need
-		}
-		r.gsoWriteBuf = make([]byte, 0, newCap)
+	// Build the iovec array: [virtio_hdr, hdr, pays...]. r.gsoIovs[0] is
+	// wired to gsoHdrBuf at construction and never changes.
+	need := 2 + len(pays)
+	if cap(r.gsoIovs) < need {
+		grown := make([]unix.Iovec, need)
+		grown[0] = r.gsoIovs[0]
+		r.gsoIovs = grown
 	} else {
-		r.gsoWriteBuf = r.gsoWriteBuf[:0]
+		r.gsoIovs = r.gsoIovs[:need]
 	}
-	r.gsoWriteBuf = append(r.gsoWriteBuf, r.gsoHdrBuf[:]...)
-	r.gsoWriteBuf = append(r.gsoWriteBuf, hdr...)
-	for _, p := range pays {
-		r.gsoWriteBuf = append(r.gsoWriteBuf, p...)
+	r.gsoIovs[1].Base = &hdr[0]
+	r.gsoIovs[1].SetLen(len(hdr))
+	for i, p := range pays {
+		r.gsoIovs[2+i].Base = &p[0]
+		r.gsoIovs[2+i].SetLen(len(p))
 	}
 
-	_, err := r.rawWriteSingle(r.gsoWriteBuf)
+	iovPtr := unsafe.Pointer(&r.gsoIovs[0])
+	iovCnt := len(r.gsoIovs)
+	// Pin EVERYTHING the kernel might still read via DMA: the backing iovec
+	// slice, the IP/TCP header buffer, and every individual payload
+	// fragment. Skipping any of these risks a use-after-free in
+	// tun_chr_write_iter if GC runs mid-syscall.
+	_, err := r.rawWrite(iovPtr, iovCnt, r.gsoIovs, hdr, pays)
 	return err
 }
 

overlay/tio/tio_poll_linux.go

@@ -3,6 +3,7 @@ package tio
 import (
 	"fmt"
 	"os"
+	"runtime"
 	"sync/atomic"
 	"syscall"
 	"unsafe"
@@ -120,6 +121,13 @@ func (t *Poll) readOne(to []byte) (int, error) {
 	}
 	for {
 		n, _, errno := syscall.Syscall(syscall.SYS_READV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
+		// Pin the iovec + destination buffer backing array across the syscall.
+		// Without these the Go runtime may move/GC them while the kernel is
+		// still writing via DMA (we pass the iovec as uintptr, which hides it
+		// from escape analysis). Same class of bug as rawWrite in the Offload
+		// path.
+		runtime.KeepAlive(iovecs)
+		runtime.KeepAlive(to)
 		if errno == 0 {
 			bytesRead := int(n)
 			if bytesRead < 4 {
@@ -166,6 +174,10 @@ func (t *Poll) Write(from []byte) (int, error) {
 	}
 	for {
 		n, _, errno := syscall.Syscall(syscall.SYS_WRITEV, uintptr(t.fd), uintptr(unsafe.Pointer(&iovecs[0])), 2)
+		// Pin the iovec + source buffer backing array across the syscall.
+		// See readOne's KeepAlive comment for rationale.
+		runtime.KeepAlive(iovecs)
+		runtime.KeepAlive(from)
 		if errno == 0 {
 			return int(n) - 4, nil
 		}