datapath: fix 12 correctness findings from tun/UDP offload review

Multi-disciplinary correctness review of the batched tun / GSO-GRO / sendmmsg
rework. Each fix has a regression test; the merged tree builds on
linux/darwin/openbsd/windows/freebsd/netbsd, vets clean, passes the unit and
e2e suites, and is -race clean.

Critical:
- C1 zero-length inner UDP datagram no longer panics the process (remote DoS):
  the UDP coalescer routes payLen==0 to passthrough instead of seeding a GSO
  slot, and WriteGSO skips empty payload iovecs as defense in depth.
- C2 segmenter no longer corrupts inner headers when gsoSize < headerLen: the
  L3+L4 header is snapshotted once and each segment stamped from the copy,
  replacing the destructive overlapping in-place slide (SegmentTCP + SegmentUDP).

High:
- H1 applyOuterECN updates the IPv4 header checksum (RFC 1624 incremental) when
  folding outer CE into the inner ToS, so passthrough packets are no longer
  dropped by the peer stack.
- H2 the GRO reject path caps the borrowed RX segment ([:n:n]) so a reject can
  no longer overrun into the next coalesced segment's Nebula header. Note:
  oversized ICMPv6 rejects that need >16B beyond the segment are now refused
  rather than sent under GRO (safe; see TOFIX.md for the scratch-buffer follow-up).
- H3 WriteBatch falls back to per-packet WriteTo for a chunk when writeSockaddr
  fails, so one bad-family destination costs only its own packet, not the batch.
- H4 UserDevice.Readers returns N distinct queue wrappers with private buffers
  (sharing the pipes) so concurrent readers no longer race/overwrite borrowed
  packet bytes.
- H5 Poll.Close / Offload.Close no longer null t.fd (matching master's
  tunFile.Close), removing the data race with a concurrent readOne load.

Medium/Low:
- M1 the UDP GSO 127-segment gate moved from kernel >=5.5 to >=6.9 (the real
  UDP_MAX_SEGMENTS 64->128 threshold), avoiding EINVAL + per-packet fallback on
  5.5-6.8 kernels.
- M2 NewMultiQueueReader replays the offload mask newTun actually negotiated
  instead of the TSO-only mask, so adding a queue no longer disables USO
  device-wide; the advertised USO capability derives from the same mask.
- M3 the shutdown eventfd is closed in pollQueueSet.Close / offloadQueueSet.Close
  (double-close guarded), fixing the per-lifecycle fd leak.
- M4 dual-stack ECN selects the cmsg by address family, not socket family: RX
  parseRecvCmsg reads both IP_TOS and IPV6_TCLASS; TX writeEntryCmsg stamps
  IP_TOS for v4/v4-mapped dests and IPV6_TCLASS for v6 (on-host verified).
- L1 newPoll no longer closes the fd on failure (matching newOffload), removing
  the double-close on QueueSet.Add error.
This commit is contained in:
JackDoan
2026-07-13 16:35:20 -05:00
parent 733dc06192
commit 44dd2e9ca4
19 changed files with 1346 additions and 72 deletions
+52 -20
View File
@@ -27,6 +27,13 @@ const (
tcpHeaderMaxLen = 60 // data-offset=15, max options
)
// maxSegHdrLen bounds the L3+L4 header we snapshot before stamping each
// segment. The largest header the segmenter supports is IPv4 (max IHL 60)
// plus TCP (max data-offset 60) = 120 bytes; the array is sized to that
// worst case so the snapshot lives on the stack with no per-call heap
// allocation.
const maxSegHdrLen = ipv4HeaderMaxLen + tcpHeaderMaxLen // 120
// Byte offsets inside an IPv4 header.
const (
ipv4TotalLenOff = 2
@@ -144,13 +151,18 @@ func CorrectHdrLen(pkt []byte, hdr *Hdr) error {
}
// SegmentTCP walks a TSO superpacket pkt, yielding each segment as a
// slice into pkt itself. Per-segment plaintext is laid out by sliding a
// freshly-patched copy of the L3+L4 header into pkt at offset i*gsoSize,
// where it sits immediately before that segment's payload chunk in the
// original buffer. The slide is destructive: iter i's header write overwrites
// the last hdrLen bytes of seg_{i-1}'s payload, which is dead by the time
// the next iteration begins. pkt is consumed by this call and must not be
// inspected by the caller after the final yield.
// slice into pkt itself. Per-segment plaintext is laid out by stamping a
// copy of the original L3+L4 header into pkt at offset i*gsoSize, where it
// sits immediately before that segment's payload chunk in the original
// buffer. The stamp is destructive but harmless: iter i's header write lands
// on pkt[i*G : i*G+hdrLen], which is the tail of seg_{i-1}'s payload (already
// consumed) and ends exactly where seg_i's payload begins, so it never clobbers
// live payload — this holds even when gsoSize < hdrLen. The header bytes are
// sourced from a pristine snapshot taken before the loop (savedHdr), NOT from
// pkt[:hdrLen], because when gsoSize < hdrLen the stamps would otherwise
// overwrite the leading header in place and every stamp after the first would
// copy corrupted bytes. pkt is consumed by this call and must not be inspected
// by the caller after the final yield.
func SegmentTCP(pkt []byte, hdrLenU, csumStartU, gsoSizeU uint16, yield func(seg []byte) error) error {
if gsoSizeU == 0 {
return fmt.Errorf("gso_size is zero")
@@ -161,6 +173,9 @@ func SegmentTCP(pkt []byte, hdrLenU, csumStartU, gsoSizeU uint16, yield func(seg
headerLen := int(hdrLenU)
csumStart := int(csumStartU)
if headerLen > maxSegHdrLen {
return fmt.Errorf("header len %d exceeds max %d", headerLen, maxSegHdrLen)
}
isV4 := pkt[0]>>4 == 4
tcpHdrLen := int(pkt[csumStart+tcpDataOffOff]>>4) * 4
@@ -205,6 +220,13 @@ func SegmentTCP(pkt []byte, hdrLenU, csumStartU, gsoSizeU uint16, yield func(seg
baseIPHdrSum = uint32(checksum.Checksum(ipTmp[:ihl], 0))
}
// Snapshot the pristine L3+L4 header once. Every segment's header is
// stamped from this copy, so overlapping stamps (gsoSize < headerLen)
// can never corrupt the source. The variable fields (seq/flags/cksum/
// totalLen/id) captured here are stale but are overwritten per segment.
var savedHdr [maxSegHdrLen]byte
copy(savedHdr[:headerLen], pkt[:headerLen])
for i := 0; i < numSeg; i++ {
segStart := i * gsoSize
segEnd := segStart + gsoSize
@@ -215,14 +237,13 @@ func SegmentTCP(pkt []byte, hdrLenU, csumStartU, gsoSizeU uint16, yield func(seg
segLen := headerLen + segPayLen
headerOff := i * gsoSize
// Slide the header into place immediately before this segment's
// payload. Iter 0's header is already at pkt[:headerLen]; for
// i ≥ 1 we copy from there. The constant-byte fields of pkt[:headerLen]
// survive iter 0's in-place patches (only seq/flags/cksum/totalLen/id
// are touched), and iter 0's stale variable-field values are
// overwritten by the per-segment patches below.
// Stamp the header into place immediately before this segment's
// payload, sourced from the pristine snapshot. Iter 0's header is
// already at pkt[:headerLen] (identical to savedHdr), so only i ≥ 1
// needs the stamp. The per-segment patches below overwrite the
// variable fields.
if i > 0 {
copy(pkt[headerOff:headerOff+headerLen], pkt[:headerLen])
copy(pkt[headerOff:headerOff+headerLen], savedHdr[:headerLen])
}
seg := pkt[headerOff : headerOff+segLen]
@@ -269,11 +290,13 @@ func SegmentTCP(pkt []byte, hdrLenU, csumStartU, gsoSizeU uint16, yield func(seg
return nil
}
// SegmentUDP walks a USO superpacket, sliding a per-segment-patched
// L3+L4 header into pkt at offset i*gsoSize and yielding pkt[i*G:i*G+segLen]
// to the caller. Per-segment patches are total_len + IPv4 csum (or IPv6
// payload_len) plus the UDP length and checksum. pkt is consumed
// destructively; see SegmentTCP for the layout reasoning.
// SegmentUDP walks a USO superpacket, stamping a per-segment-patched copy of
// the original L3+L4 header into pkt at offset i*gsoSize and yielding
// pkt[i*G:i*G+segLen] to the caller. Per-segment patches are total_len +
// IPv4 csum (or IPv6 payload_len) plus the UDP length and checksum. pkt is
// consumed destructively; see SegmentTCP for the layout reasoning, including
// why the header is stamped from a pristine snapshot rather than pkt[:hdrLen]
// (correctness when gsoSize < hdrLen).
//
// UDP-GSO leaves the IPv4 ID identical across segments (the kernel does not
// bump it), which is why the IP-level per-segment work is limited to
@@ -289,6 +312,9 @@ func SegmentUDP(pkt []byte, hdrLenU, csumStartU, gsoSizeU uint16, yield func(seg
isV4 := pkt[0]>>4 == 4
headerLen := int(hdrLenU)
csumStart := int(csumStartU)
if headerLen > maxSegHdrLen {
return fmt.Errorf("header len %d exceeds max %d", headerLen, maxSegHdrLen)
}
if headerLen-csumStart != udpHeaderLen {
return fmt.Errorf("udp header len mismatch: %d", headerLen-csumStart)
}
@@ -327,6 +353,12 @@ func SegmentUDP(pkt []byte, hdrLenU, csumStartU, gsoSizeU uint16, yield func(seg
baseIPHdrSum = uint32(checksum.Checksum(ipTmp[:ihl], 0))
}
// Snapshot the pristine L3+L4 header once and stamp every segment from
// it; see SegmentTCP for why sourcing from pkt[:headerLen] corrupts
// segments when gsoSize < headerLen.
var savedHdr [maxSegHdrLen]byte
copy(savedHdr[:headerLen], pkt[:headerLen])
for i := 0; i < numSeg; i++ {
segStart := i * gsoSize
segEnd := segStart + gsoSize
@@ -338,7 +370,7 @@ func SegmentUDP(pkt []byte, hdrLenU, csumStartU, gsoSizeU uint16, yield func(seg
headerOff := i * gsoSize
if i > 0 {
copy(pkt[headerOff:headerOff+headerLen], pkt[:headerLen])
copy(pkt[headerOff:headerOff+headerLen], savedHdr[:headerLen])
}
seg := pkt[headerOff : headerOff+segLen]