Files
nebula/iputil/packet_test.go
T
JackDoan aacb4e2334 datapath: fix 12 correctness findings from tun/UDP offload review
Multi-disciplinary correctness review of the batched tun / GSO-GRO / sendmmsg
rework. Each fix has a regression test; the merged tree builds on
linux/darwin/openbsd/windows/freebsd/netbsd, vets clean, passes the unit and
e2e suites, and is -race clean.

Critical:
- C1 zero-length inner UDP datagram no longer panics the process (remote DoS):
  the UDP coalescer routes payLen==0 to passthrough instead of seeding a GSO
  slot, and WriteGSO skips empty payload iovecs as defense in depth.
- C2 segmenter no longer corrupts inner headers when gsoSize < headerLen: the
  L3+L4 header is snapshotted once and each segment stamped from the copy,
  replacing the destructive overlapping in-place slide (SegmentTCP + SegmentUDP).

High:
- H1 applyOuterECN updates the IPv4 header checksum (RFC 1624 incremental) when
  folding outer CE into the inner ToS, so passthrough packets are no longer
  dropped by the peer stack.
- H2 the GRO reject path caps the borrowed RX segment ([:n:n]) so a reject can
  no longer overrun into the next coalesced segment's Nebula header. Note:
  oversized ICMPv6 rejects that need >16B beyond the segment are now refused
  rather than sent under GRO (safe; see TOFIX.md for the scratch-buffer follow-up).
- H3 WriteBatch falls back to per-packet WriteTo for a chunk when writeSockaddr
  fails, so one bad-family destination costs only its own packet, not the batch.
- H4 UserDevice.Readers returns N distinct queue wrappers with private buffers
  (sharing the pipes) so concurrent readers no longer race/overwrite borrowed
  packet bytes.
- H5 Poll.Close / Offload.Close no longer null t.fd (matching master's
  tunFile.Close), removing the data race with a concurrent readOne load.

Medium/Low:
- M1 the UDP GSO 127-segment gate moved from kernel >=5.5 to >=6.9 (the real
  UDP_MAX_SEGMENTS 64->128 threshold), avoiding EINVAL + per-packet fallback on
  5.5-6.8 kernels.
- M2 NewMultiQueueReader replays the offload mask newTun actually negotiated
  instead of the TSO-only mask, so adding a queue no longer disables USO
  device-wide; the advertised USO capability derives from the same mask.
- M3 the shutdown eventfd is closed in pollQueueSet.Close / offloadQueueSet.Close
  (double-close guarded), fixing the per-lifecycle fd leak.
- M4 dual-stack ECN selects the cmsg by address family, not socket family: RX
  parseRecvCmsg reads both IP_TOS and IPV6_TCLASS; TX writeEntryCmsg stamps
  IP_TOS for v4/v4-mapped dests and IPV6_TCLASS for v6 (on-host verified).
- L1 newPoll no longer closes the fd on failure (matching newOffload), removing
  the double-close on QueueSet.Add error.
2026-07-17 15:40:07 -05:00

523 lines
17 KiB
Go

package iputil
import (
"bytes"
"encoding/binary"
"net"
"testing"
"github.com/stretchr/testify/assert"
"golang.org/x/net/ipv4"
"golang.org/x/net/ipv6"
)
func Test_CreateRejectPacket(t *testing.T) {
h := ipv4.Header{
Len: 20,
Src: net.IPv4(10, 0, 0, 1),
Dst: net.IPv4(10, 0, 0, 2),
Protocol: 1, // ICMP
}
b, err := h.Marshal()
if err != nil {
t.Fatalf("h.Marhshal: %v", err)
}
b = append(b, []byte{0, 3, 0, 4}...)
expectedLen := ipv4.HeaderLen + 8 + h.Len + 4
out := make([]byte, expectedLen)
rejectPacket := CreateRejectPacket(b, out)
assert.NotNil(t, rejectPacket)
assert.Len(t, rejectPacket, expectedLen)
// ICMP with max header len
h = ipv4.Header{
Len: 60,
Src: net.IPv4(10, 0, 0, 1),
Dst: net.IPv4(10, 0, 0, 2),
Protocol: 1, // ICMP
Options: make([]byte, 40),
}
b, err = h.Marshal()
if err != nil {
t.Fatalf("h.Marhshal: %v", err)
}
b = append(b, []byte{0, 3, 0, 4, 0, 0, 0, 0}...)
expectedLen = maxIPv4RejectPacketSize
out = make([]byte, MaxRejectPacketSize)
rejectPacket = CreateRejectPacket(b, out)
assert.NotNil(t, rejectPacket)
assert.Len(t, rejectPacket, expectedLen)
// TCP with max header len
h = ipv4.Header{
Len: 60,
Src: net.IPv4(10, 0, 0, 1),
Dst: net.IPv4(10, 0, 0, 2),
Protocol: 6, // TCP
Options: make([]byte, 40),
}
b, err = h.Marshal()
if err != nil {
t.Fatalf("h.Marhshal: %v", err)
}
b = append(b, []byte{0, 3, 0, 4}...)
b = append(b, make([]byte, 16)...)
expectedLen = ipv4.HeaderLen + 20
out = make([]byte, expectedLen)
rejectPacket = CreateRejectPacket(b, out)
assert.NotNil(t, rejectPacket)
assert.Len(t, rejectPacket, expectedLen)
}
func Test_CreateRejectPacket_NoFragment(t *testing.T) {
out := make([]byte, MaxRejectPacketSize)
// IPv4: non-zero fragment offset should not generate reject packet
h := ipv4.Header{
Len: 20,
Src: net.IPv4(10, 0, 0, 1),
Dst: net.IPv4(10, 0, 0, 2),
Protocol: 17, // UDP
}
b, err := h.Marshal()
if err != nil {
t.Fatalf("h.Marshal: %v", err)
}
b = append(b, make([]byte, 8)...)
// Set fragment offset to non-zero (byte 6-7, offset in 8-byte units)
b[6] = 0x00
b[7] = 0x01
assert.Nil(t, CreateRejectPacket(b, out))
// MF flag with zero offset (first fragment) should still generate reject
b[6] = 0x20 // MF flag set
b[7] = 0x00
assert.NotNil(t, CreateRejectPacket(b, out))
// Non-fragment should still generate reject packet
b[6] = 0x00
b[7] = 0x00
assert.NotNil(t, CreateRejectPacket(b, out))
// DF flag only (not a fragment) should still generate reject packet
b[6] = 0x40
b[7] = 0x00
assert.NotNil(t, CreateRejectPacket(b, out))
}
func Test_CreateRejectPacketIPv6_NoFragment(t *testing.T) {
src := net.ParseIP("fd00::1")
dst := net.ParseIP("fd00::2")
out := make([]byte, MaxRejectPacketSize)
// IPv6 with Fragment header and non-zero offset should not generate reject
fragHeader := []byte{
17, // next header: UDP
0, // reserved
0, 9, // fragment offset=1 (shifted left 3), M=1
0, 0, 0, 1, // identification
}
udpPayload := make([]byte, 8)
payload := append(fragHeader, udpPayload...)
packet := makeIPv6Packet(src, dst, 44, payload) // next header 44 = Fragment
assert.Nil(t, CreateRejectPacket(packet, out))
// Fragment header with zero offset (first fragment) should still generate reject
fragHeader[2] = 0
fragHeader[3] = 1 // offset=0, M=1
payload = append(fragHeader, udpPayload...)
packet = makeIPv6Packet(src, dst, 44, payload)
assert.NotNil(t, CreateRejectPacket(packet, out))
}
func Test_CreateRejectPacket_NoICMPError(t *testing.T) {
out := make([]byte, MaxRejectPacketSize)
// ICMP error types should not generate reject packets
icmpErrorTypes := []byte{3, 4, 5, 11, 12}
for _, icmpType := range icmpErrorTypes {
h := ipv4.Header{
Len: 20,
Src: net.IPv4(10, 0, 0, 1),
Dst: net.IPv4(10, 0, 0, 2),
Protocol: 1, // ICMP
}
b, err := h.Marshal()
if err != nil {
t.Fatalf("h.Marshal: %v", err)
}
b = append(b, icmpType, 0, 0, 0, 0, 0, 0, 0)
rejectPacket := CreateRejectPacket(b, out)
assert.Nil(t, rejectPacket, "ICMP type %d should not generate a reject packet", icmpType)
}
// ICMP non-error types should still generate reject packets
icmpNonErrorTypes := []byte{0, 8, 13, 14}
for _, icmpType := range icmpNonErrorTypes {
h := ipv4.Header{
Len: 20,
Src: net.IPv4(10, 0, 0, 1),
Dst: net.IPv4(10, 0, 0, 2),
Protocol: 1, // ICMP
}
b, err := h.Marshal()
if err != nil {
t.Fatalf("h.Marshal: %v", err)
}
b = append(b, icmpType, 0, 0, 0, 0, 0, 0, 0)
rejectPacket := CreateRejectPacket(b, out)
assert.NotNil(t, rejectPacket, "ICMP type %d should generate a reject packet", icmpType)
}
}
// Test_CreateRejectPacket_RespectsCap guards against H2: with UDP GRO the
// scratch buffer reused to build a reject is a single coalesced segment inside
// a shared recvmmsg row. Its length covers just that segment, but an uncapped
// slice's capacity runs on into the next, not-yet-processed segment. Because
// CreateRejectPacket honors cap, capping the borrowed segment to its own length
// (cap==len) makes it physically impossible for an oversized ICMPv6 reject to
// overwrite the neighbor segment's bytes.
func Test_CreateRejectPacket_RespectsCap(t *testing.T) {
src := net.ParseIP("fd00::1")
dst := net.ParseIP("fd00::2")
// Inner IPv6 UDP packet. An ICMPv6 reject copies the whole inner packet
// plus a 48-byte header (40 IPv6 + 8 ICMPv6), so it needs 48 more bytes
// than the inner packet length.
inner := makeIPv6Packet(src, dst, 17, make([]byte, 20))
// The ciphertext scratch reused as the reject buffer is the received
// datagram: 16-byte Nebula header + inner + 16-byte AEAD tag. That is only
// 32 bytes of slack, so a full ICMPv6 reject overruns it by 16 bytes.
const nebulaOverhead = 32
segLen := len(inner) + nebulaOverhead
// Shared backing row laid out as [segment][neighbor's 16-byte Nebula header].
const neighborHdr = 16
sentinel := bytes.Repeat([]byte{0xAB}, neighborHdr)
// Uncapped: the slice's capacity reaches into the neighbor, reproducing
// the overrun that silently drops the neighbor packet.
backing := make([]byte, segLen+neighborHdr)
copy(backing[segLen:], sentinel)
reject := CreateRejectPacket(inner, backing[:segLen])
assert.NotNil(t, reject, "uncapped buffer reaches into the neighbor, so the reject is built")
assert.NotEqual(t, sentinel, backing[segLen:segLen+neighborHdr],
"without the cap the oversized reject overruns into the neighbor segment")
// Capped (the fix): cap==len, so the builder cannot exceed the segment. The
// reject does not fit, so it is refused rather than corrupting the neighbor.
backing = make([]byte, segLen+neighborHdr)
copy(backing[segLen:], sentinel)
reject = CreateRejectPacket(inner, backing[:segLen:segLen])
assert.Nil(t, reject, "capped segment is 16 bytes too small for a full ICMPv6 reject, so it is refused")
assert.Equal(t, sentinel, backing[segLen:segLen+neighborHdr],
"capped segment must leave the neighbor untouched")
}
func makeIPv6Packet(src, dst net.IP, nextHeader uint8, payload []byte) []byte {
b := make([]byte, ipv6.HeaderLen+len(payload))
b[0] = ipv6.Version << 4
binary.BigEndian.PutUint16(b[4:], uint16(len(payload)))
b[6] = nextHeader
b[7] = 64
copy(b[8:24], src.To16())
copy(b[24:40], dst.To16())
copy(b[ipv6.HeaderLen:], payload)
return b
}
func Test_CreateRejectPacketIPv6_ICMP(t *testing.T) {
src := net.ParseIP("fd00::1")
dst := net.ParseIP("fd00::2")
// Small UDP packet: entire original included in body
udpPayload := make([]byte, 20)
udpPayload[0] = 0x00 // src port high
udpPayload[1] = 0x50 // src port low (80)
udpPayload[2] = 0x01 // dst port high
udpPayload[3] = 0xBB // dst port low (443)
packet := makeIPv6Packet(src, dst, 17, udpPayload)
out := make([]byte, MaxRejectPacketSize)
rejectPacket := CreateRejectPacket(packet, out)
assert.NotNil(t, rejectPacket)
// Small packet fits entirely: 40 (ipv6 hdr) + 8 (icmpv6 hdr) + 60 (original)
expectedLen := ipv6.HeaderLen + 8 + len(packet)
assert.Len(t, rejectPacket, expectedLen)
// Verify version
assert.Equal(t, byte(ipv6.Version<<4), rejectPacket[0]&0xf0)
// Verify next header is ICMPv6 (58)
assert.Equal(t, byte(58), rejectPacket[6])
// Verify src/dst are swapped
assert.Equal(t, dst.To16(), net.IP(rejectPacket[8:24]))
assert.Equal(t, src.To16(), net.IP(rejectPacket[24:40]))
// Verify ICMPv6 type=1 (Dest Unreachable), code=1 (Administratively prohibited)
assert.Equal(t, byte(1), rejectPacket[ipv6.HeaderLen])
assert.Equal(t, byte(1), rejectPacket[ipv6.HeaderLen+1])
// Verify entire original packet is included in body
assert.Equal(t, packet, rejectPacket[ipv6.HeaderLen+8:])
// Large packet: body is truncated to 1000 bytes
largePkt := makeIPv6Packet(src, dst, 17, make([]byte, 1200))
rejectPacket = CreateRejectPacket(largePkt, out)
assert.NotNil(t, rejectPacket)
assert.Len(t, rejectPacket, ipv6.HeaderLen+8+1000)
assert.Equal(t, largePkt[:1000], rejectPacket[ipv6.HeaderLen+8:])
}
func Test_CreateRejectPacketIPv6_TCP(t *testing.T) {
src := net.ParseIP("fd00::1")
dst := net.ParseIP("fd00::2")
// TCP SYN packet (next header 6)
tcpPayload := make([]byte, 20)
tcpPayload[0] = 0x00 // src port high
tcpPayload[1] = 0x50 // src port low (80)
tcpPayload[2] = 0x01 // dst port high
tcpPayload[3] = 0xBB // dst port low (443)
binary.BigEndian.PutUint32(tcpPayload[4:], 1000) // seq
binary.BigEndian.PutUint32(tcpPayload[8:], 0) // ack seq
tcpPayload[12] = (20 >> 2) << 4 // data offset
tcpPayload[13] = 0b00000010 // SYN flag
packet := makeIPv6Packet(src, dst, 6, tcpPayload)
out := make([]byte, MaxRejectPacketSize)
rejectPacket := CreateRejectPacket(packet, out)
assert.NotNil(t, rejectPacket)
// Expected: 40 (ipv6 hdr) + 20 (tcp RST)
expectedLen := ipv6.HeaderLen + 20
assert.Len(t, rejectPacket, expectedLen)
// Verify version
assert.Equal(t, byte(ipv6.Version<<4), rejectPacket[0]&0xf0)
// Verify next header is TCP (6)
assert.Equal(t, byte(6), rejectPacket[6])
// Verify src/dst are swapped
assert.Equal(t, dst.To16(), net.IP(rejectPacket[8:24]))
assert.Equal(t, src.To16(), net.IP(rejectPacket[24:40]))
// Verify ports are swapped
tcpOut := rejectPacket[ipv6.HeaderLen:]
assert.Equal(t, uint16(443), binary.BigEndian.Uint16(tcpOut[0:2]))
assert.Equal(t, uint16(80), binary.BigEndian.Uint16(tcpOut[2:4]))
// RST+ACK flags (since input was SYN without ACK)
assert.Equal(t, byte(0b00010100), tcpOut[13])
// ack_seq = original seq (1000) + SYN (1) + FIN (0) + segment data (0)
assert.Equal(t, uint32(1001), binary.BigEndian.Uint32(tcpOut[8:]))
}
func Test_CreateRejectPacketIPv6_TCPWithACK(t *testing.T) {
src := net.ParseIP("fd00::1")
dst := net.ParseIP("fd00::2")
// TCP packet with ACK set
tcpPayload := make([]byte, 20)
tcpPayload[0] = 0x00
tcpPayload[1] = 0x50
tcpPayload[2] = 0x01
tcpPayload[3] = 0xBB
binary.BigEndian.PutUint32(tcpPayload[4:], 1000) // seq
binary.BigEndian.PutUint32(tcpPayload[8:], 2000) // ack seq
tcpPayload[12] = (20 >> 2) << 4 // data offset
tcpPayload[13] = 0b00010000 // ACK flag
packet := makeIPv6Packet(src, dst, 6, tcpPayload)
out := make([]byte, MaxRejectPacketSize)
rejectPacket := CreateRejectPacket(packet, out)
assert.NotNil(t, rejectPacket)
tcpOut := rejectPacket[ipv6.HeaderLen:]
// RST only (no ACK) since input had ACK
assert.Equal(t, byte(0b00000100), tcpOut[13])
// seq = original ack_seq
assert.Equal(t, uint32(2000), binary.BigEndian.Uint32(tcpOut[4:]))
}
func Test_CreateRejectPacketIPv6_NoICMPError(t *testing.T) {
src := net.ParseIP("fd00::1")
dst := net.ParseIP("fd00::2")
out := make([]byte, MaxRejectPacketSize)
// ICMPv6 error types (1-4) should not generate reject packets
for icmpType := byte(1); icmpType <= 4; icmpType++ {
payload := make([]byte, 8)
payload[0] = icmpType
packet := makeIPv6Packet(src, dst, 58, payload)
rejectPacket := CreateRejectPacket(packet, out)
assert.Nil(t, rejectPacket, "ICMPv6 type %d should not generate a reject packet", icmpType)
}
// ICMPv6 non-error types should still generate reject packets
nonErrorTypes := []byte{128, 129, 133, 134}
for _, icmpType := range nonErrorTypes {
payload := make([]byte, 8)
payload[0] = icmpType
packet := makeIPv6Packet(src, dst, 58, payload)
rejectPacket := CreateRejectPacket(packet, out)
assert.NotNil(t, rejectPacket, "ICMPv6 type %d should generate a reject packet", icmpType)
}
}
func Test_CreateRejectPacketIPv6_TooShort(t *testing.T) {
// Packet too short to be valid IPv6
out := make([]byte, MaxRejectPacketSize)
assert.Nil(t, CreateRejectPacket([]byte{0x60}, out))
assert.Nil(t, CreateRejectPacket(make([]byte, 39), out))
}
func Test_CreateRejectPacketIPv6_ExtensionHeaders(t *testing.T) {
src := net.ParseIP("fd00::1")
dst := net.ParseIP("fd00::2")
// IPv6 + Hop-by-Hop extension header + TCP
hopByHop := []byte{
6, // next header: TCP
0, // length (8 bytes total)
0, 0, // padding
0, 0, 0, 0,
}
tcpPayload := make([]byte, 20)
tcpPayload[0] = 0x00
tcpPayload[1] = 0x50
tcpPayload[2] = 0x01
tcpPayload[3] = 0xBB
binary.BigEndian.PutUint32(tcpPayload[4:], 1000)
binary.BigEndian.PutUint32(tcpPayload[8:], 2000)
tcpPayload[12] = (20 >> 2) << 4
tcpPayload[13] = 0b00010000 // ACK
payload := append(hopByHop, tcpPayload...)
packet := makeIPv6Packet(src, dst, 0, payload) // next header 0 = Hop-by-Hop
out := make([]byte, MaxRejectPacketSize)
rejectPacket := CreateRejectPacket(packet, out)
assert.NotNil(t, rejectPacket)
// Should produce TCP RST
expectedLen := ipv6.HeaderLen + 20
assert.Len(t, rejectPacket, expectedLen)
assert.Equal(t, byte(6), rejectPacket[6]) // next header is TCP
tcpOut := rejectPacket[ipv6.HeaderLen:]
assert.Equal(t, byte(0b00000100), tcpOut[13]) // RST only
}
func TestCreateICMPEchoResponse_IPv4(t *testing.T) {
// Build a simple IPv4 ICMP Echo Request
packet := make([]byte, 28)
packet[0] = 0x45 // version 4, IHL 5
binary.BigEndian.PutUint16(packet[2:], uint16(28)) // total length
packet[8] = 64 // TTL
packet[9] = 1 // protocol ICMP
copy(packet[12:16], net.IPv4(10, 0, 0, 1).To4()) // src
copy(packet[16:20], net.IPv4(10, 0, 0, 2).To4()) // dst
packet[20] = 8 // ICMP Echo Request
out := make([]byte, len(packet))
result := CreateICMPEchoResponse(packet, out)
assert.NotNil(t, result)
assert.Equal(t, byte(0x45), result[0])
// src/dst swapped
assert.Equal(t, net.IPv4(10, 0, 0, 2).To4(), net.IP(result[12:16]))
assert.Equal(t, net.IPv4(10, 0, 0, 1).To4(), net.IP(result[16:20]))
// ICMP Echo Reply
assert.Equal(t, byte(0), result[20])
}
func TestCreateICMPEchoResponse_IPv6(t *testing.T) {
src := net.ParseIP("fd00::1").To16()
dst := net.ParseIP("fd00::2").To16()
// Build an IPv6 ICMPv6 Echo Request packet
// IPv6 header (40 bytes) + ICMPv6 (8 bytes)
packet := make([]byte, 48)
packet[0] = 0x60 // version 6
payloadLen := uint16(8) // ICMPv6 header only
binary.BigEndian.PutUint16(packet[4:], payloadLen)
packet[6] = 58 // Next Header: ICMPv6
packet[7] = 64 // Hop Limit
copy(packet[8:24], src) // src address
copy(packet[24:40], dst) // dst address
// ICMPv6 Echo Request
icmp := packet[40:]
icmp[0] = 128 // type: Echo Request
icmp[1] = 0 // code
binary.BigEndian.PutUint16(icmp[4:], 1) // identifier
binary.BigEndian.PutUint16(icmp[6:], 1) // sequence number
// Compute correct checksum for the request
csum := ipv6PseudoheaderChecksum(src, dst, 58, uint32(payloadLen))
binary.BigEndian.PutUint16(icmp[2:], tcpipChecksum(icmp, csum))
out := make([]byte, len(packet))
result := CreateICMPEchoResponse(packet, out)
assert.NotNil(t, result)
// Version should still be 6
assert.Equal(t, byte(6), result[0]>>4)
// src/dst swapped
assert.Equal(t, dst, net.IP(result[8:24]))
assert.Equal(t, src, net.IP(result[24:40]))
// ICMPv6 Echo Reply type
assert.Equal(t, byte(129), result[40])
// Verify checksum is valid (tcpipChecksum returns 0 when data+checksum is correct)
respIcmp := result[40:]
verifyCsum := ipv6PseudoheaderChecksum(result[8:24], result[24:40], 58, uint32(payloadLen))
assert.Equal(t, uint16(0), tcpipChecksum(respIcmp, verifyCsum))
}
func TestCreateICMPEchoResponse_IPv6_NotEchoRequest(t *testing.T) {
src := net.ParseIP("fd00::1").To16()
dst := net.ParseIP("fd00::2").To16()
packet := make([]byte, 48)
packet[0] = 0x60
binary.BigEndian.PutUint16(packet[4:], 8)
packet[6] = 58
packet[7] = 64
copy(packet[8:24], src)
copy(packet[24:40], dst)
// ICMPv6 type 1 (Destination Unreachable) - not Echo Request
packet[40] = 1
out := make([]byte, len(packet))
result := CreateICMPEchoResponse(packet, out)
assert.Nil(t, result)
}
func TestCreateICMPEchoResponse_IPv6_NotICMPv6(t *testing.T) {
src := net.ParseIP("fd00::1").To16()
dst := net.ParseIP("fd00::2").To16()
packet := make([]byte, 48)
packet[0] = 0x60
binary.BigEndian.PutUint16(packet[4:], 8)
packet[6] = 6 // TCP, not ICMPv6
packet[7] = 64
copy(packet[8:24], src)
copy(packet[24:40], dst)
out := make([]byte, len(packet))
result := CreateICMPEchoResponse(packet, out)
assert.Nil(t, result)
}