mirror of
https://github.com/slackhq/nebula.git
synced 2026-10-07 11:27:54 +02:00
the definitive tun offloads branch (#1704)
This commit is contained in:
Jack Doan
@@ -0,0 +1,75 @@
|
||||
//go:build linux && !android
|
||||
// +build linux,!android
|
||||
|
||||
package virtio
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// Size is the on-wire length of struct virtio_net_hdr the kernel
|
||||
// prepends/expects on a TUN opened with IFF_VNET_HDR (TUNSETVNETHDRSZ
|
||||
// not set).
|
||||
const Size = 10
|
||||
|
||||
// Hdr is the Go view of the legacy virtio_net_hdr.
|
||||
type Hdr struct {
|
||||
Flags uint8
|
||||
gsoType uint8 //private to avoid mistakes wrt the 0x80 VIRTIO_NET_HDR_GSO_ECN flag, ORed with the other "GSO types"
|
||||
HdrLen uint16
|
||||
GSOSize uint16
|
||||
CsumStart uint16
|
||||
CsumOffset uint16
|
||||
}
|
||||
|
||||
func NewHeader(flags, gsoType uint8, hdrLen, gsoSize, csumStart, csumOffset uint16) Hdr {
|
||||
return Hdr{
|
||||
Flags: flags,
|
||||
gsoType: gsoType,
|
||||
HdrLen: hdrLen,
|
||||
GSOSize: gsoSize,
|
||||
CsumStart: csumStart,
|
||||
CsumOffset: csumOffset,
|
||||
}
|
||||
}
|
||||
|
||||
// Decode reads a virtio_net_hdr in host byte order (TUN default; we never
|
||||
// call TUNSETVNETLE so the kernel matches our endianness).
|
||||
func (h *Hdr) Decode(b []byte) {
|
||||
h.Flags = b[0]
|
||||
h.gsoType = b[1]
|
||||
h.HdrLen = binary.NativeEndian.Uint16(b[2:4])
|
||||
h.GSOSize = binary.NativeEndian.Uint16(b[4:6])
|
||||
h.CsumStart = binary.NativeEndian.Uint16(b[6:8])
|
||||
h.CsumOffset = binary.NativeEndian.Uint16(b[8:10])
|
||||
}
|
||||
|
||||
func EncodeHeader(b []byte, flags, gsoType uint8, hdrLen, gsoSize, csumStart, csumOffset uint16) {
|
||||
b[0] = flags
|
||||
b[1] = gsoType
|
||||
binary.NativeEndian.PutUint16(b[2:4], hdrLen)
|
||||
binary.NativeEndian.PutUint16(b[4:6], gsoSize)
|
||||
binary.NativeEndian.PutUint16(b[6:8], csumStart)
|
||||
binary.NativeEndian.PutUint16(b[8:10], csumOffset)
|
||||
}
|
||||
|
||||
// Encode is the inverse of Decode: writes the virtio_net_hdr fields into b
|
||||
// (must be at least Size bytes). Used to emit a TSO superpacket on egress.
|
||||
func (h *Hdr) Encode(b []byte) {
|
||||
EncodeHeader(b, h.Flags, h.gsoType, h.HdrLen, h.GSOSize, h.CsumStart, h.CsumOffset)
|
||||
}
|
||||
|
||||
// GSOType returns gsoType with the ECN-flag masked out
|
||||
func (h *Hdr) GSOType() uint8 {
|
||||
return h.gsoType &^ unix.VIRTIO_NET_HDR_GSO_ECN
|
||||
}
|
||||
|
||||
func (h *Hdr) HasECNFlag() bool {
|
||||
return h.gsoType&unix.VIRTIO_NET_HDR_GSO_ECN != 0
|
||||
}
|
||||
|
||||
func (h *Hdr) SetGSOType(x uint8) {
|
||||
h.gsoType = x
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
//go:build !linux || android
|
||||
|
||||
package virtio
|
||||
@@ -0,0 +1,441 @@
|
||||
//go:build linux && !android
|
||||
// +build linux,!android
|
||||
|
||||
// Package virtio implements the pure validation, header-correction, and
|
||||
// per-segment slicing logic for kernel-supplied TSO/USO superpackets on
|
||||
// IFF_VNET_HDR TUN devices. It is FD-free and depends only on the byte
|
||||
// layout of the virtio_net_hdr and the IP/TCP/UDP headers it describes,
|
||||
// so it can be unit-tested in isolation from the tio Queue runtime.
|
||||
package virtio
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/slackhq/nebula/overlay/checksum"
|
||||
)
|
||||
|
||||
// Protocol header size bounds used to validate / cap kernel-supplied offsets.
|
||||
const (
|
||||
ipv4HeaderMinLen = 20 // IHL=5, no options
|
||||
ipv4HeaderMaxLen = 60 // IHL=15, max options
|
||||
ipv6FixedLen = 40 // IPv6 base header; extensions would extend this
|
||||
tcpHeaderMinLen = 20 // data-offset=5, no options
|
||||
tcpHeaderMaxLen = 60 // data-offset=15, max options
|
||||
)
|
||||
|
||||
// maxSegHdrLen bounds the L3+L4 header we snapshot before stamping each segment.
|
||||
// The largest header the segmenter supports is IPv4 (max IHL 60) plus TCP (max data-offset 60) = 120 bytes
|
||||
const maxSegHdrLen = ipv4HeaderMaxLen + tcpHeaderMaxLen // 120
|
||||
|
||||
// Byte offsets inside an IPv4 header.
|
||||
const (
|
||||
ipv4TotalLenOff = 2
|
||||
ipv4IDOff = 4
|
||||
ipv4ChecksumOff = 10
|
||||
ipv4SrcOff = 12
|
||||
ipv4AddrsEnd = 20 // end of dst address (ipv4SrcOff + 2*4)
|
||||
)
|
||||
|
||||
// Byte offsets inside an IPv6 header.
|
||||
const (
|
||||
ipv6PayloadLenOff = 4
|
||||
ipv6SrcOff = 8
|
||||
ipv6AddrsEnd = 40 // end of dst address (ipv6SrcOff + 2*16)
|
||||
)
|
||||
|
||||
// Byte offsets inside a TCP header (relative to its start, i.e. csumStart).
|
||||
const (
|
||||
tcpSeqOff = 4
|
||||
tcpDataOffOff = 12 // upper nibble is header len in 32-bit words
|
||||
tcpFlagsOff = 13
|
||||
tcpChecksumOff = 16
|
||||
)
|
||||
|
||||
// UDP header is fixed at 8 bytes: {sport, dport, length, checksum}.
|
||||
const (
|
||||
udpHeaderLen = 8
|
||||
udpLengthOff = 4
|
||||
udpChecksumOff = 6
|
||||
)
|
||||
|
||||
var errPacketTooShort = errors.New("packet too short")
|
||||
|
||||
// tcpFinPshMask is cleared on every segment except the last of a TSO burst.
|
||||
const tcpFinPshMask = 0x09 // FIN(0x01) | PSH(0x08)
|
||||
|
||||
// tcpCwrFlag is cleared on every segment except the first.
|
||||
// Per RFC 3168 §6.1.2 the CWR bit signals a one-shot transition (the sender just halved its window)
|
||||
// and must appear on the first segment of a TSO burst only.
|
||||
const tcpCwrFlag = 0x80
|
||||
|
||||
// CheckValid rejects packets whose virtio_net_hdr/IP combination would
|
||||
// cause a downstream miscompute. The TUN should never emit RSC_INFO and
|
||||
// the GSO type must agree with the IP version nibble.
|
||||
func CheckValid(pkt []byte, hdr Hdr) error {
|
||||
if hdr.Flags&unix.VIRTIO_NET_HDR_F_RSC_INFO != 0 {
|
||||
return fmt.Errorf("virtio RSC_INFO flag not supported on TUN reads")
|
||||
}
|
||||
if len(pkt) < ipv4HeaderMinLen {
|
||||
return errPacketTooShort
|
||||
}
|
||||
ipVersion := pkt[0] >> 4
|
||||
if ipVersion == 6 && len(pkt) < ipv6FixedLen {
|
||||
return errPacketTooShort
|
||||
}
|
||||
|
||||
gsoType := hdr.GSOType()
|
||||
if gsoType != unix.VIRTIO_NET_HDR_GSO_NONE && hdr.GSOSize == 0 {
|
||||
// A GSO type with no segment size would dodge IsSuperpacket() downstream and
|
||||
// travel as a plain jumbo datagram with an unfinished checksum.
|
||||
return fmt.Errorf("virtio GSO type %#x with zero gso_size", hdr.gsoType)
|
||||
}
|
||||
if hdr.HasECNFlag() && !(gsoType == unix.VIRTIO_NET_HDR_GSO_TCPV4 || gsoType == unix.VIRTIO_NET_HDR_GSO_TCPV6) {
|
||||
return fmt.Errorf("virtio GSO_ECN qualifier on non-TCP GSO type %#x", hdr.gsoType)
|
||||
}
|
||||
switch gsoType {
|
||||
case unix.VIRTIO_NET_HDR_GSO_TCPV4:
|
||||
if ipVersion != 4 {
|
||||
return fmt.Errorf("invalid IP version %d for GSO type %d", ipVersion, hdr.gsoType)
|
||||
}
|
||||
case unix.VIRTIO_NET_HDR_GSO_TCPV6:
|
||||
if ipVersion != 6 {
|
||||
return fmt.Errorf("invalid IP version %d for GSO type %d", ipVersion, hdr.gsoType)
|
||||
}
|
||||
case unix.VIRTIO_NET_HDR_GSO_UDP_L4:
|
||||
// USO carries either v4 or v6; the leading nibble disambiguates.
|
||||
if !(ipVersion == 4 || ipVersion == 6) {
|
||||
return fmt.Errorf("invalid IP version %d for GSO type %d", ipVersion, hdr.gsoType)
|
||||
}
|
||||
default:
|
||||
if !(ipVersion == 6 || ipVersion == 4) {
|
||||
return fmt.Errorf("invalid IP version %d for GSO type %d", ipVersion, hdr.gsoType)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// CorrectHdrLen rewrites hdr.HdrLen based on the actual transport header length read out of pkt.
|
||||
// The kernel's hdr.HdrLen on the FORWARD path can be the length of the entire first packet, so we don't trust it.
|
||||
func CorrectHdrLen(pkt []byte, hdr *Hdr) error {
|
||||
// Thank you wireguard-go for documenting these edge-cases
|
||||
// Don't trust hdr.hdrLen from the kernel as it can be equal to the length
|
||||
// of the entire first packet when the kernel is handling it as part of a FORWARD path.
|
||||
// Instead, parse the transport header length and add it onto csumStart, which is synonymous for IP header length.
|
||||
|
||||
if hdr.GSOType() == unix.VIRTIO_NET_HDR_GSO_UDP_L4 {
|
||||
hdr.HdrLen = hdr.CsumStart + 8
|
||||
} else {
|
||||
if len(pkt) <= int(hdr.CsumStart+tcpDataOffOff) {
|
||||
return errors.New("packet is too short")
|
||||
}
|
||||
|
||||
tcpHLen := uint16(pkt[hdr.CsumStart+tcpDataOffOff] >> 4 * 4)
|
||||
if tcpHLen < tcpHeaderMinLen || tcpHLen > tcpHeaderMaxLen {
|
||||
return fmt.Errorf("tcp header len is invalid: %d", tcpHLen)
|
||||
}
|
||||
hdr.HdrLen = hdr.CsumStart + tcpHLen
|
||||
}
|
||||
|
||||
if len(pkt) < int(hdr.HdrLen) {
|
||||
return fmt.Errorf("length of packet (%d) < virtioNetHdr.HdrLen (%d)", len(pkt), hdr.HdrLen)
|
||||
}
|
||||
|
||||
if hdr.HdrLen < hdr.CsumStart {
|
||||
return fmt.Errorf("virtioNetHdr.HdrLen (%d) < virtioNetHdr.CsumStart (%d)", hdr.HdrLen, hdr.CsumStart)
|
||||
}
|
||||
cSumAt := int(hdr.CsumStart + hdr.CsumOffset)
|
||||
if cSumAt+1 >= len(pkt) {
|
||||
return fmt.Errorf("end of checksum offset (%d) exceeds packet length (%d)", cSumAt+1, len(pkt))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// segCount returns how many segments a payload of payLen bytes splits into at gsoSize,
|
||||
// with a floor of one so a header-only superpacket still yields a single segment.
|
||||
func segCount(payLen, gsoSize int) int {
|
||||
n := (payLen + gsoSize - 1) / gsoSize
|
||||
if n == 0 {
|
||||
return 1
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// basePseudoSum folds the part of the L4 pseudo-header sum that is identical
|
||||
// for every segment: the source and destination addresses plus the protocol
|
||||
// number. The per-segment L4 length is added by the caller inside the loop.
|
||||
func basePseudoSum(pkt []byte, isV4 bool, proto uint32) uint32 {
|
||||
if isV4 {
|
||||
return uint32(checksum.Checksum(pkt[ipv4SrcOff:ipv4AddrsEnd], 0)) + proto
|
||||
}
|
||||
return uint32(checksum.Checksum(pkt[ipv6SrcOff:ipv6AddrsEnd], 0)) + proto
|
||||
}
|
||||
|
||||
// baseIPv4HdrSum folds the IPv4 header checksum over the fields that stay constant across segments.
|
||||
// csumStart is the L3 header length, which bounds a valid IHL.
|
||||
func baseIPv4HdrSum(pkt []byte, csumStart int) (uint32, error) {
|
||||
ihl := int(pkt[0]&0x0f) * 4
|
||||
if ihl < ipv4HeaderMinLen || ihl > csumStart {
|
||||
return 0, fmt.Errorf("bad IPv4 IHL: %d", ihl)
|
||||
}
|
||||
// total_len, the ID, and the checksum field itself are excluded: all three are rewritten per segment.
|
||||
sum := uint32(checksum.Checksum(pkt[:ihl], 0))
|
||||
sum += uint32(^binary.BigEndian.Uint16(pkt[ipv4TotalLenOff : ipv4TotalLenOff+2]))
|
||||
sum += uint32(^binary.BigEndian.Uint16(pkt[ipv4ChecksumOff : ipv4ChecksumOff+2]))
|
||||
sum += uint32(^binary.BigEndian.Uint16(pkt[ipv4IDOff : ipv4IDOff+2]))
|
||||
sum = (sum & 0xffff) + (sum >> 16)
|
||||
sum = (sum & 0xffff) + (sum >> 16)
|
||||
return sum, nil
|
||||
}
|
||||
|
||||
// baseTCPHdrSum folds the TCP header checksum over everything the segment loop does not rewrite
|
||||
func baseTCPHdrSum(pkt []byte, csumStart, headerLen int) uint32 {
|
||||
seq := binary.BigEndian.Uint32(pkt[csumStart+tcpSeqOff : csumStart+tcpSeqOff+4])
|
||||
flags := uint16(pkt[csumStart+tcpFlagsOff])
|
||||
|
||||
sum := uint32(checksum.Checksum(pkt[csumStart:headerLen], 0))
|
||||
sum += uint32(^uint16(seq >> 16))
|
||||
sum += uint32(^uint16(seq))
|
||||
sum += uint32(^flags)
|
||||
sum += uint32(^binary.BigEndian.Uint16(pkt[csumStart+tcpChecksumOff : csumStart+tcpChecksumOff+2]))
|
||||
sum = (sum & 0xffff) + (sum >> 16)
|
||||
sum = (sum & 0xffff) + (sum >> 16)
|
||||
return sum
|
||||
}
|
||||
|
||||
// SegmentTCP walks a TSO superpacket pkt, yielding each segment as a slice into pkt.
|
||||
// Per-segment plaintext is laid out by stamping a copy of the original L3+L4 header into pkt at offset i*gsoSize,
|
||||
// where it sits immediately before that segment's payload chunk in the original buffer.
|
||||
// pkt is consumed by this call and must not be inspected by the caller after the final yield.
|
||||
func SegmentTCP(pkt []byte, hdrLenU, csumStartU, gsoSizeU uint16, yield func(seg []byte) error) error {
|
||||
if gsoSizeU == 0 {
|
||||
return fmt.Errorf("gso_size is zero")
|
||||
}
|
||||
if csumStartU == 0 {
|
||||
return fmt.Errorf("csum_start is zero")
|
||||
}
|
||||
|
||||
headerLen := int(hdrLenU)
|
||||
csumStart := int(csumStartU)
|
||||
if headerLen > maxSegHdrLen {
|
||||
return fmt.Errorf("header len %d exceeds max %d", headerLen, maxSegHdrLen)
|
||||
}
|
||||
isV4 := pkt[0]>>4 == 4
|
||||
|
||||
tcpHdrLen := int(pkt[csumStart+tcpDataOffOff]>>4) * 4
|
||||
payLen := len(pkt) - headerLen
|
||||
gsoSize := int(gsoSizeU)
|
||||
numSeg := segCount(payLen, gsoSize)
|
||||
|
||||
origSeq := binary.BigEndian.Uint32(pkt[csumStart+tcpSeqOff : csumStart+tcpSeqOff+4])
|
||||
origFlags := pkt[csumStart+tcpFlagsOff]
|
||||
|
||||
baseProtoSum := basePseudoSum(pkt, isV4, unix.IPPROTO_TCP)
|
||||
baseTcpHdrSum := baseTCPHdrSum(pkt, csumStart, headerLen)
|
||||
|
||||
var origIPID uint16
|
||||
var baseIPHdrSum uint32
|
||||
if isV4 {
|
||||
origIPID = binary.BigEndian.Uint16(pkt[ipv4IDOff : ipv4IDOff+2])
|
||||
var err error
|
||||
// TSO bumps the ID per segment, so it stays out of the base sum.
|
||||
baseIPHdrSum, err = baseIPv4HdrSum(pkt, csumStart)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Snapshot the pristine L3+L4 header once. '
|
||||
// Every segment's header is stamped from this copy, so overlapping stamps (gsoSize < headerLen) can never corrupt the source.
|
||||
var savedHdr [maxSegHdrLen]byte
|
||||
copy(savedHdr[:headerLen], pkt[:headerLen])
|
||||
|
||||
for i := 0; i < numSeg; i++ {
|
||||
segStart := i * gsoSize
|
||||
segEnd := segStart + gsoSize
|
||||
if segEnd > payLen {
|
||||
segEnd = payLen
|
||||
}
|
||||
segPayLen := segEnd - segStart
|
||||
segLen := headerLen + segPayLen
|
||||
headerOff := i * gsoSize
|
||||
|
||||
// Stamp the header into place immediately before this segment's payload, sourced from the snapshot.
|
||||
// The per-segment patches below overwrite the variable fields. (seq/flags/cksum/totalLen/id)
|
||||
if i > 0 {
|
||||
// Iter 0's header is already at pkt[:headerLen] (identical to savedHdr), so only i >= 1 needs the stamp
|
||||
copy(pkt[headerOff:headerOff+headerLen], savedHdr[:headerLen])
|
||||
}
|
||||
seg := pkt[headerOff : headerOff+segLen]
|
||||
|
||||
segSeq := origSeq + uint32(segStart)
|
||||
segFlags := origFlags
|
||||
if i != 0 {
|
||||
segFlags &^= tcpCwrFlag
|
||||
}
|
||||
if i != numSeg-1 {
|
||||
segFlags &^= tcpFinPshMask
|
||||
}
|
||||
totalLen := segLen
|
||||
|
||||
if isV4 {
|
||||
segID := origIPID + uint16(i)
|
||||
binary.BigEndian.PutUint16(seg[ipv4TotalLenOff:ipv4TotalLenOff+2], uint16(totalLen))
|
||||
binary.BigEndian.PutUint16(seg[ipv4IDOff:ipv4IDOff+2], segID)
|
||||
ipSum := baseIPHdrSum + uint32(totalLen) + uint32(segID)
|
||||
binary.BigEndian.PutUint16(seg[ipv4ChecksumOff:ipv4ChecksumOff+2], foldComplement(ipSum))
|
||||
} else {
|
||||
binary.BigEndian.PutUint16(seg[ipv6PayloadLenOff:ipv6PayloadLenOff+2], uint16(headerLen-ipv6FixedLen+segPayLen))
|
||||
}
|
||||
|
||||
binary.BigEndian.PutUint32(seg[csumStart+tcpSeqOff:csumStart+tcpSeqOff+4], segSeq)
|
||||
seg[csumStart+tcpFlagsOff] = segFlags
|
||||
|
||||
tcpLen := tcpHdrLen + segPayLen
|
||||
// Payload bytes still live at their original offset in pkt.
|
||||
// The header slide above only writes into pkt[i*GSOSize : i*GSOSize+header], which is the tail of seg_{i-1}'s payload (already consumed)
|
||||
// and never overlaps seg_i's own payload at pkt[header+i*GSOSize : header+(i+1)*GSOSize].
|
||||
paySum := uint32(checksum.Checksum(pkt[headerLen+segStart:headerLen+segEnd], 0))
|
||||
wide := uint64(baseTcpHdrSum) + uint64(paySum) + uint64(baseProtoSum)
|
||||
wide += uint64(segSeq) + uint64(segFlags) + uint64(tcpLen)
|
||||
wide = (wide & 0xffffffff) + (wide >> 32)
|
||||
wide = (wide & 0xffffffff) + (wide >> 32)
|
||||
binary.BigEndian.PutUint16(seg[csumStart+tcpChecksumOff:csumStart+tcpChecksumOff+2], foldComplement(uint32(wide)))
|
||||
|
||||
if err := yield(seg); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// SegmentUDP walks a USO superpacket, stamping a per-segment-patched copy of the original L3+L4 header
|
||||
// into pkt at offset i*GSOSize and yielding pkt[i*GSOSize:i*GSOSize+segLen] to the caller.
|
||||
// Per-segment patches are total_len + IPv4 csum (or IPv6 payload_len) plus the UDP length and checksum.
|
||||
// pkt is consumed destructively.
|
||||
func SegmentUDP(pkt []byte, hdrLenU, csumStartU, gsoSizeU uint16, yield func(seg []byte) error) error {
|
||||
if gsoSizeU == 0 {
|
||||
return fmt.Errorf("gso_size is zero")
|
||||
}
|
||||
if csumStartU == 0 {
|
||||
return fmt.Errorf("csum_start is zero")
|
||||
}
|
||||
|
||||
isV4 := pkt[0]>>4 == 4
|
||||
headerLen := int(hdrLenU)
|
||||
csumStart := int(csumStartU)
|
||||
if headerLen > maxSegHdrLen {
|
||||
return fmt.Errorf("header len %d exceeds max %d", headerLen, maxSegHdrLen)
|
||||
}
|
||||
if headerLen-csumStart != udpHeaderLen {
|
||||
return fmt.Errorf("udp header len mismatch: %d", headerLen-csumStart)
|
||||
}
|
||||
|
||||
payLen := len(pkt) - headerLen
|
||||
gsoSize := int(gsoSizeU)
|
||||
numSeg := segCount(payLen, gsoSize)
|
||||
|
||||
baseProtoSum := basePseudoSum(pkt, isV4, unix.IPPROTO_UDP)
|
||||
|
||||
var origIPID uint16
|
||||
var baseIPHdrSum uint32
|
||||
if isV4 {
|
||||
origIPID = binary.BigEndian.Uint16(pkt[ipv4IDOff : ipv4IDOff+2])
|
||||
var err error
|
||||
// Software UDP GSO bumps the ID per segment just like TSO
|
||||
// (inet_gso_segment's fixed-ID case is TCP-only), so it stays out of the base sum.
|
||||
baseIPHdrSum, err = baseIPv4HdrSum(pkt, csumStart)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Snapshot the pristine L3+L4 header once and stamp every segment from it
|
||||
var savedHdr [maxSegHdrLen]byte
|
||||
copy(savedHdr[:headerLen], pkt[:headerLen])
|
||||
|
||||
for i := 0; i < numSeg; i++ {
|
||||
segStart := i * gsoSize
|
||||
segEnd := segStart + gsoSize
|
||||
if segEnd > payLen {
|
||||
segEnd = payLen
|
||||
}
|
||||
segPayLen := segEnd - segStart
|
||||
segLen := headerLen + segPayLen
|
||||
headerOff := i * gsoSize
|
||||
|
||||
if i > 0 {
|
||||
copy(pkt[headerOff:headerOff+headerLen], savedHdr[:headerLen])
|
||||
}
|
||||
seg := pkt[headerOff : headerOff+segLen]
|
||||
|
||||
totalLen := segLen
|
||||
udpLen := udpHeaderLen + segPayLen
|
||||
|
||||
if isV4 {
|
||||
segID := origIPID + uint16(i)
|
||||
binary.BigEndian.PutUint16(seg[ipv4TotalLenOff:ipv4TotalLenOff+2], uint16(totalLen))
|
||||
binary.BigEndian.PutUint16(seg[ipv4IDOff:ipv4IDOff+2], segID)
|
||||
ipSum := baseIPHdrSum + uint32(totalLen) + uint32(segID)
|
||||
binary.BigEndian.PutUint16(seg[ipv4ChecksumOff:ipv4ChecksumOff+2], foldComplement(ipSum))
|
||||
} else {
|
||||
binary.BigEndian.PutUint16(seg[ipv6PayloadLenOff:ipv6PayloadLenOff+2], uint16(headerLen-ipv6FixedLen+segPayLen))
|
||||
}
|
||||
|
||||
binary.BigEndian.PutUint16(seg[csumStart+udpLengthOff:csumStart+udpLengthOff+2], uint16(udpLen))
|
||||
|
||||
// Sum the UDP header (length just written, checksum zeroed) together with
|
||||
// this segment's payload in one pass, seeded with the pseudo-header sum.
|
||||
seg[csumStart+udpChecksumOff], seg[csumStart+udpChecksumOff+1] = 0, 0
|
||||
pseudo := baseProtoSum + uint32(udpLen)
|
||||
pseudo = (pseudo & 0xffff) + (pseudo >> 16)
|
||||
pseudo = (pseudo & 0xffff) + (pseudo >> 16)
|
||||
csum := ^checksum.Checksum(seg[csumStart:], uint16(pseudo))
|
||||
if csum == 0 {
|
||||
csum = 0xffff
|
||||
}
|
||||
binary.BigEndian.PutUint16(seg[csumStart+udpChecksumOff:csumStart+udpChecksumOff+2], csum)
|
||||
|
||||
if err := yield(seg); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// FinishChecksum computes the L4 checksum for a non-GSO packet that the kernel handed us with NEEDS_CSUM set.
|
||||
// CsumStart / CsumOffset point at the 16-bit checksum field.
|
||||
// We zero it, fold a full sum from the partial one that the kernel provided, and store the result.
|
||||
func FinishChecksum(seg []byte, hdr Hdr) error {
|
||||
cs := int(hdr.CsumStart)
|
||||
co := int(hdr.CsumOffset)
|
||||
if cs+co+2 > len(seg) {
|
||||
return fmt.Errorf("csum offsets out of range: start=%d offset=%d len=%d", cs, co, len(seg))
|
||||
}
|
||||
// The kernel stores a partial pseudo-header sum at [cs+co:]; sum over the
|
||||
// L4 region starting at cs, folding the prior partial in as the seed.
|
||||
partial := binary.BigEndian.Uint16(seg[cs+co : cs+co+2])
|
||||
seg[cs+co] = 0
|
||||
seg[cs+co+1] = 0
|
||||
csum := ^checksum.Checksum(seg[cs:], partial)
|
||||
// RFC 768: UDP transmits a computed zero as all ones, since all-zero is the reserved "no checksum" value.
|
||||
if co == udpChecksumOff && csum == 0 {
|
||||
csum = 0xffff
|
||||
}
|
||||
binary.BigEndian.PutUint16(seg[cs+co:cs+co+2], csum)
|
||||
return nil
|
||||
}
|
||||
|
||||
// foldComplement folds a 32-bit one's-complement partial sum to 16 bits and
|
||||
// complements it, yielding the on-wire Internet checksum value.
|
||||
func foldComplement(sum uint32) uint16 {
|
||||
sum = (sum & 0xffff) + (sum >> 16)
|
||||
sum = (sum & 0xffff) + (sum >> 16)
|
||||
return ^uint16(sum)
|
||||
}
|
||||
@@ -0,0 +1,602 @@
|
||||
//go:build linux && !android
|
||||
// +build linux,!android
|
||||
|
||||
package virtio
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"testing"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
|
||||
"github.com/slackhq/nebula/overlay/checksum"
|
||||
)
|
||||
|
||||
// verifyChecksum confirms that the one's-complement sum across b, seeded with
|
||||
// a folded pseudo-header sum, equals all-ones (a valid on-wire checksum).
|
||||
// A corrupted header stamped into a segment makes this fail even when the
|
||||
// checksum field itself was computed from the (pristine) base sums, because
|
||||
// the bytes the receiver would sum no longer match what was checksummed.
|
||||
func verifyChecksum(b []byte, pseudo uint16) bool {
|
||||
return checksum.Checksum(b, pseudo) == 0xffff
|
||||
}
|
||||
|
||||
// pseudoHeaderIPv4 folds the TCP/UDP pseudo-header sum from a segment's own
|
||||
// address and length fields, used to independently verify its L4 checksum.
|
||||
func pseudoHeaderIPv4(src, dst []byte, proto byte, l4Len int) uint16 {
|
||||
s := uint32(checksum.Checksum(src, 0)) + uint32(checksum.Checksum(dst, 0))
|
||||
s += uint32(proto) + uint32(l4Len)
|
||||
s = (s & 0xffff) + (s >> 16)
|
||||
s = (s & 0xffff) + (s >> 16)
|
||||
return uint16(s)
|
||||
}
|
||||
|
||||
// buildTCPv4Super constructs a synthetic IPv4/TCP TSO superpacket with a
|
||||
// payload of payLen bytes and returns it alongside the header fields the
|
||||
// segmenter needs. The header is a fixed 40 bytes (20 IPv4 + 20 TCP).
|
||||
func buildTCPv4Super(payLen int) (pkt []byte, hdrLen, csumStart uint16) {
|
||||
const ipLen = 20
|
||||
const tcpLen = 20
|
||||
pkt = make([]byte, ipLen+tcpLen+payLen)
|
||||
|
||||
// IPv4 header.
|
||||
pkt[0] = 0x45 // version 4, IHL 5
|
||||
binary.BigEndian.PutUint16(pkt[2:4], uint16(ipLen+tcpLen+payLen))
|
||||
binary.BigEndian.PutUint16(pkt[4:6], 0x4242) // ID
|
||||
pkt[8] = 64 // TTL
|
||||
pkt[9] = unix.IPPROTO_TCP
|
||||
copy(pkt[12:16], []byte{10, 0, 0, 1}) // src
|
||||
copy(pkt[16:20], []byte{10, 0, 0, 2}) // dst
|
||||
|
||||
// TCP header.
|
||||
binary.BigEndian.PutUint16(pkt[20:22], 12345) // sport
|
||||
binary.BigEndian.PutUint16(pkt[22:24], 80) // dport
|
||||
binary.BigEndian.PutUint32(pkt[24:28], 10000) // seq
|
||||
binary.BigEndian.PutUint32(pkt[28:32], 20000) // ack
|
||||
pkt[32] = 0x50 // data offset 5 words
|
||||
pkt[33] = 0x18 // ACK | PSH
|
||||
binary.BigEndian.PutUint16(pkt[34:36], 65535) // window
|
||||
|
||||
for i := 0; i < payLen; i++ {
|
||||
pkt[ipLen+tcpLen+i] = byte(i & 0xff)
|
||||
}
|
||||
return pkt, ipLen + tcpLen, ipLen
|
||||
}
|
||||
|
||||
// buildUDPv4Super constructs a synthetic IPv4/UDP USO superpacket with a
|
||||
// payload of payLen bytes. Header is a fixed 28 bytes (20 IPv4 + 8 UDP).
|
||||
func buildUDPv4Super(payLen int) (pkt []byte, hdrLen, csumStart uint16) {
|
||||
const ipLen = 20
|
||||
const udpLen = 8
|
||||
pkt = make([]byte, ipLen+udpLen+payLen)
|
||||
|
||||
pkt[0] = 0x45
|
||||
binary.BigEndian.PutUint16(pkt[2:4], uint16(ipLen+udpLen+payLen))
|
||||
binary.BigEndian.PutUint16(pkt[4:6], 0x4242)
|
||||
pkt[8] = 64
|
||||
pkt[9] = unix.IPPROTO_UDP
|
||||
copy(pkt[12:16], []byte{10, 0, 0, 1})
|
||||
copy(pkt[16:20], []byte{10, 0, 0, 2})
|
||||
|
||||
binary.BigEndian.PutUint16(pkt[20:22], 12345) // sport
|
||||
binary.BigEndian.PutUint16(pkt[22:24], 53) // dport
|
||||
|
||||
for i := 0; i < payLen; i++ {
|
||||
pkt[ipLen+udpLen+i] = byte(i & 0xff)
|
||||
}
|
||||
return pkt, ipLen + udpLen, ipLen
|
||||
}
|
||||
|
||||
// collectTCP segments a fresh copy of pkt and returns each segment as an
|
||||
// independent slice so assertions can run after segmentation completes.
|
||||
func collectTCP(t *testing.T, pkt []byte, hdrLen, csumStart, gsoSize uint16) [][]byte {
|
||||
t.Helper()
|
||||
work := append([]byte(nil), pkt...)
|
||||
var out [][]byte
|
||||
err := SegmentTCP(work, hdrLen, csumStart, gsoSize, func(seg []byte) error {
|
||||
out = append(out, append([]byte(nil), seg...))
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("SegmentTCP: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func collectUDP(t *testing.T, pkt []byte, hdrLen, csumStart, gsoSize uint16) [][]byte {
|
||||
t.Helper()
|
||||
work := append([]byte(nil), pkt...)
|
||||
var out [][]byte
|
||||
err := SegmentUDP(work, hdrLen, csumStart, gsoSize, func(seg []byte) error {
|
||||
out = append(out, append([]byte(nil), seg...))
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("SegmentUDP: %v", err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// TestSegmentTCPHeaderNotCorrupted is the regression test for the in-place
|
||||
// header-slide bug: when gsoSize < headerLen the old code stamped each
|
||||
// segment's header from pkt[:headerLen], which had already been overwritten
|
||||
// by the previous segment's overlapping stamp, so segments 2..n carried a
|
||||
// corrupted header (garbage src/dst/ports/seq). Every segment must instead
|
||||
// carry the ORIGINAL constant header fields with correct per-segment seq.
|
||||
func TestSegmentTCPHeaderNotCorrupted(t *testing.T) {
|
||||
const origSeq = 10000
|
||||
cases := []struct {
|
||||
name string
|
||||
payLen int
|
||||
gsoSize uint16
|
||||
}{
|
||||
// gsoSize (8) < headerLen (40): the bug's trigger. Even split.
|
||||
{"small-gso-even", 40, 8},
|
||||
// gsoSize (8) < headerLen (40) with a short final segment.
|
||||
{"small-gso-odd-tail", 44, 8},
|
||||
// gsoSize (100) >= headerLen (40): the normal path, must still work.
|
||||
{"normal-gso", 250, 100},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
pkt, hdrLen, csumStart := buildTCPv4Super(tc.payLen)
|
||||
gso := int(tc.gsoSize)
|
||||
wantSeg := (tc.payLen + gso - 1) / gso
|
||||
segs := collectTCP(t, pkt, hdrLen, csumStart, tc.gsoSize)
|
||||
if len(segs) != wantSeg {
|
||||
t.Fatalf("got %d segments, want %d", len(segs), wantSeg)
|
||||
}
|
||||
|
||||
off := 0
|
||||
for i, seg := range segs {
|
||||
// Constant header fields must be identical to the original in
|
||||
// EVERY segment. These are exactly the bytes the old code
|
||||
// corrupted in segments 2..n.
|
||||
if got := seg[0]; got != 0x45 {
|
||||
t.Errorf("seg %d: version/IHL byte=%#x want 0x45", i, got)
|
||||
}
|
||||
if seg[9] != unix.IPPROTO_TCP {
|
||||
t.Errorf("seg %d: proto=%d want %d", i, seg[9], unix.IPPROTO_TCP)
|
||||
}
|
||||
if !bytes.Equal(seg[12:16], []byte{10, 0, 0, 1}) {
|
||||
t.Errorf("seg %d: src=%v want [10 0 0 1]", i, seg[12:16])
|
||||
}
|
||||
if !bytes.Equal(seg[16:20], []byte{10, 0, 0, 2}) {
|
||||
t.Errorf("seg %d: dst=%v want [10 0 0 2]", i, seg[16:20])
|
||||
}
|
||||
if sport := binary.BigEndian.Uint16(seg[20:22]); sport != 12345 {
|
||||
t.Errorf("seg %d: sport=%d want 12345", i, sport)
|
||||
}
|
||||
if dport := binary.BigEndian.Uint16(seg[22:24]); dport != 80 {
|
||||
t.Errorf("seg %d: dport=%d want 80", i, dport)
|
||||
}
|
||||
if ack := binary.BigEndian.Uint32(seg[28:32]); ack != 20000 {
|
||||
t.Errorf("seg %d: ack=%d want 20000", i, ack)
|
||||
}
|
||||
if seg[32] != 0x50 {
|
||||
t.Errorf("seg %d: data-offset byte=%#x want 0x50", i, seg[32])
|
||||
}
|
||||
|
||||
// Per-segment seq must advance by the payload offset.
|
||||
segStart := i * gso
|
||||
if seq := binary.BigEndian.Uint32(seg[24:28]); seq != uint32(origSeq+segStart) {
|
||||
t.Errorf("seg %d: seq=%d want %d", i, seq, origSeq+segStart)
|
||||
}
|
||||
|
||||
// Payload bytes must be the original contiguous slice.
|
||||
segPayLen := len(seg) - int(hdrLen)
|
||||
wantPay := make([]byte, segPayLen)
|
||||
for k := 0; k < segPayLen; k++ {
|
||||
wantPay[k] = byte((off + k) & 0xff)
|
||||
}
|
||||
if !bytes.Equal(seg[hdrLen:], wantPay) {
|
||||
t.Errorf("seg %d: payload mismatch", i)
|
||||
}
|
||||
off += segPayLen
|
||||
|
||||
// End-to-end: the stamped header must checksum-verify. A
|
||||
// corrupted header fails here because the written checksum was
|
||||
// derived from the pristine header.
|
||||
if !verifyChecksum(seg[:20], 0) {
|
||||
t.Errorf("seg %d: bad IPv4 header checksum", i)
|
||||
}
|
||||
psum := pseudoHeaderIPv4(seg[12:16], seg[16:20], unix.IPPROTO_TCP, len(seg)-20)
|
||||
if !verifyChecksum(seg[20:], psum) {
|
||||
t.Errorf("seg %d: bad TCP checksum", i)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestCorrectHdrLenChecksumBound guards the checksum-field bounds check in
|
||||
// CorrectHdrLen. The checksum field sits at CsumStart+CsumOffset, so the check
|
||||
// must be computed from CsumStart+CsumOffset — NOT CsumStart+CsumStart, a
|
||||
// regression that doubled CsumStart and thus over-tightened the bound (since
|
||||
// CsumOffset, 6 for UDP / 16 for TCP, is always < CsumStart >= 20). That bogus
|
||||
// bound spuriously rejected valid small USO superpackets in decodeRead.
|
||||
func TestCorrectHdrLenChecksumBound(t *testing.T) {
|
||||
// A valid IPv4 USO superpacket: 20B IPv4 + 8B UDP + two 6-byte segments
|
||||
// (payload 12) = 40 bytes total. CsumStart=20, CsumOffset=6, so the UDP
|
||||
// checksum field lives at bytes 26..27, comfortably inside the 40-byte
|
||||
// packet. The OLD formula computed cSumAt = CsumStart+CsumStart = 40 and
|
||||
// rejected on cSumAt+1 (41) >= len(pkt) (40); the fix (CsumStart+CsumOffset
|
||||
// = 26) accepts. This case FAILS against the CsumStart+CsumStart regression.
|
||||
t.Run("valid-small-uso-accepted", func(t *testing.T) {
|
||||
pkt, _, csumStart := buildUDPv4Super(12) // total len 40
|
||||
hdr := NewHeader(
|
||||
unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, /*flags*/
|
||||
unix.VIRTIO_NET_HDR_GSO_UDP_L4, /*gsoType*/
|
||||
0, /*hdrLen*/
|
||||
6, /*gsoSize: two 6-byte segments*/
|
||||
csumStart, /*csumStart*/
|
||||
6, /*csumOffset*/
|
||||
)
|
||||
if err := CorrectHdrLen(pkt, &hdr); err != nil {
|
||||
t.Fatalf("CorrectHdrLen rejected a valid 40-byte USO superpacket: %v", err)
|
||||
}
|
||||
if hdr.HdrLen != csumStart+udpHeaderLen {
|
||||
t.Errorf("HdrLen = %d, want %d", hdr.HdrLen, csumStart+udpHeaderLen)
|
||||
}
|
||||
})
|
||||
|
||||
// A genuinely-too-short packet: CsumStart=20, CsumOffset=6 means the
|
||||
// checksum field would end at byte 27, but the packet is only 25 bytes
|
||||
// (CsumStart+CsumOffset+2 = 28 > 25). CorrectHdrLen must still reject it.
|
||||
t.Run("too-short-rejected", func(t *testing.T) {
|
||||
pkt := make([]byte, 25)
|
||||
pkt[0] = 0x45 // IPv4, IHL 5
|
||||
hdr := NewHeader(
|
||||
unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, /*flags*/
|
||||
unix.VIRTIO_NET_HDR_GSO_UDP_L4, /*gsoType*/
|
||||
0, /*hdrLen*/
|
||||
6, /*gsoSize*/
|
||||
20, /*csumStart*/
|
||||
6, /*csumOffset*/
|
||||
)
|
||||
if err := CorrectHdrLen(pkt, &hdr); err == nil {
|
||||
t.Fatalf("CorrectHdrLen accepted a too-short (25-byte) packet")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// TestSegmentUDPHeaderNotCorrupted is the USO counterpart: SegmentUDP performs
|
||||
// the same header stamp and must be correct when gsoSize < headerLen.
|
||||
func TestSegmentUDPHeaderNotCorrupted(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
payLen int
|
||||
gsoSize uint16
|
||||
}{
|
||||
{"small-gso-even", 40, 8},
|
||||
{"small-gso-odd-tail", 44, 8},
|
||||
{"normal-gso", 250, 100},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
pkt, hdrLen, csumStart := buildUDPv4Super(tc.payLen)
|
||||
gso := int(tc.gsoSize)
|
||||
wantSeg := (tc.payLen + gso - 1) / gso
|
||||
segs := collectUDP(t, pkt, hdrLen, csumStart, tc.gsoSize)
|
||||
if len(segs) != wantSeg {
|
||||
t.Fatalf("got %d segments, want %d", len(segs), wantSeg)
|
||||
}
|
||||
|
||||
off := 0
|
||||
for i, seg := range segs {
|
||||
if got := seg[0]; got != 0x45 {
|
||||
t.Errorf("seg %d: version/IHL byte=%#x want 0x45", i, got)
|
||||
}
|
||||
if seg[9] != unix.IPPROTO_UDP {
|
||||
t.Errorf("seg %d: proto=%d want %d", i, seg[9], unix.IPPROTO_UDP)
|
||||
}
|
||||
if !bytes.Equal(seg[12:16], []byte{10, 0, 0, 1}) {
|
||||
t.Errorf("seg %d: src=%v want [10 0 0 1]", i, seg[12:16])
|
||||
}
|
||||
if !bytes.Equal(seg[16:20], []byte{10, 0, 0, 2}) {
|
||||
t.Errorf("seg %d: dst=%v want [10 0 0 2]", i, seg[16:20])
|
||||
}
|
||||
if sport := binary.BigEndian.Uint16(seg[20:22]); sport != 12345 {
|
||||
t.Errorf("seg %d: sport=%d want 12345", i, sport)
|
||||
}
|
||||
if dport := binary.BigEndian.Uint16(seg[22:24]); dport != 53 {
|
||||
t.Errorf("seg %d: dport=%d want 53", i, dport)
|
||||
}
|
||||
// Software UDP GSO bumps the IPv4 ID per segment just like TSO
|
||||
// (inet_gso_segment's fixed-ID case is TCP-only).
|
||||
if id := binary.BigEndian.Uint16(seg[4:6]); id != 0x4242+uint16(i) {
|
||||
t.Errorf("seg %d: ip id=%#x want %#x", i, id, 0x4242+uint16(i))
|
||||
}
|
||||
|
||||
segPayLen := len(seg) - int(hdrLen)
|
||||
if udpLen := binary.BigEndian.Uint16(seg[24:26]); udpLen != uint16(8+segPayLen) {
|
||||
t.Errorf("seg %d: udp len=%d want %d", i, udpLen, 8+segPayLen)
|
||||
}
|
||||
|
||||
wantPay := make([]byte, segPayLen)
|
||||
for k := 0; k < segPayLen; k++ {
|
||||
wantPay[k] = byte((off + k) & 0xff)
|
||||
}
|
||||
if !bytes.Equal(seg[hdrLen:], wantPay) {
|
||||
t.Errorf("seg %d: payload mismatch", i)
|
||||
}
|
||||
off += segPayLen
|
||||
|
||||
if !verifyChecksum(seg[:20], 0) {
|
||||
t.Errorf("seg %d: bad IPv4 header checksum", i)
|
||||
}
|
||||
psum := pseudoHeaderIPv4(seg[12:16], seg[16:20], unix.IPPROTO_UDP, len(seg)-20)
|
||||
if !verifyChecksum(seg[20:], psum) {
|
||||
t.Errorf("seg %d: bad UDP checksum", i)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// buildUDPv4Single constructs a single IPv4/UDP datagram with the checksum field pre-loaded with the folded
|
||||
// pseudo-header sum, which is how a NEEDS_CSUM (CHECKSUM_PARTIAL) packet arrives from the tun.
|
||||
func buildUDPv4Single(payload []byte) (pkt []byte, hdr Hdr) {
|
||||
const ipLen, udpLen = 20, 8
|
||||
pkt = make([]byte, ipLen+udpLen+len(payload))
|
||||
|
||||
pkt[0] = 0x45
|
||||
binary.BigEndian.PutUint16(pkt[2:4], uint16(len(pkt)))
|
||||
pkt[8] = 64
|
||||
pkt[9] = unix.IPPROTO_UDP
|
||||
copy(pkt[12:16], []byte{10, 0, 0, 1})
|
||||
copy(pkt[16:20], []byte{10, 0, 0, 2})
|
||||
|
||||
binary.BigEndian.PutUint16(pkt[ipLen:ipLen+2], 12345)
|
||||
binary.BigEndian.PutUint16(pkt[ipLen+2:ipLen+4], 53)
|
||||
binary.BigEndian.PutUint16(pkt[ipLen+4:ipLen+6], uint16(udpLen+len(payload)))
|
||||
copy(pkt[ipLen+udpLen:], payload)
|
||||
|
||||
pseudo := pseudoHeaderIPv4(pkt[12:16], pkt[16:20], unix.IPPROTO_UDP, udpLen+len(payload))
|
||||
binary.BigEndian.PutUint16(pkt[ipLen+udpChecksumOff:ipLen+udpChecksumOff+2], pseudo)
|
||||
|
||||
return pkt, NewHeader(unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, unix.VIRTIO_NET_HDR_GSO_NONE, 0, 0, ipLen, udpChecksumOff)
|
||||
}
|
||||
|
||||
// TestFinishChecksumUDPZeroStoresAllOnes pins RFC 768: a UDP checksum that computes to zero goes on the wire as
|
||||
// 0xffff, because all-zero is the reserved "no checksum" encoding that IPv6 rejects outright.
|
||||
func TestFinishChecksumUDPZeroStoresAllOnes(t *testing.T) {
|
||||
var payload []byte
|
||||
for i := 0; i < 0x10000; i++ {
|
||||
p := []byte{byte(i >> 8), byte(i)}
|
||||
pkt, hdr := buildUDPv4Single(p)
|
||||
cs, co := int(hdr.CsumStart), int(hdr.CsumOffset)
|
||||
partial := binary.BigEndian.Uint16(pkt[cs+co : cs+co+2])
|
||||
pkt[cs+co], pkt[cs+co+1] = 0, 0
|
||||
if ^checksum.Checksum(pkt[cs:], partial) == 0 {
|
||||
payload = p
|
||||
break
|
||||
}
|
||||
}
|
||||
if payload == nil {
|
||||
t.Fatal("no 2-byte payload produced a zero checksum")
|
||||
}
|
||||
|
||||
pkt, hdr := buildUDPv4Single(payload)
|
||||
if err := FinishChecksum(pkt, hdr); err != nil {
|
||||
t.Fatalf("FinishChecksum: %v", err)
|
||||
}
|
||||
off := int(hdr.CsumStart) + int(hdr.CsumOffset)
|
||||
if got := binary.BigEndian.Uint16(pkt[off : off+2]); got != 0xffff {
|
||||
t.Fatalf("stored %#04x, want 0xffff for a UDP checksum that folds to zero", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFinishChecksumTCPZeroPreserved is the other half of the protocol split: zero is a legal TCP checksum and must
|
||||
// be stored as-is, so the UDP rewrite must not fire on a TCP csum_offset.
|
||||
func TestFinishChecksumTCPZeroPreserved(t *testing.T) {
|
||||
const cs, co = 20, tcpChecksumOff
|
||||
|
||||
// Seed the partial so the completed checksum lands on zero, the case the UDP path rewrites.
|
||||
seg := make([]byte, cs+co+2)
|
||||
for i := range seg[cs:] {
|
||||
seg[cs+i] = byte(i * 7)
|
||||
}
|
||||
var partial uint16
|
||||
for i := 0; i <= 0xffff; i++ {
|
||||
binary.BigEndian.PutUint16(seg[cs+co:cs+co+2], uint16(i))
|
||||
probe := append([]byte(nil), seg...)
|
||||
probe[cs+co], probe[cs+co+1] = 0, 0
|
||||
if ^checksum.Checksum(probe[cs:], uint16(i)) == 0 {
|
||||
partial = uint16(i)
|
||||
break
|
||||
}
|
||||
}
|
||||
binary.BigEndian.PutUint16(seg[cs+co:cs+co+2], partial)
|
||||
|
||||
hdr := NewHeader(unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, unix.VIRTIO_NET_HDR_GSO_NONE, 0, 0, cs, co)
|
||||
if err := FinishChecksum(seg, hdr); err != nil {
|
||||
t.Fatalf("FinishChecksum: %v", err)
|
||||
}
|
||||
if got := binary.BigEndian.Uint16(seg[cs+co : cs+co+2]); got != 0 {
|
||||
t.Fatalf("stored %#04x, want 0x0000 (zero is a legal TCP checksum)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFinishChecksumUDPValidates confirms the ordinary path still produces a checksum a receiver accepts.
|
||||
func TestFinishChecksumUDPValidates(t *testing.T) {
|
||||
payload := []byte("the definitive tun offloads branch")
|
||||
pkt, hdr := buildUDPv4Single(payload)
|
||||
if err := FinishChecksum(pkt, hdr); err != nil {
|
||||
t.Fatalf("FinishChecksum: %v", err)
|
||||
}
|
||||
pseudo := pseudoHeaderIPv4(pkt[12:16], pkt[16:20], unix.IPPROTO_UDP, 8+len(payload))
|
||||
if !verifyChecksum(pkt[hdr.CsumStart:], pseudo) {
|
||||
t.Fatal("completed UDP checksum does not validate")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCheckValidMasksGSOECN: GSO_ECN is a qualifier bit the kernel ORs
|
||||
// into gso_type for TSO superpackets with CWR set. CheckValid must
|
||||
// validate an ECN-qualified type as its base type — previously TCPV4|ECN
|
||||
// fell into the default case and skipped the IP-version agreement check.
|
||||
// The qualifier is TCP-only, so it must be rejected on UDP_L4.
|
||||
func TestCheckValidMasksGSOECN(t *testing.T) {
|
||||
v4pkt, _, _ := buildTCPv4Super(100)
|
||||
v6pkt := make([]byte, len(v4pkt))
|
||||
copy(v6pkt, v4pkt)
|
||||
v6pkt[0] = 0x60 // claim IPv6
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
pkt []byte
|
||||
gsoType uint8
|
||||
wantErr bool
|
||||
}{
|
||||
{"tcpv4-ecn-v4", v4pkt, unix.VIRTIO_NET_HDR_GSO_TCPV4 | unix.VIRTIO_NET_HDR_GSO_ECN, false},
|
||||
{"tcpv4-ecn-v6-mismatch", v6pkt, unix.VIRTIO_NET_HDR_GSO_TCPV4 | unix.VIRTIO_NET_HDR_GSO_ECN, true},
|
||||
{"tcpv6-ecn-v4-mismatch", v4pkt, unix.VIRTIO_NET_HDR_GSO_TCPV6 | unix.VIRTIO_NET_HDR_GSO_ECN, true},
|
||||
{"udp-l4-ecn-rejected", v4pkt, unix.VIRTIO_NET_HDR_GSO_UDP_L4 | unix.VIRTIO_NET_HDR_GSO_ECN, true},
|
||||
{"tcpv4-plain-v4", v4pkt, unix.VIRTIO_NET_HDR_GSO_TCPV4, false},
|
||||
{"tcpv4-plain-v6-mismatch", v6pkt, unix.VIRTIO_NET_HDR_GSO_TCPV4, true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
err := CheckValid(tc.pkt, NewHeader(0, tc.gsoType, 0, 100, 0, 0))
|
||||
if tc.wantErr && err == nil {
|
||||
t.Errorf("CheckValid(gsoType=%#x) = nil, want error", tc.gsoType)
|
||||
}
|
||||
if !tc.wantErr && err != nil {
|
||||
t.Errorf("CheckValid(gsoType=%#x) = %v, want nil", tc.gsoType, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestCheckValidRejectsZeroGSOSize: a GSO-typed header with gso_size=0 must be
|
||||
// rejected. It would produce a Packet whose GSOInfo.IsSuperpacket() is false,
|
||||
// dodging both segmentation and FinishChecksum on its way downstream.
|
||||
func TestCheckValidRejectsZeroGSOSize(t *testing.T) {
|
||||
v4pkt, _, _ := buildTCPv4Super(100)
|
||||
if err := CheckValid(v4pkt, NewHeader(0, unix.VIRTIO_NET_HDR_GSO_TCPV4, 0, 0, 0, 0)); err == nil {
|
||||
t.Fatal("CheckValid accepted a GSO-typed header with gso_size=0")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFoldComplementMatchesReference checks the segmenter's fold-and-invert
|
||||
// against an independent RFC 1071 reference fold, hitting the carry edge
|
||||
// cases (values whose first fold produces another carry).
|
||||
func TestFoldComplementMatchesReference(t *testing.T) {
|
||||
refFold := func(s uint64) uint16 {
|
||||
for s>>16 != 0 {
|
||||
s = s&0xffff + s>>16
|
||||
}
|
||||
return uint16(s)
|
||||
}
|
||||
cases := []uint32{
|
||||
0, 1, 0xffff,
|
||||
0x10000, // single carry
|
||||
0x1fffe, // 0xffff + 0xffff: carry produces another 0xffff
|
||||
0xffff0000, // high half only
|
||||
0xfffeffff, // first fold yields another carry
|
||||
0xffffffff, // worst case
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got, want := foldComplement(c), ^refFold(uint64(c)); got != want {
|
||||
t.Errorf("foldComplement(%#x) = %#x, want %#x", c, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// referenceBaseIPv4HdrSum and referenceBaseTCPHdrSum are the straightforward
|
||||
// implementations that baseIPv4HdrSum/baseTCPHdrSum replaced: copy the header
|
||||
// into scratch, zero the fields the segment loop rewrites, sum. The production
|
||||
// versions instead sum in place and subtract those fields via one's-complement
|
||||
// arithmetic, which is faster but far less obvious — particularly for the TCP
|
||||
// flags byte, which is only half of a 16-bit word. These references exist so
|
||||
// that trade is checked rather than asserted.
|
||||
func referenceBaseIPv4HdrSum(pkt []byte, ihl int) uint32 {
|
||||
var ipTmp [ipv4HeaderMaxLen]byte
|
||||
copy(ipTmp[:ihl], pkt[:ihl])
|
||||
ipTmp[ipv4TotalLenOff], ipTmp[ipv4TotalLenOff+1] = 0, 0
|
||||
ipTmp[ipv4ChecksumOff], ipTmp[ipv4ChecksumOff+1] = 0, 0
|
||||
ipTmp[ipv4IDOff], ipTmp[ipv4IDOff+1] = 0, 0
|
||||
return uint32(checksum.Checksum(ipTmp[:ihl], 0))
|
||||
}
|
||||
|
||||
func referenceBaseTCPHdrSum(pkt []byte, csumStart, headerLen int) uint32 {
|
||||
tcpLen := headerLen - csumStart
|
||||
var tmp [tcpHeaderMaxLen]byte
|
||||
copy(tmp[:tcpLen], pkt[csumStart:headerLen])
|
||||
tmp[tcpSeqOff], tmp[tcpSeqOff+1], tmp[tcpSeqOff+2], tmp[tcpSeqOff+3] = 0, 0, 0, 0
|
||||
tmp[tcpFlagsOff] = 0
|
||||
tmp[tcpChecksumOff], tmp[tcpChecksumOff+1] = 0, 0
|
||||
return uint32(checksum.Checksum(tmp[:tcpLen], 0))
|
||||
}
|
||||
|
||||
// randSeed is a tiny deterministic PRNG so this test needs no imports beyond
|
||||
// what the file already has and reproduces identically on every run.
|
||||
func randByte(state *uint32) byte {
|
||||
*state = *state*1664525 + 1013904223
|
||||
return byte(*state >> 24)
|
||||
}
|
||||
|
||||
func TestBaseSumsMatchZeroingReference(t *testing.T) {
|
||||
state := uint32(12345)
|
||||
|
||||
t.Run("ipv4", func(t *testing.T) {
|
||||
for ihl := ipv4HeaderMinLen; ihl <= ipv4HeaderMaxLen; ihl += 4 {
|
||||
for iter := 0; iter < 5000; iter++ {
|
||||
pkt := make([]byte, ihl)
|
||||
for i := range pkt {
|
||||
pkt[i] = randByte(&state)
|
||||
}
|
||||
pkt[0] = byte(0x40 | (ihl / 4))
|
||||
|
||||
want := referenceBaseIPv4HdrSum(pkt, ihl)
|
||||
got, err := baseIPv4HdrSum(pkt, ihl)
|
||||
if err != nil {
|
||||
t.Fatalf("ihl=%d: %v", ihl, err)
|
||||
}
|
||||
// Compare the value that reaches the wire: the raw partial
|
||||
// sums may legally differ by one's-complement -0 vs +0.
|
||||
for _, tl := range []uint32{20, 1500, 65535} {
|
||||
for _, id := range []uint32{0, 0x4242, 0xffff} {
|
||||
if a, b := foldComplement(want+tl+id), foldComplement(got+tl+id); a != b {
|
||||
t.Fatalf("ihl=%d tl=%d id=%d: %#04x != %#04x", ihl, tl, id, a, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("tcp", func(t *testing.T) {
|
||||
const csumStart = 20
|
||||
for dataOff := 5; dataOff <= 15; dataOff++ {
|
||||
tcpLen := dataOff * 4
|
||||
headerLen := csumStart + tcpLen
|
||||
for iter := 0; iter < 5000; iter++ {
|
||||
pkt := make([]byte, headerLen+64)
|
||||
for i := range pkt {
|
||||
pkt[i] = randByte(&state)
|
||||
}
|
||||
pkt[0] = 0x45
|
||||
pkt[csumStart+tcpDataOffOff] = byte(dataOff << 4)
|
||||
|
||||
want := referenceBaseTCPHdrSum(pkt, csumStart, headerLen)
|
||||
got := baseTCPHdrSum(pkt, csumStart, headerLen)
|
||||
for _, seq := range []uint32{0, 1, 0x4242_4242, 0xffff_ffff} {
|
||||
for _, fl := range []uint32{0x00, 0x10, 0x18, 0x19, 0xff} {
|
||||
for _, l4 := range []uint32{20, 1460, 65535} {
|
||||
a := foldComplement(want + seq + fl + l4)
|
||||
b := foldComplement(got + seq + fl + l4)
|
||||
if a != b {
|
||||
t.Fatalf("dataOff=%d seq=%#x fl=%#x l4=%d: %#04x != %#04x",
|
||||
dataOff, seq, fl, l4, a, b)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user