mirror of
https://github.com/slackhq/nebula.git
synced 2026-11-15 07:06:52 +01:00
Ryan Huber
f3f98e1690
Set tun.staged.workers to a positive count on Linux, with tun.use_offloads
enabled, to move encryption off each tun reader. The reader filters and
reserves counters for a whole read, hands the batch to that many encrypt
workers, and an ordered sender flushes the batches in read order. Eight slots
per routine bound the batches in flight; a full pipeline blocks the reader.
Zero workers, the default, keeps today's inline path, as do tun queues that
cannot hand off their read buffer and builds whose AES-GCM needs nonce-ordered
encryption (BoringCrypto, FIPS).
IMPORTANT: staged senders need peers with the 8192-entry replay window of
master. Released v1.11.x peers use 1024 and will drop reordered batches. The
window is not negotiated.
Readers keep today's pinning. The sender is only pinned with an explicit
tun.cpu_affinity: sender i uses entry i+1, unless that is its reader's CPU,
so a one-entry list leaves the sender unpinned rather than sharing it: two
busy transmit threads on one CPU serialise (eight TCP streams over 100 GbE
fell from 25.9 to 22.8 Gbit/s). Without a list the sender is not pinned. Placing it by queue index on a
hybrid CPU put it on an efficiency core, and one TCP stream over 100 GbE
reached 18.6-19.6 Gbit/s pinned against 20.2-20.6 unpinned.
New API, each needed across a package boundary:
- tio.Offload.SwapRxBuf hands the buffer of the last Read to the reader and
reads into the one given. It is a method on the concrete queue, found through
an unexported interface in nebula, so tio.Queue and its other
implementations are unchanged.
- batch.SendBatch.Reset drops queued packets without sending, for a sender
whose sockets are already closed; Flush now uses it.
Metrics, registered only for active pipelines: flow.tx.batches counts reads
handed to the sender, flow.stage.{tx_read,tx_encrypt,tx_send}.q<N>.busy_ns the
elapsed busy time of each stage, to find the limiting one.
Tests cover ordering and counters under backpressure with both ciphers (AES
skipped where nonce order is required), read errors with batches in flight,
shutdown, exhaustion, relays, fallback, firewall parity, and the actual reader
and send thread affinity for every placement, with pinning on and off: the
sender is pinned only to an explicit entry other than its reader's CPU.
1141 lines
37 KiB
Go
1141 lines
37 KiB
Go
//go:build linux && !android && !e2e_testing
|
|
|
|
package tio
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"os"
|
|
"testing"
|
|
|
|
"golang.org/x/sys/unix"
|
|
"gvisor.dev/gvisor/pkg/tcpip/checksum"
|
|
|
|
"github.com/slackhq/nebula/overlay/tio/virtio"
|
|
)
|
|
|
|
// testSegScratchSize is a generous segmentation scratch sized to fit any
|
|
// of the synthetic TSO/USO superpackets these tests generate (one
|
|
// worst-case 64 KiB superpacket plus replicated per-segment headers).
|
|
const testSegScratchSize = 192 * 1024
|
|
|
|
// TestProtoFromGSOTypeMasksECN guards the CWR-superpacket drop bug: the
|
|
// kernel qualifies a TSO superpacket whose TCP header carries CWR with
|
|
// VIRTIO_NET_HDR_GSO_ECN (we negotiate TUN_F_TSO_ECN, so it WILL send
|
|
// them once ECN feedback flows), and the decoder must mask that bit
|
|
// rather than reject the packet as an unknown type.
|
|
func TestProtoFromGSOTypeMasksECN(t *testing.T) {
|
|
cases := []struct {
|
|
typ uint8
|
|
want GSOProto
|
|
}{
|
|
{unix.VIRTIO_NET_HDR_GSO_TCPV4, GSOProtoTCP},
|
|
{unix.VIRTIO_NET_HDR_GSO_TCPV4 | unix.VIRTIO_NET_HDR_GSO_ECN, GSOProtoTCP},
|
|
{unix.VIRTIO_NET_HDR_GSO_TCPV6, GSOProtoTCP},
|
|
{unix.VIRTIO_NET_HDR_GSO_TCPV6 | unix.VIRTIO_NET_HDR_GSO_ECN, GSOProtoTCP},
|
|
{unix.VIRTIO_NET_HDR_GSO_UDP_L4, GSOProtoUDP},
|
|
}
|
|
for _, c := range cases {
|
|
got, err := protoFromGSOType(c.typ)
|
|
if err != nil || got != c.want {
|
|
t.Errorf("protoFromGSOType(%#x) = (%v, %v), want (%v, nil)", c.typ, got, err, c.want)
|
|
}
|
|
}
|
|
if _, err := protoFromGSOType(unix.VIRTIO_NET_HDR_GSO_NONE); err == nil {
|
|
t.Error("GSO_NONE must still be rejected")
|
|
}
|
|
if _, err := protoFromGSOType(unix.VIRTIO_NET_HDR_GSO_ECN); err == nil {
|
|
t.Error("a bare ECN bit with no base type must still be rejected")
|
|
}
|
|
}
|
|
|
|
// verifyChecksum confirms that the one's-complement sum across `b`, seeded
|
|
// with a folded pseudo-header sum, equals all-ones (valid).
|
|
func verifyChecksum(b []byte, pseudo uint16) bool {
|
|
return checksum.Checksum(b, pseudo) == 0xffff
|
|
}
|
|
|
|
// segmentForTest is the test-only counterpart to the production
|
|
// SegmentSuperpacket path. It handles GSO_NONE (with optional
|
|
// finishChecksum) inline and dispatches GSO superpackets through
|
|
// SegmentSuperpacket, draining each yielded segment into a
|
|
// freshly-copied [][]byte slot so callers can iterate after the call
|
|
// returns. Tests pre-set hdr.HdrLen correctly, so correctHdrLen is not
|
|
// invoked here.
|
|
func segmentForTest(pkt []byte, hdr virtio.Hdr, out *[][]byte, scratch []byte) error {
|
|
if hdr.GSOType() == unix.VIRTIO_NET_HDR_GSO_NONE {
|
|
cp := append([]byte(nil), pkt...)
|
|
if hdr.Flags&unix.VIRTIO_NET_HDR_F_NEEDS_CSUM != 0 {
|
|
if err := virtio.FinishChecksum(cp, hdr); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
*out = append(*out, cp)
|
|
return nil
|
|
}
|
|
proto, err := protoFromGSOType(hdr.GSOType())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
gso := GSOInfo{
|
|
Size: hdr.GSOSize,
|
|
HdrLen: hdr.HdrLen,
|
|
CsumStart: hdr.CsumStart,
|
|
Proto: proto,
|
|
}
|
|
return SegmentSuperpacket(Packet{Bytes: pkt, GSO: gso}, func(seg []byte) error {
|
|
*out = append(*out, append([]byte(nil), seg...))
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// pseudoHeaderIPv4 returns the folded pseudo-header sum used to verify a
|
|
// TCP/UDP segment's checksum in tests. src/dst are 4 bytes each.
|
|
func pseudoHeaderIPv4(src, dst []byte, proto byte, l4Len int) uint16 {
|
|
s := uint32(checksum.Checksum(src, 0)) + uint32(checksum.Checksum(dst, 0))
|
|
s += uint32(proto) + uint32(l4Len)
|
|
s = (s & 0xffff) + (s >> 16)
|
|
s = (s & 0xffff) + (s >> 16)
|
|
return uint16(s)
|
|
}
|
|
|
|
// pseudoHeaderIPv6 returns the folded pseudo-header sum used to verify a
|
|
// TCP/UDP segment's checksum in tests. src/dst are 16 bytes each.
|
|
func pseudoHeaderIPv6(src, dst []byte, proto byte, l4Len int) uint16 {
|
|
s := uint32(checksum.Checksum(src, 0)) + uint32(checksum.Checksum(dst, 0))
|
|
s += uint32(l4Len>>16) + uint32(l4Len&0xffff) + uint32(proto)
|
|
s = (s & 0xffff) + (s >> 16)
|
|
s = (s & 0xffff) + (s >> 16)
|
|
return uint16(s)
|
|
}
|
|
|
|
// buildTSOv4 builds a synthetic IPv4/TCP TSO superpacket with a payload of
|
|
// `payLen` bytes split at `mss`.
|
|
func buildTSOv4(t *testing.T, payLen, mss int) ([]byte, virtio.Hdr) {
|
|
t.Helper()
|
|
const ipLen = 20
|
|
const tcpLen = 20
|
|
pkt := make([]byte, ipLen+tcpLen+payLen)
|
|
|
|
// IPv4 header
|
|
pkt[0] = 0x45 // version 4, IHL 5
|
|
// total length is meaningless for TSO but set it anyway
|
|
binary.BigEndian.PutUint16(pkt[2:4], uint16(ipLen+tcpLen+payLen))
|
|
binary.BigEndian.PutUint16(pkt[4:6], 0x4242) // original ID
|
|
pkt[8] = 64 // TTL
|
|
pkt[9] = unix.IPPROTO_TCP
|
|
copy(pkt[12:16], []byte{10, 0, 0, 1}) // src
|
|
copy(pkt[16:20], []byte{10, 0, 0, 2}) // dst
|
|
|
|
// TCP header
|
|
binary.BigEndian.PutUint16(pkt[20:22], 12345) // sport
|
|
binary.BigEndian.PutUint16(pkt[22:24], 80) // dport
|
|
binary.BigEndian.PutUint32(pkt[24:28], 10000) // seq
|
|
binary.BigEndian.PutUint32(pkt[28:32], 20000) // ack
|
|
pkt[32] = 0x50 // data offset 5 words
|
|
pkt[33] = 0x18 // ACK | PSH
|
|
binary.BigEndian.PutUint16(pkt[34:36], 65535) // window
|
|
|
|
// payload
|
|
for i := range payLen {
|
|
pkt[ipLen+tcpLen+i] = byte(i & 0xff)
|
|
}
|
|
return pkt, virtio.NewHeader(
|
|
unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, /*flags*/
|
|
unix.VIRTIO_NET_HDR_GSO_TCPV4, /*gsoType*/
|
|
uint16(ipLen+tcpLen), /*hdrLen*/
|
|
uint16(mss), /*gsoSize*/
|
|
uint16(ipLen), /*csumStart*/
|
|
16, /*csumOffset*/
|
|
)
|
|
}
|
|
|
|
func TestSegmentTCPv4(t *testing.T) {
|
|
const mss = 100
|
|
const numSeg = 3
|
|
pkt, hdr := buildTSOv4(t, mss*numSeg, mss)
|
|
|
|
scratch := make([]byte, testSegScratchSize)
|
|
var out [][]byte
|
|
if err := segmentForTest(pkt, hdr, &out, scratch); err != nil {
|
|
t.Fatalf("segmentForTest: %v", err)
|
|
}
|
|
if len(out) != numSeg {
|
|
t.Fatalf("expected %d segments, got %d", numSeg, len(out))
|
|
}
|
|
|
|
for i, seg := range out {
|
|
if len(seg) != 40+mss {
|
|
t.Errorf("seg %d: unexpected len %d", i, len(seg))
|
|
}
|
|
totalLen := binary.BigEndian.Uint16(seg[2:4])
|
|
if totalLen != uint16(40+mss) {
|
|
t.Errorf("seg %d: total_len=%d want %d", i, totalLen, 40+mss)
|
|
}
|
|
id := binary.BigEndian.Uint16(seg[4:6])
|
|
if id != 0x4242+uint16(i) {
|
|
t.Errorf("seg %d: ip id=%#x want %#x", i, id, 0x4242+uint16(i))
|
|
}
|
|
seq := binary.BigEndian.Uint32(seg[24:28])
|
|
wantSeq := uint32(10000 + i*mss)
|
|
if seq != wantSeq {
|
|
t.Errorf("seg %d: seq=%d want %d", i, seq, wantSeq)
|
|
}
|
|
flags := seg[33]
|
|
wantFlags := byte(0x10) // ACK only, PSH cleared
|
|
if i == numSeg-1 {
|
|
wantFlags = 0x18 // ACK | PSH preserved on last
|
|
}
|
|
if flags != wantFlags {
|
|
t.Errorf("seg %d: flags=%#x want %#x", i, flags, wantFlags)
|
|
}
|
|
// IPv4 header checksum must verify against itself.
|
|
if !verifyChecksum(seg[:20], 0) {
|
|
t.Errorf("seg %d: bad IPv4 header checksum", i)
|
|
}
|
|
// TCP checksum must verify against the pseudo-header.
|
|
psum := pseudoHeaderIPv4(seg[12:16], seg[16:20], unix.IPPROTO_TCP, 20+mss)
|
|
if !verifyChecksum(seg[20:], psum) {
|
|
t.Errorf("seg %d: bad TCP checksum", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSegmentTCPv4OddTail(t *testing.T) {
|
|
// Payload of 250 bytes with MSS 100 → segments of 100, 100, 50.
|
|
pkt, hdr := buildTSOv4(t, 250, 100)
|
|
scratch := make([]byte, testSegScratchSize)
|
|
var out [][]byte
|
|
if err := segmentForTest(pkt, hdr, &out, scratch); err != nil {
|
|
t.Fatalf("segmentForTest: %v", err)
|
|
}
|
|
if len(out) != 3 {
|
|
t.Fatalf("want 3 segments, got %d", len(out))
|
|
}
|
|
wantPayLens := []int{100, 100, 50}
|
|
for i, seg := range out {
|
|
if len(seg)-40 != wantPayLens[i] {
|
|
t.Errorf("seg %d: pay len %d want %d", i, len(seg)-40, wantPayLens[i])
|
|
}
|
|
if !verifyChecksum(seg[:20], 0) {
|
|
t.Errorf("seg %d: bad IPv4 header checksum", i)
|
|
}
|
|
psum := pseudoHeaderIPv4(seg[12:16], seg[16:20], unix.IPPROTO_TCP, 20+wantPayLens[i])
|
|
if !verifyChecksum(seg[20:], psum) {
|
|
t.Errorf("seg %d: bad TCP checksum", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSegmentTCPv6(t *testing.T) {
|
|
const ipLen = 40
|
|
const tcpLen = 20
|
|
const mss = 120
|
|
const numSeg = 2
|
|
payLen := mss * numSeg
|
|
pkt := make([]byte, ipLen+tcpLen+payLen)
|
|
|
|
// IPv6 header
|
|
pkt[0] = 0x60 // version 6
|
|
binary.BigEndian.PutUint16(pkt[4:6], uint16(tcpLen+payLen))
|
|
pkt[6] = unix.IPPROTO_TCP
|
|
pkt[7] = 64
|
|
// src/dst fe80::1 / fe80::2
|
|
pkt[8] = 0xfe
|
|
pkt[9] = 0x80
|
|
pkt[23] = 1
|
|
pkt[24] = 0xfe
|
|
pkt[25] = 0x80
|
|
pkt[39] = 2
|
|
|
|
// TCP header
|
|
binary.BigEndian.PutUint16(pkt[40:42], 12345)
|
|
binary.BigEndian.PutUint16(pkt[42:44], 80)
|
|
binary.BigEndian.PutUint32(pkt[44:48], 7)
|
|
binary.BigEndian.PutUint32(pkt[48:52], 99)
|
|
pkt[52] = 0x50
|
|
pkt[53] = 0x19 // FIN | ACK | PSH — exercise FIN clearing too
|
|
binary.BigEndian.PutUint16(pkt[54:56], 65535)
|
|
|
|
for i := range payLen {
|
|
pkt[ipLen+tcpLen+i] = byte(i)
|
|
}
|
|
|
|
hdr := virtio.NewHeader(
|
|
unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, /*flags*/
|
|
unix.VIRTIO_NET_HDR_GSO_TCPV6, /*gsoType*/
|
|
uint16(ipLen+tcpLen), /*hdrLen*/
|
|
uint16(mss), /*gsoSize*/
|
|
uint16(ipLen), /*csumStart*/
|
|
16, /*csumOffset*/
|
|
)
|
|
|
|
scratch := make([]byte, testSegScratchSize)
|
|
var out [][]byte
|
|
if err := segmentForTest(pkt, hdr, &out, scratch); err != nil {
|
|
t.Fatalf("segmentForTest: %v", err)
|
|
}
|
|
if len(out) != numSeg {
|
|
t.Fatalf("want %d segments, got %d", numSeg, len(out))
|
|
}
|
|
|
|
for i, seg := range out {
|
|
if len(seg) != ipLen+tcpLen+mss {
|
|
t.Errorf("seg %d: len %d want %d", i, len(seg), ipLen+tcpLen+mss)
|
|
}
|
|
pl := binary.BigEndian.Uint16(seg[4:6])
|
|
if pl != uint16(tcpLen+mss) {
|
|
t.Errorf("seg %d: payload_length=%d want %d", i, pl, tcpLen+mss)
|
|
}
|
|
seq := binary.BigEndian.Uint32(seg[44:48])
|
|
if seq != uint32(7+i*mss) {
|
|
t.Errorf("seg %d: seq=%d want %d", i, seq, 7+i*mss)
|
|
}
|
|
flags := seg[53]
|
|
// Original flags = 0x19 (FIN|ACK|PSH). FIN(0x01)+PSH(0x08) should be
|
|
// cleared on all but the last; ACK(0x10) always preserved.
|
|
wantFlags := byte(0x10)
|
|
if i == numSeg-1 {
|
|
wantFlags = 0x19
|
|
}
|
|
if flags != wantFlags {
|
|
t.Errorf("seg %d: flags=%#x want %#x", i, flags, wantFlags)
|
|
}
|
|
psum := pseudoHeaderIPv6(seg[8:24], seg[24:40], unix.IPPROTO_TCP, tcpLen+mss)
|
|
if !verifyChecksum(seg[ipLen:], psum) {
|
|
t.Errorf("seg %d: bad TCP checksum", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSegmentGSONonePassesThrough(t *testing.T) {
|
|
pkt, hdr := buildTSOv4(t, 100, 100)
|
|
hdr.SetGSOType(unix.VIRTIO_NET_HDR_GSO_NONE)
|
|
hdr.Flags = 0 // no NEEDS_CSUM, leave packet untouched
|
|
|
|
scratch := make([]byte, testSegScratchSize)
|
|
var out [][]byte
|
|
if err := segmentForTest(pkt, hdr, &out, scratch); err != nil {
|
|
t.Fatalf("segmentForTest: %v", err)
|
|
}
|
|
if len(out) != 1 {
|
|
t.Fatalf("want 1 segment, got %d", len(out))
|
|
}
|
|
if len(out[0]) != len(pkt) {
|
|
t.Fatalf("unexpected length: %d vs %d", len(out[0]), len(pkt))
|
|
}
|
|
}
|
|
|
|
// TestSegmentRejectsLegacyUDPGSO ensures the legacy GSO_UDP (UFO) marker is
|
|
// still rejected; only modern GSO_UDP_L4 (USO) is supported.
|
|
func TestSegmentRejectsLegacyUDPGSO(t *testing.T) {
|
|
hdr := virtio.NewHeader(0, unix.VIRTIO_NET_HDR_GSO_UDP, 0, 0, 0, 0)
|
|
var out [][]byte
|
|
if err := segmentForTest(nil, hdr, &out, nil); err == nil {
|
|
t.Fatalf("expected rejection for legacy UDP GSO")
|
|
}
|
|
}
|
|
|
|
// buildUSOv4 builds a synthetic IPv4/UDP USO superpacket with payload of
|
|
// payLen bytes, segmented at gsoSize.
|
|
func buildUSOv4(t *testing.T, payLen, gsoSize int) ([]byte, virtio.Hdr) {
|
|
t.Helper()
|
|
const ipLen = 20
|
|
const udpLen = 8
|
|
pkt := make([]byte, ipLen+udpLen+payLen)
|
|
|
|
// IPv4 header
|
|
pkt[0] = 0x45 // version 4, IHL 5
|
|
binary.BigEndian.PutUint16(pkt[2:4], uint16(ipLen+udpLen+payLen))
|
|
binary.BigEndian.PutUint16(pkt[4:6], 0x4242)
|
|
pkt[8] = 64
|
|
pkt[9] = unix.IPPROTO_UDP
|
|
copy(pkt[12:16], []byte{10, 0, 0, 1})
|
|
copy(pkt[16:20], []byte{10, 0, 0, 2})
|
|
|
|
// UDP header. The kernel hands us a USO superpacket whose length field
|
|
// covers the WHOLE superpacket; the segmenter overwrites it per segment.
|
|
// Populating it here matters: leaving it zero makes the base-checksum path
|
|
// that must exclude it untestable, since excluding zero is a no-op.
|
|
binary.BigEndian.PutUint16(pkt[20:22], 12345) // sport
|
|
binary.BigEndian.PutUint16(pkt[22:24], 53) // dport
|
|
binary.BigEndian.PutUint16(pkt[24:26], uint16(udpLen+payLen)) // superpacket length
|
|
|
|
for i := range payLen {
|
|
pkt[ipLen+udpLen+i] = byte(i & 0xff)
|
|
}
|
|
|
|
return pkt, virtio.NewHeader(
|
|
unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, /*flags*/
|
|
unix.VIRTIO_NET_HDR_GSO_UDP_L4, /*gsoType*/
|
|
uint16(ipLen+udpLen), /*hdrLen*/
|
|
uint16(gsoSize), /*gsoSize*/
|
|
uint16(ipLen), /*csumStart*/
|
|
6, /*csumOffset*/
|
|
)
|
|
}
|
|
|
|
func TestSegmentUDPv4(t *testing.T) {
|
|
const gso = 100
|
|
const numSeg = 3
|
|
pkt, hdr := buildUSOv4(t, gso*numSeg, gso)
|
|
|
|
scratch := make([]byte, testSegScratchSize)
|
|
var out [][]byte
|
|
if err := segmentForTest(pkt, hdr, &out, scratch); err != nil {
|
|
t.Fatalf("segmentForTest: %v", err)
|
|
}
|
|
if len(out) != numSeg {
|
|
t.Fatalf("expected %d segments, got %d", numSeg, len(out))
|
|
}
|
|
|
|
for i, seg := range out {
|
|
if len(seg) != 28+gso {
|
|
t.Errorf("seg %d: len %d want %d", i, len(seg), 28+gso)
|
|
}
|
|
totalLen := binary.BigEndian.Uint16(seg[2:4])
|
|
if totalLen != uint16(28+gso) {
|
|
t.Errorf("seg %d: total_len=%d want %d", i, totalLen, 28+gso)
|
|
}
|
|
// Software UDP GSO bumps the IPv4 ID per segment exactly like TSO
|
|
// (inet_gso_segment's fixed-ID case is TCP-only); wireguard-go's
|
|
// gsoSplit increments unconditionally too.
|
|
id := binary.BigEndian.Uint16(seg[4:6])
|
|
if id != 0x4242+uint16(i) {
|
|
t.Errorf("seg %d: ip id=%#x want %#x", i, id, 0x4242+uint16(i))
|
|
}
|
|
udpLen := binary.BigEndian.Uint16(seg[24:26])
|
|
if udpLen != uint16(8+gso) {
|
|
t.Errorf("seg %d: udp len=%d want %d", i, udpLen, 8+gso)
|
|
}
|
|
if !verifyChecksum(seg[:20], 0) {
|
|
t.Errorf("seg %d: bad IPv4 header checksum", i)
|
|
}
|
|
psum := pseudoHeaderIPv4(seg[12:16], seg[16:20], unix.IPPROTO_UDP, 8+gso)
|
|
if !verifyChecksum(seg[20:], psum) {
|
|
t.Errorf("seg %d: bad UDP checksum", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSegmentUDPv4OddTail(t *testing.T) {
|
|
// 250 bytes payload, gsoSize=100 → segments of 100, 100, 50.
|
|
pkt, hdr := buildUSOv4(t, 250, 100)
|
|
scratch := make([]byte, testSegScratchSize)
|
|
var out [][]byte
|
|
if err := segmentForTest(pkt, hdr, &out, scratch); err != nil {
|
|
t.Fatalf("segmentForTest: %v", err)
|
|
}
|
|
if len(out) != 3 {
|
|
t.Fatalf("want 3 segments, got %d", len(out))
|
|
}
|
|
wantPay := []int{100, 100, 50}
|
|
for i, seg := range out {
|
|
if len(seg)-28 != wantPay[i] {
|
|
t.Errorf("seg %d: pay len %d want %d", i, len(seg)-28, wantPay[i])
|
|
}
|
|
udpLen := binary.BigEndian.Uint16(seg[24:26])
|
|
if udpLen != uint16(8+wantPay[i]) {
|
|
t.Errorf("seg %d: udp len=%d want %d", i, udpLen, 8+wantPay[i])
|
|
}
|
|
if !verifyChecksum(seg[:20], 0) {
|
|
t.Errorf("seg %d: bad IPv4 header checksum", i)
|
|
}
|
|
psum := pseudoHeaderIPv4(seg[12:16], seg[16:20], unix.IPPROTO_UDP, 8+wantPay[i])
|
|
if !verifyChecksum(seg[20:], psum) {
|
|
t.Errorf("seg %d: bad UDP checksum", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestSegmentUDPv6(t *testing.T) {
|
|
const ipLen = 40
|
|
const udpLen = 8
|
|
const gso = 120
|
|
const numSeg = 2
|
|
payLen := gso * numSeg
|
|
pkt := make([]byte, ipLen+udpLen+payLen)
|
|
|
|
// IPv6 header
|
|
pkt[0] = 0x60
|
|
binary.BigEndian.PutUint16(pkt[4:6], uint16(udpLen+payLen))
|
|
pkt[6] = unix.IPPROTO_UDP
|
|
pkt[7] = 64
|
|
pkt[8] = 0xfe
|
|
pkt[9] = 0x80
|
|
pkt[23] = 1
|
|
pkt[24] = 0xfe
|
|
pkt[25] = 0x80
|
|
pkt[39] = 2
|
|
|
|
binary.BigEndian.PutUint16(pkt[40:42], 12345)
|
|
binary.BigEndian.PutUint16(pkt[42:44], 53)
|
|
// Superpacket-wide length, as the kernel supplies it; see buildUSOv4.
|
|
binary.BigEndian.PutUint16(pkt[44:46], uint16(udpLen+payLen))
|
|
|
|
for i := range payLen {
|
|
pkt[ipLen+udpLen+i] = byte(i)
|
|
}
|
|
|
|
hdr := virtio.NewHeader(
|
|
unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, /*flags*/
|
|
unix.VIRTIO_NET_HDR_GSO_UDP_L4, /*gsoType*/
|
|
uint16(ipLen+udpLen), /*hdrLen*/
|
|
uint16(gso), /*gsoSize*/
|
|
uint16(ipLen), /*csumStart*/
|
|
6, /*csumOffset*/
|
|
)
|
|
|
|
scratch := make([]byte, testSegScratchSize)
|
|
var out [][]byte
|
|
if err := segmentForTest(pkt, hdr, &out, scratch); err != nil {
|
|
t.Fatalf("segmentForTest: %v", err)
|
|
}
|
|
if len(out) != numSeg {
|
|
t.Fatalf("want %d segments, got %d", numSeg, len(out))
|
|
}
|
|
|
|
for i, seg := range out {
|
|
if len(seg) != ipLen+udpLen+gso {
|
|
t.Errorf("seg %d: len %d want %d", i, len(seg), ipLen+udpLen+gso)
|
|
}
|
|
pl := binary.BigEndian.Uint16(seg[4:6])
|
|
if pl != uint16(udpLen+gso) {
|
|
t.Errorf("seg %d: payload_length=%d want %d", i, pl, udpLen+gso)
|
|
}
|
|
ul := binary.BigEndian.Uint16(seg[ipLen+4 : ipLen+6])
|
|
if ul != uint16(udpLen+gso) {
|
|
t.Errorf("seg %d: udp len=%d want %d", i, ul, udpLen+gso)
|
|
}
|
|
psum := pseudoHeaderIPv6(seg[8:24], seg[24:40], unix.IPPROTO_UDP, udpLen+gso)
|
|
if !verifyChecksum(seg[ipLen:], psum) {
|
|
t.Errorf("seg %d: bad UDP checksum", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestSegmentUDPCEPropagates confirms IP-level CE marks on the seed appear on
|
|
// every segment. UDP has no transport-level CWR/ECE: the IP TOS/TC byte is
|
|
// copied verbatim into every segment by the segment-prefix copy.
|
|
func TestSegmentUDPCEPropagates(t *testing.T) {
|
|
pkt, hdr := buildUSOv4(t, 200, 100)
|
|
pkt[1] = 0x03 // CE codepoint in IP-ECN
|
|
|
|
scratch := make([]byte, testSegScratchSize)
|
|
var out [][]byte
|
|
if err := segmentForTest(pkt, hdr, &out, scratch); err != nil {
|
|
t.Fatalf("segmentForTest: %v", err)
|
|
}
|
|
if len(out) != 2 {
|
|
t.Fatalf("want 2 segments, got %d", len(out))
|
|
}
|
|
for i, seg := range out {
|
|
if seg[1]&0x03 != 0x03 {
|
|
t.Errorf("seg %d: CE missing (tos=%#x)", i, seg[1])
|
|
}
|
|
if !verifyChecksum(seg[:20], 0) {
|
|
t.Errorf("seg %d: bad IPv4 header checksum", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestSegmentTCPCwrFirstSegmentOnly confirms RFC 3168 §6.1.2: when a TSO
|
|
// burst's seed has CWR set, only the first emitted segment carries CWR.
|
|
// ECE is preserved on every segment (different signal, persistent state).
|
|
func TestSegmentTCPCwrFirstSegmentOnly(t *testing.T) {
|
|
const mss = 100
|
|
const numSeg = 3
|
|
pkt, hdr := buildTSOv4(t, mss*numSeg, mss)
|
|
// Seed flags: CWR | ECE | ACK | PSH.
|
|
pkt[33] = 0x80 | 0x40 | 0x10 | 0x08
|
|
|
|
scratch := make([]byte, testSegScratchSize)
|
|
var out [][]byte
|
|
if err := segmentForTest(pkt, hdr, &out, scratch); err != nil {
|
|
t.Fatalf("segmentForTest: %v", err)
|
|
}
|
|
if len(out) != numSeg {
|
|
t.Fatalf("expected %d segments, got %d", numSeg, len(out))
|
|
}
|
|
for i, seg := range out {
|
|
flags := seg[33]
|
|
hasCwr := flags&0x80 != 0
|
|
hasEce := flags&0x40 != 0
|
|
hasPsh := flags&0x08 != 0
|
|
wantCwr := i == 0
|
|
wantPsh := i == numSeg-1
|
|
if hasCwr != wantCwr {
|
|
t.Errorf("seg %d: CWR=%v want %v (flags=%#x)", i, hasCwr, wantCwr, flags)
|
|
}
|
|
if !hasEce {
|
|
t.Errorf("seg %d: ECE missing (flags=%#x)", i, flags)
|
|
}
|
|
if hasPsh != wantPsh {
|
|
t.Errorf("seg %d: PSH=%v want %v (flags=%#x)", i, hasPsh, wantPsh, flags)
|
|
}
|
|
// IP and TCP checksums must still verify after the flag rewrite.
|
|
if !verifyChecksum(seg[:20], 0) {
|
|
t.Errorf("seg %d: bad IPv4 header checksum", i)
|
|
}
|
|
psum := pseudoHeaderIPv4(seg[12:16], seg[16:20], unix.IPPROTO_TCP, 20+mss)
|
|
if !verifyChecksum(seg[20:], psum) {
|
|
t.Errorf("seg %d: bad TCP checksum", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
func BenchmarkSegmentTCPv4(b *testing.B) {
|
|
sizes := []struct {
|
|
name string
|
|
payLen int
|
|
mss int
|
|
}{
|
|
{"64KiB_MSS1460", 65000, 1460},
|
|
{"16KiB_MSS1460", 16384, 1460},
|
|
{"4KiB_MSS1460", 4096, 1460},
|
|
}
|
|
for _, sz := range sizes {
|
|
b.Run(sz.name, func(b *testing.B) {
|
|
const ipLen = 20
|
|
const tcpLen = 20
|
|
pkt := make([]byte, ipLen+tcpLen+sz.payLen)
|
|
pkt[0] = 0x45
|
|
binary.BigEndian.PutUint16(pkt[2:4], uint16(ipLen+tcpLen+sz.payLen))
|
|
binary.BigEndian.PutUint16(pkt[4:6], 0x4242)
|
|
pkt[8] = 64
|
|
pkt[9] = unix.IPPROTO_TCP
|
|
copy(pkt[12:16], []byte{10, 0, 0, 1})
|
|
copy(pkt[16:20], []byte{10, 0, 0, 2})
|
|
binary.BigEndian.PutUint16(pkt[20:22], 12345)
|
|
binary.BigEndian.PutUint16(pkt[22:24], 80)
|
|
binary.BigEndian.PutUint32(pkt[24:28], 10000)
|
|
binary.BigEndian.PutUint32(pkt[28:32], 20000)
|
|
pkt[32] = 0x50
|
|
pkt[33] = 0x18
|
|
binary.BigEndian.PutUint16(pkt[34:36], 65535)
|
|
for i := 0; i < sz.payLen; i++ {
|
|
pkt[ipLen+tcpLen+i] = byte(i)
|
|
}
|
|
hdr := virtio.NewHeader(
|
|
unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, /*flags*/
|
|
unix.VIRTIO_NET_HDR_GSO_TCPV4, /*gsoType*/
|
|
uint16(ipLen+tcpLen), /*hdrLen*/
|
|
uint16(sz.mss), /*gsoSize*/
|
|
uint16(ipLen), /*csumStart*/
|
|
16, /*csumOffset*/
|
|
)
|
|
|
|
scratch := make([]byte, testSegScratchSize)
|
|
out := make([][]byte, 0, 64)
|
|
|
|
// SegmentSuperpacket consumes its input destructively; restore
|
|
// pkt from a master copy each iteration. The restore mirrors the
|
|
// kernel→userspace copy that hands a fresh GSO blob to the
|
|
// segmenter in production, so it's representative cost rather
|
|
// than bench overhead.
|
|
master := append([]byte(nil), pkt...)
|
|
work := make([]byte, len(pkt))
|
|
|
|
b.SetBytes(int64(len(pkt)))
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
copy(work, master)
|
|
out = out[:0]
|
|
if err := segmentForTest(work, hdr, &out, scratch); err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestTunFileWriteVnetHdrNoAlloc verifies the IFF_VNET_HDR fast-path write is
|
|
// allocation-free. We write to /dev/null so every call succeeds synchronously.
|
|
func TestTunFileWriteVnetHdrNoAlloc(t *testing.T) {
|
|
fd, err := unix.Open("/dev/null", os.O_WRONLY, 0)
|
|
if err != nil {
|
|
t.Fatalf("open /dev/null: %v", err)
|
|
}
|
|
t.Cleanup(func() { _ = unix.Close(fd) })
|
|
|
|
tf := &Offload{fd: fd}
|
|
|
|
payload := make([]byte, 1400)
|
|
// Warm up (first call may trigger one-time internal allocations elsewhere).
|
|
if _, err := tf.Write(payload); err != nil {
|
|
t.Fatalf("Write: %v", err)
|
|
}
|
|
|
|
allocs := testing.AllocsPerRun(1000, func() {
|
|
if _, err := tf.Write(payload); err != nil {
|
|
t.Fatalf("Write: %v", err)
|
|
}
|
|
})
|
|
if allocs != 0 {
|
|
t.Fatalf("Write allocated %.1f times per call, want 0", allocs)
|
|
}
|
|
}
|
|
|
|
// TestSegmentSuperpacketNoAlloc pins the segmenters' zero-allocation
|
|
// contract. Both SegmentTCP and SegmentUDP derive their per-superpacket
|
|
// constants into fixed-size arrays (tmp/ipTmp/savedHdr) that must stay on
|
|
// the stack, and both take a yield closure that must not escape. Any of
|
|
// those escaping turns one allocation into one-per-superpacket on the
|
|
// hottest path in the reader, which BenchmarkSegmentSuperpacketAllocsTSO
|
|
// reports but nothing fails on. This does.
|
|
//
|
|
// The yield closure here only touches captured scalars: appending segments
|
|
// to a slice would allocate in the test itself and mask the measurement.
|
|
func TestSegmentSuperpacketNoAlloc(t *testing.T) {
|
|
const mss = 1400
|
|
const numSeg = 8
|
|
|
|
cases := []struct {
|
|
name string
|
|
build func() ([]byte, virtio.Hdr)
|
|
}{
|
|
{"tso-v4", func() ([]byte, virtio.Hdr) { return buildTSOv4(t, mss*numSeg, mss) }},
|
|
{"uso-v4", func() ([]byte, virtio.Hdr) { return buildUSOv4(t, mss*numSeg, mss) }},
|
|
}
|
|
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
master, hdr := tc.build()
|
|
proto, err := protoFromGSOType(hdr.GSOType())
|
|
if err != nil {
|
|
t.Fatalf("protoFromGSOType: %v", err)
|
|
}
|
|
work := make([]byte, len(master))
|
|
p := Packet{Bytes: work, GSO: GSOInfo{
|
|
Size: hdr.GSOSize,
|
|
HdrLen: hdr.HdrLen,
|
|
CsumStart: hdr.CsumStart,
|
|
Proto: proto,
|
|
}}
|
|
|
|
// Segmentation consumes its input destructively, so restore from
|
|
// the master copy each run; copy(2) into an existing slice does
|
|
// not allocate. seen/bytes keep the closure from being optimized
|
|
// away and double as a sanity check that work actually happened.
|
|
var seen, bytes int
|
|
run := func() {
|
|
copy(work, master)
|
|
seen, bytes = 0, 0
|
|
if err := SegmentSuperpacket(p, func(seg []byte) error {
|
|
seen++
|
|
bytes += len(seg)
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatalf("SegmentSuperpacket: %v", err)
|
|
}
|
|
}
|
|
|
|
run() // warm up: absorb any one-time allocation elsewhere
|
|
if seen != numSeg {
|
|
t.Fatalf("yielded %d segments, want %d", seen, numSeg)
|
|
}
|
|
|
|
if allocs := testing.AllocsPerRun(200, run); allocs != 0 {
|
|
t.Fatalf("SegmentSuperpacket allocated %.1f times per call, want 0", allocs)
|
|
}
|
|
if seen != numSeg || bytes == 0 {
|
|
t.Fatalf("post-measure sanity: seen=%d bytes=%d", seen, bytes)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// buildTSOv6 builds a synthetic IPv6/TCP TSO superpacket with payLen bytes
|
|
// of payload, segmented at gso. Returns the packet bytes only; the
|
|
// virtio_net_hdr is the caller's responsibility.
|
|
func buildTSOv6(payLen, gso int) []byte {
|
|
const ipLen = 40
|
|
const tcpLen = 20
|
|
pkt := make([]byte, ipLen+tcpLen+payLen)
|
|
|
|
pkt[0] = 0x60 // version 6
|
|
binary.BigEndian.PutUint16(pkt[4:6], uint16(tcpLen+payLen))
|
|
pkt[6] = unix.IPPROTO_TCP
|
|
pkt[7] = 64
|
|
pkt[8] = 0xfe
|
|
pkt[9] = 0x80
|
|
pkt[23] = 1
|
|
pkt[24] = 0xfe
|
|
pkt[25] = 0x80
|
|
pkt[39] = 2
|
|
|
|
binary.BigEndian.PutUint16(pkt[40:42], 12345)
|
|
binary.BigEndian.PutUint16(pkt[42:44], 80)
|
|
binary.BigEndian.PutUint32(pkt[44:48], 7)
|
|
binary.BigEndian.PutUint32(pkt[48:52], 99)
|
|
pkt[52] = 0x50
|
|
pkt[53] = 0x10 // ACK only
|
|
binary.BigEndian.PutUint16(pkt[54:56], 65535)
|
|
|
|
for i := range payLen {
|
|
pkt[ipLen+tcpLen+i] = byte(i)
|
|
}
|
|
return pkt
|
|
}
|
|
|
|
// TestDecodeReadFitsMaxTSOAtDrainThreshold proves the rxBuf sizing is
|
|
// correct: when rxOff is at the maximum value the drain headroom check
|
|
// allows, decodeRead must still be able to absorb a worst-case 64KiB
|
|
// TSO superpacket without dropping the burst. With segmentation deferred
|
|
// to encrypt time, decodeRead writes only the kernel-supplied bytes into
|
|
// rxBuf, so the size requirement is just "fit one worst-case input."
|
|
//
|
|
// Regression history: in a prior layout the rx buffer doubled as the
|
|
// segmentation output, a near-threshold drain read returned "scratch too
|
|
// small", the whole 45-segment TSO burst was dropped, and the remote's TCP
|
|
// fast-retransmit collapsed cwnd. Keeping this test in the new layout
|
|
// guards against re-introducing a drain headroom shortfall.
|
|
func TestDecodeReadFitsMaxTSOAtDrainThreshold(t *testing.T) {
|
|
const ipv6HdrLen = 40
|
|
const tcpHdrLen = 20
|
|
const headerLen = ipv6HdrLen + tcpHdrLen
|
|
// Maximum TUN read body at the drain threshold. readv bounds the body
|
|
// iovec by the space actually left in rxBuf, and the drain gate keeps that
|
|
// at >= tunRxBufSize, so that is the largest superpacket the kernel can
|
|
// hand back on the last permitted drain read.
|
|
pktLen := tunRxBufSize
|
|
payLen := pktLen - headerLen
|
|
const targetSegs = 64
|
|
gsoSize := (payLen + targetSegs - 1) / targetSegs
|
|
|
|
pkt := buildTSOv6(payLen, gsoSize)
|
|
if len(pkt) != pktLen {
|
|
t.Fatalf("buildTSOv6 produced %d bytes, want %d", len(pkt), pktLen)
|
|
}
|
|
|
|
o := &Offload{
|
|
rxBuf: make([]byte, tunRxBufCap),
|
|
}
|
|
// rxOff at the maximum value the drain headroom check permits before
|
|
// it would refuse another read. Any drain-time read up to this
|
|
// threshold MUST still process correctly.
|
|
o.rxOff = tunRxBufCap - tunRxBufSize
|
|
|
|
// Stage the body in rxBuf as if readv(2) just placed it there.
|
|
copy(o.rxBuf[o.rxOff:], pkt)
|
|
|
|
// Encode the matching virtio_net_hdr.
|
|
hdr := virtio.NewHeader(
|
|
unix.VIRTIO_NET_HDR_F_NEEDS_CSUM, /*flags*/
|
|
unix.VIRTIO_NET_HDR_GSO_TCPV6, /*gsoType*/
|
|
uint16(headerLen), /*hdrLen*/
|
|
uint16(gsoSize), /*gsoSize*/
|
|
uint16(ipv6HdrLen), /*csumStart*/
|
|
16, /*csumOffset*/
|
|
)
|
|
hdr.Encode(o.readVnetScratch[:])
|
|
|
|
startRxOff := o.rxOff
|
|
if err := o.decodeRead(pktLen); err != nil {
|
|
t.Fatalf("decodeRead at drain threshold returned %v — rxBuf sizing regression: "+
|
|
"tunRxBufSize=%d must hold one worst-case input (%d)",
|
|
err, tunRxBufSize, pktLen)
|
|
}
|
|
|
|
if len(o.pending) != 1 {
|
|
t.Fatalf("got %d packets, want 1 superpacket entry", len(o.pending))
|
|
}
|
|
got := o.pending[0]
|
|
if !got.GSO.IsSuperpacket() {
|
|
t.Fatalf("expected superpacket GSO metadata, got %+v", got.GSO)
|
|
}
|
|
if got.GSO.Proto != GSOProtoTCP {
|
|
t.Errorf("GSO.Proto=%d want TCP", got.GSO.Proto)
|
|
}
|
|
if got.GSO.Size != uint16(gsoSize) {
|
|
t.Errorf("GSO.Size=%d want %d", got.GSO.Size, gsoSize)
|
|
}
|
|
if got.GSO.HdrLen != uint16(headerLen) {
|
|
t.Errorf("GSO.HdrLen=%d want %d", got.GSO.HdrLen, headerLen)
|
|
}
|
|
if got.GSO.CsumStart != uint16(ipv6HdrLen) {
|
|
t.Errorf("GSO.CsumStart=%d want %d", got.GSO.CsumStart, ipv6HdrLen)
|
|
}
|
|
if len(got.Bytes) != pktLen {
|
|
t.Errorf("len(Bytes)=%d want %d", len(got.Bytes), pktLen)
|
|
}
|
|
|
|
// rxOff advances exactly by the kernel-supplied body length — no
|
|
// segmentation output to account for any more.
|
|
if o.rxOff != startRxOff+pktLen {
|
|
t.Errorf("rxOff=%d want %d", o.rxOff, startRxOff+pktLen)
|
|
}
|
|
if o.rxOff > tunRxBufCap {
|
|
t.Fatalf("rxOff=%d overran rxBuf (cap=%d)", o.rxOff, tunRxBufCap)
|
|
}
|
|
|
|
// Validate that segmenting the returned superpacket reproduces the
|
|
// expected per-segment IPv6 payload length and TCP checksum.
|
|
wantSegs := (payLen + gsoSize - 1) / gsoSize
|
|
gotSegs := 0
|
|
if err := SegmentSuperpacket(got, func(seg []byte) error {
|
|
defer func() { gotSegs++ }()
|
|
if len(seg) < headerLen+1 {
|
|
t.Errorf("seg %d too short: %d", gotSegs, len(seg))
|
|
return nil
|
|
}
|
|
if seg[0]>>4 != 6 {
|
|
t.Errorf("seg %d: bad IP version %#x", gotSegs, seg[0])
|
|
}
|
|
segPay := len(seg) - headerLen
|
|
gotPL := binary.BigEndian.Uint16(seg[4:6])
|
|
if gotPL != uint16(tcpHdrLen+segPay) {
|
|
t.Errorf("seg %d: payload_len=%d want %d", gotSegs, gotPL, tcpHdrLen+segPay)
|
|
}
|
|
psum := pseudoHeaderIPv6(seg[8:24], seg[24:40], unix.IPPROTO_TCP, tcpHdrLen+segPay)
|
|
if !verifyChecksum(seg[ipv6HdrLen:], psum) {
|
|
t.Errorf("seg %d: bad TCP checksum", gotSegs)
|
|
}
|
|
return nil
|
|
}); err != nil {
|
|
t.Fatalf("SegmentSuperpacket: %v", err)
|
|
}
|
|
if gotSegs != wantSegs {
|
|
t.Fatalf("got %d segments, want %d", gotSegs, wantSegs)
|
|
}
|
|
}
|
|
|
|
// TestOffloadWriteZeroLength: a zero-length Write must be a no-op, not a
|
|
// panic. The guard used to live below the &buf[0] that tripped on it.
|
|
func TestOffloadWriteZeroLength(t *testing.T) {
|
|
tf := &Offload{fd: -1} // any write reaching the fd would fail loudly
|
|
for _, buf := range [][]byte{nil, {}} {
|
|
n, err := tf.Write(buf)
|
|
if n != 0 || err != nil {
|
|
t.Errorf("Write(len=0) = (%d, %v), want (0, nil)", n, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestOffloadSwapRxBuf: SwapRxBuf hands over the buffer the last Read filled and reads into the one it is given.
|
|
func TestOffloadSwapRxBuf(t *testing.T) {
|
|
o := &Offload{rxBuf: make([]byte, tunRxBufCap)}
|
|
read := o.rxBuf
|
|
owned := o.SwapRxBuf(nil)
|
|
if &owned[0] != &read[0] || len(o.rxBuf) != tunRxBufCap || &o.rxBuf[0] == &read[0] {
|
|
t.Fatal("SwapRxBuf(nil) must hand over the last read buffer and allocate a new one")
|
|
}
|
|
if back := o.SwapRxBuf(owned); &back[0] == &owned[0] || &o.rxBuf[0] != &owned[0] {
|
|
t.Fatal("SwapRxBuf must install the buffer it is given")
|
|
}
|
|
defer func() {
|
|
if recover() == nil {
|
|
t.Fatal("SwapRxBuf must reject a short buffer")
|
|
}
|
|
}()
|
|
o.SwapRxBuf(make([]byte, 10))
|
|
}
|
|
|
|
// TestWriteGSOSuperpacketGeometry decodes the vnet header the kernel would see for a multi-segment write:
|
|
// the GSO type must match the proto and IP version
|
|
// gso_size must be the per-segment size (the kernel rejects a superpacket with gso_size == 0),
|
|
// and the csum fields must point at the transport header's checksum slot.
|
|
// Write through a pipe so the bytes can be read back and decoded.
|
|
func TestWriteGSOSuperpacketGeometry(t *testing.T) {
|
|
var pfds [2]int
|
|
if err := unix.Pipe(pfds[:]); err != nil {
|
|
t.Fatalf("pipe: %v", err)
|
|
}
|
|
t.Cleanup(func() { unix.Close(pfds[0]); unix.Close(pfds[1]) })
|
|
|
|
o := &Offload{fd: pfds[1], gsoIovs: make([]unix.Iovec, 2, gsoMaxIovs)}
|
|
o.gsoIovs[0].Base = &o.gsoHdrBuf[0]
|
|
o.gsoIovs[0].SetLen(virtio.Size)
|
|
|
|
ipHdr := make([]byte, 20)
|
|
ipHdr[0] = 0x45
|
|
udpHdr := make([]byte, 8)
|
|
seg := make([]byte, 1200)
|
|
|
|
if err := o.WriteGSO(ipHdr, udpHdr, [][]byte{seg, seg}, GSOProtoUDP); err != nil {
|
|
t.Fatalf("WriteGSO: %v", err)
|
|
}
|
|
|
|
buf := make([]byte, virtio.Size+len(ipHdr)+len(udpHdr)+2*len(seg)+64)
|
|
n, err := unix.Read(pfds[0], buf)
|
|
if err != nil {
|
|
t.Fatalf("read pipe: %v", err)
|
|
}
|
|
var vhdr virtio.Hdr
|
|
vhdr.Decode(buf[:virtio.Size])
|
|
if vhdr.GSOType() != unix.VIRTIO_NET_HDR_GSO_UDP_L4 {
|
|
t.Errorf("gsoType=%d want UDP_L4", vhdr.GSOType())
|
|
}
|
|
if vhdr.GSOSize != 1200 {
|
|
t.Errorf("GSOSize=%d want 1200 (per-segment size from pays[0])", vhdr.GSOSize)
|
|
}
|
|
if vhdr.HdrLen != uint16(len(ipHdr)+len(udpHdr)) {
|
|
t.Errorf("HdrLen=%d want %d", vhdr.HdrLen, len(ipHdr)+len(udpHdr))
|
|
}
|
|
if vhdr.CsumStart != uint16(len(ipHdr)) || vhdr.CsumOffset != 6 {
|
|
t.Errorf("csum start/offset = %d/%d want %d/6", vhdr.CsumStart, vhdr.CsumOffset, len(ipHdr))
|
|
}
|
|
if want := virtio.Size + len(ipHdr) + len(udpHdr) + 2*len(seg); n != want {
|
|
t.Errorf("wrote %d bytes want %d", n, want)
|
|
}
|
|
}
|
|
|
|
// TestWriteGSORejectsBadGeometry pins the length-check contracts
|
|
func TestWriteGSORejectsBadGeometry(t *testing.T) {
|
|
fd, err := unix.Open("/dev/null", os.O_WRONLY, 0)
|
|
if err != nil {
|
|
t.Fatalf("open /dev/null: %v", err)
|
|
}
|
|
t.Cleanup(func() { _ = unix.Close(fd) })
|
|
|
|
o := &Offload{fd: fd, gsoIovs: make([]unix.Iovec, 2, gsoMaxIovs)}
|
|
o.gsoIovs[0].Base = &o.gsoHdrBuf[0]
|
|
o.gsoIovs[0].SetLen(virtio.Size)
|
|
|
|
ipHdr := make([]byte, 20)
|
|
ipHdr[0] = 0x45
|
|
udpHdr := make([]byte, 8)
|
|
tcpHdr := make([]byte, 20)
|
|
seg := make([]byte, 1200)
|
|
|
|
cases := []struct {
|
|
name string
|
|
hdr, thdr []byte
|
|
pays [][]byte
|
|
proto GSOProto
|
|
wantErr bool
|
|
}{
|
|
{"empty-ip-hdr-with-payload", nil, udpHdr, [][]byte{seg}, GSOProtoUDP, true},
|
|
{"udp-transport-too-short-for-csum", ipHdr, udpHdr[:6], [][]byte{seg}, GSOProtoUDP, true},
|
|
{"tcp-transport-too-short-for-csum", ipHdr, tcpHdr[:16], [][]byte{seg}, GSOProtoTCP, true},
|
|
{"superpacket-over-65535", ipHdr, tcpHdr, [][]byte{make([]byte, 40000), make([]byte, 40000)}, GSOProtoTCP, true},
|
|
{"sole-payload-empty", ipHdr, udpHdr, [][]byte{{}}, GSOProtoUDP, true},
|
|
{"leading-empty-fragment", ipHdr, udpHdr, [][]byte{{}, seg, seg}, GSOProtoUDP, true},
|
|
{"trailing-empty-fragment", ipHdr, tcpHdr, [][]byte{seg, {}}, GSOProtoTCP, true},
|
|
{"oversize-middle-fragment", ipHdr, udpHdr, [][]byte{seg, make([]byte, 1201), seg}, GSOProtoUDP, true},
|
|
{"undersize-middle-fragment", ipHdr, udpHdr, [][]byte{seg, make([]byte, 100), seg}, GSOProtoUDP, true},
|
|
{"oversize-last-fragment", ipHdr, tcpHdr, [][]byte{seg, make([]byte, 1201)}, GSOProtoTCP, true},
|
|
{"short-last-fragment-ok", ipHdr, udpHdr, [][]byte{seg, seg, make([]byte, 100)}, GSOProtoUDP, false},
|
|
{"multi-segment-bad-ip-version", []byte{0x05}, udpHdr, [][]byte{seg, seg}, GSOProtoUDP, true},
|
|
{"single-segment-bad-ip-version-ok", []byte{0x05}, udpHdr, [][]byte{seg}, GSOProtoUDP, false},
|
|
{"no-pays-noop", ipHdr, udpHdr, nil, GSOProtoUDP, false},
|
|
{"valid-udp", ipHdr, udpHdr, [][]byte{seg, seg}, GSOProtoUDP, false},
|
|
{"valid-tcp", ipHdr, tcpHdr, [][]byte{seg, seg}, GSOProtoTCP, false},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
err := o.WriteGSO(tc.hdr, tc.thdr, tc.pays, tc.proto)
|
|
if tc.wantErr && err == nil {
|
|
t.Errorf("WriteGSO = nil, want error")
|
|
}
|
|
if !tc.wantErr && err != nil {
|
|
t.Errorf("WriteGSO = %v, want nil", err)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// BenchmarkSegmentUDPv4 is the USO counterpart to BenchmarkSegmentTCPv4. The
|
|
// yield is a no-op so the measurement is segmentation plus checksum work only.
|
|
func BenchmarkSegmentUDPv4(b *testing.B) {
|
|
sizes := []struct {
|
|
name string
|
|
payLen int
|
|
gsoSize int
|
|
}{
|
|
{"64KiB_GSO1400", 64000, 1400},
|
|
{"16KiB_GSO1400", 16384, 1400},
|
|
{"4KiB_GSO1400", 4096, 1400},
|
|
}
|
|
for _, sz := range sizes {
|
|
b.Run(sz.name, func(b *testing.B) {
|
|
const ipLen = 20
|
|
const udpLen = 8
|
|
pkt := make([]byte, ipLen+udpLen+sz.payLen)
|
|
pkt[0] = 0x45
|
|
binary.BigEndian.PutUint16(pkt[2:4], uint16(ipLen+udpLen+sz.payLen))
|
|
binary.BigEndian.PutUint16(pkt[4:6], 0x4242)
|
|
pkt[8] = 64
|
|
pkt[9] = unix.IPPROTO_UDP
|
|
copy(pkt[12:16], []byte{10, 0, 0, 1})
|
|
copy(pkt[16:20], []byte{10, 0, 0, 2})
|
|
binary.BigEndian.PutUint16(pkt[20:22], 12345)
|
|
binary.BigEndian.PutUint16(pkt[22:24], 53)
|
|
binary.BigEndian.PutUint16(pkt[24:26], uint16(udpLen+sz.payLen))
|
|
for i := 0; i < sz.payLen; i++ {
|
|
pkt[ipLen+udpLen+i] = byte(i)
|
|
}
|
|
|
|
master := append([]byte(nil), pkt...)
|
|
work := make([]byte, len(pkt))
|
|
p := Packet{Bytes: work, GSO: GSOInfo{
|
|
Size: uint16(sz.gsoSize),
|
|
HdrLen: ipLen + udpLen,
|
|
CsumStart: ipLen,
|
|
Proto: GSOProtoUDP,
|
|
}}
|
|
|
|
b.SetBytes(int64(len(pkt)))
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
copy(work, master)
|
|
if err := SegmentSuperpacket(p, func(seg []byte) error { return nil }); err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// BenchmarkSegmentUDPv6 mirrors BenchmarkSegmentUDPv4 for IPv6, where the
|
|
// pseudo-header address sum is 32 bytes rather than 8.
|
|
func BenchmarkSegmentUDPv6(b *testing.B) {
|
|
sizes := []struct {
|
|
name string
|
|
payLen int
|
|
gsoSize int
|
|
}{
|
|
{"64KiB_GSO1400", 64000, 1400},
|
|
{"16KiB_GSO1400", 16384, 1400},
|
|
{"4KiB_GSO1400", 4096, 1400},
|
|
}
|
|
for _, sz := range sizes {
|
|
b.Run(sz.name, func(b *testing.B) {
|
|
const ipLen = 40
|
|
const udpLen = 8
|
|
pkt := make([]byte, ipLen+udpLen+sz.payLen)
|
|
pkt[0] = 0x60
|
|
binary.BigEndian.PutUint16(pkt[4:6], uint16(udpLen+sz.payLen))
|
|
pkt[6] = unix.IPPROTO_UDP
|
|
pkt[7] = 64
|
|
pkt[8], pkt[9], pkt[23] = 0xfe, 0x80, 1
|
|
pkt[24], pkt[25], pkt[39] = 0xfe, 0x80, 2
|
|
binary.BigEndian.PutUint16(pkt[40:42], 12345)
|
|
binary.BigEndian.PutUint16(pkt[42:44], 53)
|
|
binary.BigEndian.PutUint16(pkt[44:46], uint16(udpLen+sz.payLen))
|
|
for i := 0; i < sz.payLen; i++ {
|
|
pkt[ipLen+udpLen+i] = byte(i)
|
|
}
|
|
|
|
master := append([]byte(nil), pkt...)
|
|
work := make([]byte, len(pkt))
|
|
p := Packet{Bytes: work, GSO: GSOInfo{
|
|
Size: uint16(sz.gsoSize),
|
|
HdrLen: ipLen + udpLen,
|
|
CsumStart: ipLen,
|
|
Proto: GSOProtoUDP,
|
|
}}
|
|
|
|
b.SetBytes(int64(len(pkt)))
|
|
b.ReportAllocs()
|
|
b.ResetTimer()
|
|
for i := 0; i < b.N; i++ {
|
|
copy(work, master)
|
|
if err := SegmentSuperpacket(p, func(seg []byte) error { return nil }); err != nil {
|
|
b.Fatal(err)
|
|
}
|
|
}
|
|
})
|
|
}
|
|
}
|